electria/omnisearch
Watch
1
0
Fork
You've already forked omnisearch
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'indev'

Browse source
This commit is contained in:
frosty 2026-08-16 23:45:43 -04:00
commit e352971bc5
No known key found for this signature in database
GPG key ID: 32DCAC76C4388DBF
21 changed files with 763 additions and 519 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Makefile
View file

@ -7,8 +7,9 @@ GIT_HASH := $(shell git rev-parse --short HEAD)
GIT_DATE := $(shell git log -1 --format='%ad' --date='format:%y.%m.%d')
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
GIT_REMOTE := $(shell git remote get-url origin)
GIT_TAG := $(shell git describe --tags --exact-match HEAD 2>/dev/null)
VERSION := $(GIT_DATE)+$(GIT_HASH)_$(GIT_BRANCH)
VERSION := $(if $(GIT_TAG),$(GIT_TAG),$(GIT_DATE)+$(GIT_HASH)_$(GIT_BRANCH))
CFLAGS := -Wall -Wextra -O2 -Isrc -DVERSION='"$(VERSION)"' -DGIT_REMOTE='"$(GIT_REMOTE)"'

5
example-config.ini
View file

@ -31,8 +31,11 @@ port = 8087
[engines]
# Use * for all engines, or specify comma-separated list (e.g., ddg,yahoo)
# Use *,-engine to exclude specific engines (e.g., *,-startpage)
# Available engines: ddg, startpage, yahoo, mojeek
# Available engines: ddg, startpage, yahoo, mojeek, yacy
engines="*"
# YaCy instance to use for the yacy engine (optional, default: http://127.0.0.1:8090)
# Requires "yacy" to be in the engines list above.
#yacy_instance = http://127.0.0.1:8090
[rate_limit]
# Rate limit searches per interval

2
locales/en_gb.ini
View file

@ -21,6 +21,8 @@ display_language_label = "Display Language"
language_desc = "Choose your preferred language."
engines_label = "Search Engines"
engines_desc = "Choose which search engines to use. Only engines enabled on the server are shown."
default_engine_label = "Default Search Engine"
all_engines_option = "All enabled engines"
save_settings_button = "Save Settings"
no_results = "No results found"
error_images = "Error fetching images"

2
locales/en_us.ini
View file

@ -21,6 +21,8 @@ display_language_label = "Display Language"
language_desc = "Choose your preferred language."
engines_label = "Search Engines"
engines_desc = "Choose which search engines to use. Only engines enabled on the server are shown."
default_engine_label = "Default Search Engine"
all_engines_option = "All enabled engines"
save_settings_button = "Save Settings"
no_results = "No results found"
error_images = "Error fetching images"

34
locales/fi_fi.ini Normal file
View file

@ -0,0 +1,34 @@
[Meta]
Id = "fi_fi"
Name = "Suomi"
Direction = "ltr"
[Keys]
search_placeholder = "Hae netistä..."
search_button = "Hae"
surprise_me_button = "Kokeilen onneani"
all_tab = "Kaikki"
images_tab = "Kuvat"
settings_tab = "Asetukset"
settings_title = "Asetukset"
theme_label = "Ulkonäkö"
theme_desc = "Valitse haluamasi väriteema."
theme_system = "Järjestelmä"
theme_light = "Vaalea"
theme_dark = "Tumma"
language_label = "Kieli"
display_language_label = "Näyttökieli"
language_desc = "Valitse haluamasi kieli."
engines_label = "Hakukoneet"
engines_desc = "Valitse mitä hakukoneita käytetään. Vain palvelimella käyttöön otetut hakukoneet näytetään."
save_settings_button = "Tallenna asetukset"
no_results = "Ei tuloksia"
error_images = "Virhe kuvien haussa"
rate_limit = "Hidasta hieman! Sinulta on tullut liian monta hakupyyntöä!"
warning_fetch_error = "pyyntö epäonnistui ennen kuin OmniSearch ehti lukea hakutuloksia."
warning_parse_mismatch = "palautti hakutulokset muodossa jota OmniSearch ei osaa jäsentää."
warning_blocked = "palautti captchan tai muun estosivun hakutulosten sijaan."
read_more = "Lue Lisää"
view_cached = "Välimuistissa"
view_image = "Kuva"
visit_site = "Sivu"

64
src/Config.c
View file

@ -3,6 +3,17 @@
#include <stdlib.h>
#include <string.h>
static char *trim(char *value) {
while (*value == ' ' || *value == '\t')
value++;
char *end = value + strlen(value);
while (end > value && (end[-1] == ' ' || end[-1] == '\t'))
end--;
*end = '\0';
return value;
}
int load_config(const char *filename, Config *config) {
FILE *file = fopen(filename, "r");
if (!file) {
@ -33,51 +44,31 @@ int load_config(const char *filename, Config *config) {
char *delimiter = strchr(line, '=');
if (delimiter) {
*delimiter = '\0';
char *key = line;
char *value = delimiter + 1;
char *key = trim(line);
char *value = trim(delimiter + 1);
while (*key == ' ' || *key == '\t')
key++;
while (*value == ' ' || *value == '\t')
value++;
char *key_end = key + strlen(key) - 1;
while (key_end > key && (*key_end == ' ' || *key_end == '\t')) {
*key_end = '\0';
key_end--;
}
char *value_end = value + strlen(value) - 1;
while (value_end > value && (*value_end == ' ' || *value_end == '\t' ||
*value_end == '"' || *value_end == '\'')) {
*value_end = '\0';
value_end--;
}
while (*value == ' ' || *value == '\t')
value++;
char *value_end = value + strlen(value);
while (value_end > value &&
(value_end[-1] == '"' || value_end[-1] == '\''))
*--value_end = '\0';
while (*value == '"' || *value == '\'')
value++;
if (strcmp(section, "server") == 0) {
if (strcmp(key, "host") == 0) {
strncpy(config->host, value, sizeof(config->host) - 1);
config->host[sizeof(config->host) - 1] = '\0';
snprintf(config->host, sizeof(config->host), "%s", value);
} else if (strcmp(key, "port") == 0) {
config->port = atoi(value);
} else if (strcmp(key, "locale") == 0) {
strncpy(config->default_locale, value,
sizeof(config->default_locale) - 1);
config->default_locale[sizeof(config->default_locale) - 1] = '\0';
snprintf(config->default_locale, sizeof(config->default_locale), "%s",
value);
}
} else if (strcmp(section, "proxy") == 0) {
if (strcmp(key, "proxy") == 0) {
strncpy(config->proxy, value, sizeof(config->proxy) - 1);
config->proxy[sizeof(config->proxy) - 1] = '\0';
snprintf(config->proxy, sizeof(config->proxy), "%s", value);
} else if (strcmp(key, "list_file") == 0) {
strncpy(config->proxy_list_file, value,
sizeof(config->proxy_list_file) - 1);
config->proxy_list_file[sizeof(config->proxy_list_file) - 1] = '\0';
snprintf(config->proxy_list_file, sizeof(config->proxy_list_file),
"%s", value);
} else if (strcmp(key, "max_retries") == 0) {
config->max_proxy_retries = atoi(value);
} else if (strcmp(key, "randomize_username") == 0) {
@ -87,8 +78,7 @@ int load_config(const char *filename, Config *config) {
}
} else if (strcmp(section, "cache") == 0) {
if (strcmp(key, "dir") == 0) {
strncpy(config->cache_dir, value, sizeof(config->cache_dir) - 1);
config->cache_dir[sizeof(config->cache_dir) - 1] = '\0';
snprintf(config->cache_dir, sizeof(config->cache_dir), "%s", value);
} else if (strcmp(key, "ttl_search") == 0) {
config->cache_ttl_search = atoi(value);
} else if (strcmp(key, "ttl_infobox") == 0) {
@ -98,8 +88,10 @@ int load_config(const char *filename, Config *config) {
}
} else if (strcmp(section, "engines") == 0) {
if (strcmp(key, "engines") == 0) {
strncpy(config->engines, value, sizeof(config->engines) - 1);
config->engines[sizeof(config->engines) - 1] = '\0';
snprintf(config->engines, sizeof(config->engines), "%s", value);
} else if (strcmp(key, "yacy_instance") == 0) {
snprintf(config->yacy_instance, sizeof(config->yacy_instance), "%s",
value);
}
} else if (strcmp(section, "rate_limit") == 0) {
if (strcmp(key, "search_requests") == 0) {

2
src/Config.h
View file

@ -8,6 +8,7 @@
#define DEFAULT_CACHE_TTL_INFOBOX 86400
#define DEFAULT_CACHE_TTL_IMAGE 604800
#define DEFAULT_MAX_PROXY_RETRIES 3
#define DEFAULT_YACY_INSTANCE "http://127.0.0.1:8090"
#define BUFFER_SIZE_SMALL 256
#define BUFFER_SIZE_MEDIUM 512
@ -46,6 +47,7 @@ typedef struct {
int cache_ttl_infobox;
int cache_ttl_image;
char engines[512];
char yacy_instance[512];
int rate_limit_search_requests;
int rate_limit_search_interval;
int rate_limit_images_requests;

18
src/Main.c
View file

@ -78,6 +78,7 @@ int main() {
.cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX,
.cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE,
.engines = "",
.yacy_instance = "",
.rate_limit_search_requests = 0,
.rate_limit_search_interval = 0,
.rate_limit_images_requests = 0,
@ -101,6 +102,23 @@ int main() {
}
apply_engines_config(cfg.engines);
configure_yacy_engine(cfg.yacy_instance);
if (cfg.yacy_instance[0] != '\0') {
int yacy_enabled = 0;
for (int i = 0; i < ENGINE_COUNT; i++) {
if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
yacy_enabled = ENGINE_REGISTRY[i].enabled;
break;
}
}
if (!yacy_enabled) {
fprintf(
stderr,
"[INFO] YaCy instance configured but the yacy engine is not "
"enabled (add it to the engines list, e.g. engines=\"*,yacy\")\n");
}
}
if (cache_init(cfg.cache_dir) != 0) {
fprintf(stderr,

85
src/Routes/Images.c
View file

@ -76,14 +76,12 @@ int images_handler(UrlParams *params) {
if (!error_images_msg)
error_images_msg = "Error fetching images";
char ***pager_matrix = NULL;
int *pager_inner_counts = NULL;
int pager_count =
build_pagination(page, images_href_builder, (void *)raw_query,
&pager_matrix, &pager_inner_counts);
if (pager_count > 0) {
context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix,
pager_count, pager_inner_counts);
StringMatrix pagination;
string_matrix_build_pagination(page, images_href_builder, (void *)raw_query,
&pagination);
if (pagination.count > 0) {
context_set_array_of_arrays(&ctx, "pagination_links", pagination.rows,
pagination.count, pagination.field_counts);
}
char *display_query = url_decode_query(raw_query);
@ -91,8 +89,9 @@ int images_handler(UrlParams *params) {
if (!raw_query || strlen(raw_query) == 0) {
send_redirect("/");
if (display_query)
free(display_query);
string_matrix_free(&pagination);
free(display_query);
free(locale);
free_context(&ctx);
return -1;
}
@ -128,7 +127,9 @@ int images_handler(UrlParams *params) {
snprintf(response, sizeof(response), "<h1>%s</h1>", rate_limit_msg);
send_response(response);
free(request_cache_key);
string_matrix_free(&pagination);
free(display_query);
free(locale);
free_context(&ctx);
return -1;
}
@ -147,37 +148,37 @@ int images_handler(UrlParams *params) {
snprintf(error_html, sizeof(error_html), "<h1>%s</h1>", error_images_msg);
send_response(error_html);
free(request_cache_key);
string_matrix_free(&pagination);
free(display_query);
free(locale);
free_context(&ctx);
return -1;
}
char ***image_matrix = malloc(sizeof(char **) * result_count);
int *inner_counts = malloc(sizeof(int) * result_count);
if (!image_matrix || !inner_counts) {
if (image_matrix)
free(image_matrix);
if (inner_counts)
free(inner_counts);
free_image_results(results, result_count);
free(request_cache_key);
free(display_query);
free_context(&ctx);
return -1;
}
StringMatrix images;
string_matrix_init(&images);
for (int i = 0; i < result_count; i++) {
image_matrix[i] = malloc(sizeof(char *) * IMAGE_RESULT_FIELDS);
image_matrix[i][0] = strdup(results[i].thumbnail_url);
image_matrix[i][1] = strdup(results[i].title);
image_matrix[i][2] = strdup(results[i].page_url);
image_matrix[i][3] = strdup(results[i].full_url);
inner_counts[i] = IMAGE_RESULT_FIELDS;
const char *values[IMAGE_RESULT_FIELDS] = {
results[i].thumbnail_url,
results[i].title,
results[i].page_url,
results[i].full_url,
};
if (string_matrix_append(&images, values, IMAGE_RESULT_FIELDS) != 0) {
string_matrix_free(&images);
string_matrix_free(&pagination);
free_image_results(results, result_count);
free(request_cache_key);
free(display_query);
free(locale);
free_context(&ctx);
return -1;
}
}
context_set_array_of_arrays(&ctx, "images", image_matrix, result_count,
inner_counts);
context_set_array_of_arrays(&ctx, "images", images.rows, images.count,
images.field_counts);
char *rendered = render_template("images.html", &ctx);
if (rendered) {
@ -187,27 +188,13 @@ int images_handler(UrlParams *params) {
send_response("<h1>Error rendering image results</h1>");
}
for (int i = 0; i < result_count; i++) {
for (int j = 0; j < IMAGE_RESULT_FIELDS; j++)
free(image_matrix[i][j]);
free(image_matrix[i]);
}
free(image_matrix);
free(inner_counts);
if (pager_count > 0) {
for (int i = 0; i < pager_count; i++) {
for (int j = 0; j < LINK_FIELD_COUNT; j++)
free(pager_matrix[i][j]);
free(pager_matrix[i]);
}
free(pager_matrix);
free(pager_inner_counts);
}
string_matrix_free(&images);
string_matrix_free(&pagination);
free_image_results(results, result_count);
free(request_cache_key);
free(display_query);
free(locale);
free_context(&ctx);
return 0;

467
src/Routes/Search.c
View file

@ -140,29 +140,31 @@ static InfoBox fetch_currency_wrapper(char *query) {
return fetch_currency_data(query);
}
char *get_base_url(const char *input) {
if (!input) return NULL;
if (!input)
return NULL;
const char *start = input;
const char *start = input;
const char *protocol_pos = strstr(input, "://");
if (protocol_pos) {
start = protocol_pos + 3;
}
const char *protocol_pos = strstr(input, "://");
if (protocol_pos) {
start = protocol_pos + 3;
}
const char *end = start;
while (*end && *end != '/' && *end != '?' && *end != '#') {
end++;
}
const char *end = start;
while (*end && *end != '/' && *end != '?' && *end != '#') {
end++;
}
size_t len = end - start;
size_t len = end - start;
char *domain = (char *)malloc(len + 1);
if (!domain) return NULL;
char *domain = (char *)malloc(len + 1);
if (!domain)
return NULL;
strncpy(domain, start, len);
domain[len] = '\0';
strncpy(domain, start, len);
domain[len] = '\0';
return domain;
return domain;
}
static int is_calculator_query(const char *query) {
if (!query)
@ -249,88 +251,65 @@ static void *infobox_thread_func(void *arg) {
return NULL;
}
static int add_infobox_to_collection(InfoBox *infobox, char ****collection,
int **inner_counts, int current_count) {
*collection =
(char ***)realloc(*collection, sizeof(char **) * (current_count + 1));
*inner_counts =
(int *)realloc(*inner_counts, sizeof(int) * (current_count + 1));
(*collection)[current_count] =
(char **)malloc(sizeof(char *) * INFOBOX_FIELD_COUNT);
(*collection)[current_count][0] =
infobox->title ? strdup(infobox->title) : NULL;
(*collection)[current_count][1] =
infobox->thumbnail_url ? strdup(infobox->thumbnail_url) : NULL;
(*collection)[current_count][2] =
infobox->extract ? strdup(infobox->extract) : NULL;
(*collection)[current_count][3] = infobox->url ? strdup(infobox->url) : NULL;
(*collection)[current_count][4] = infobox->url ? strdup(infobox->url) : NULL;
(*inner_counts)[current_count] = INFOBOX_FIELD_COUNT;
return current_count + 1;
static int add_infobox_to_collection(InfoBox *infobox,
StringMatrix *collection) {
const char *values[INFOBOX_FIELD_COUNT] = {
infobox->title, infobox->thumbnail_url, infobox->extract,
infobox->url, infobox->url,
};
return string_matrix_append(collection, values, INFOBOX_FIELD_COUNT);
}
static int add_warning_to_collection(const char *engine_name,
const char *warning_message,
char ****collection, int **inner_counts,
int current_count) {
char ***new_collection =
(char ***)malloc(sizeof(char **) * (current_count + 1));
int *new_inner_counts =
(int *)malloc(sizeof(int) * (current_count + 1));
if (!new_collection || !new_inner_counts) {
free(new_collection);
free(new_inner_counts);
return current_count;
}
if (*collection && current_count > 0) {
memcpy(new_collection, *collection, sizeof(char **) * current_count);
}
if (*inner_counts && current_count > 0) {
memcpy(new_inner_counts, *inner_counts, sizeof(int) * current_count);
}
free(*collection);
free(*inner_counts);
*collection = new_collection;
*inner_counts = new_inner_counts;
(*collection)[current_count] = (char **)malloc(sizeof(char *) * 2);
if (!(*collection)[current_count])
return current_count;
(*collection)[current_count][0] = strdup(engine_name ? engine_name : "");
(*collection)[current_count][1] =
strdup(warning_message ? warning_message : "");
if (!(*collection)[current_count][0] || !(*collection)[current_count][1]) {
free((*collection)[current_count][0]);
free((*collection)[current_count][1]);
free((*collection)[current_count]);
return current_count;
}
(*inner_counts)[current_count] = 2;
return current_count + 1;
StringMatrix *collection) {
const char *values[] = {engine_name, warning_message};
return string_matrix_append(collection, values, 2);
}
static const char *warning_message_for_job(const ScrapeJob *job, const char *locale) {
static void free_user_engine_list(char **user_engines, int user_engine_count) {
for (int i = 0; i < user_engine_count; i++)
free(user_engines[i]);
free(user_engines);
}
static void free_search_results(SearchResult **all_results, ScrapeJob *jobs,
int job_count) {
for (int i = 0; i < job_count; i++) {
for (int j = 0; all_results[i] && j < jobs[i].results_count; j++) {
free(all_results[i][j].url);
free(all_results[i][j].title);
free(all_results[i][j].snippet);
}
free(all_results[i]);
}
}
static void free_infobox_results(InfoBoxThreadData *data) {
for (int i = 0; i < HANDLER_COUNT; i++) {
free_infobox(&data[i].result);
}
}
static const char *warning_message_for_job(const ScrapeJob *job,
const char *locale) {
switch (job->status) {
case SCRAPE_STATUS_FETCH_ERROR: {
const char *msg = beaker_get_locale_value(locale, "warning_fetch_error");
return msg ? msg : "request failed before OmniSearch could read search results.";
return msg ? msg
: "request failed before OmniSearch could read search results.";
}
case SCRAPE_STATUS_PARSE_MISMATCH: {
const char *msg = beaker_get_locale_value(locale, "warning_parse_mismatch");
return msg ? msg : "returned search results in a format OmniSearch could not parse.";
return msg ? msg
: "returned search results in a format OmniSearch could not "
"parse.";
}
case SCRAPE_STATUS_BLOCKED: {
const char *msg = beaker_get_locale_value(locale, "warning_blocked");
return msg ? msg : "returned a captcha or another blocking page instead of search results.";
return msg ? msg
: "returned a captcha or another blocking page instead of "
"search results.";
}
default:
return NULL;
@ -384,8 +363,8 @@ static int engine_allowed_for_user(const SearchEngine *eng, char **user_ids,
static char *build_search_href(const char *query, const char *engine_id,
int page) {
const char *safe_query = query ? query : "";
int use_engine = engine_id && engine_id[0] != '\0' &&
!engine_id_matches(engine_id, "all");
int use_engine =
engine_id && engine_id[0] != '\0' && !engine_id_matches(engine_id, "all");
size_t needed = strlen("/search?q=") + strlen(safe_query) + 1;
if (use_engine)
@ -437,13 +416,18 @@ int results_handler(UrlParams *params) {
extern Config global_config;
TemplateContext ctx = new_context();
char *raw_query = "";
const char *selected_engine_id = "all";
char selected_engine_id[32];
char *default_engine = get_default_search_engine();
snprintf(selected_engine_id, sizeof(selected_engine_id), "%s",
default_engine);
free(default_engine);
int page = 1;
int btnI = 0;
char **user_engines = NULL;
int user_engine_count = 0;
int has_user_pref = (get_user_engines(&user_engines, &user_engine_count) == 0);
int has_user_pref =
(get_user_engines(&user_engines, &user_engine_count) == 0);
if (params) {
for (int i = 0; i < params->count; i++) {
@ -454,7 +438,8 @@ int results_handler(UrlParams *params) {
if (parsed > 1)
page = parsed;
} else if (strcmp(params->params[i].key, "engine") == 0) {
selected_engine_id = params->params[i].value;
snprintf(selected_engine_id, sizeof(selected_engine_id), "%s",
params->params[i].value);
} else if (strcmp(params->params[i].key, "btnI") == 0) {
btnI = atoi(params->params[i].value);
}
@ -471,9 +456,11 @@ int results_handler(UrlParams *params) {
beaker_set_locale(&ctx, locale);
const char *rate_limit_msg = beaker_get_locale_value(locale, "rate_limit");
if (!rate_limit_msg) rate_limit_msg = "Slow down! Too many searches from you!";
if (!rate_limit_msg)
rate_limit_msg = "Slow down! Too many searches from you!";
const char *no_results_msg = beaker_get_locale_value(locale, "no_results");
if (!no_results_msg) no_results_msg = "No results found";
if (!no_results_msg)
no_results_msg = "No results found";
char page_str[16];
snprintf(page_str, sizeof(page_str), "%d", page);
@ -491,18 +478,24 @@ int results_handler(UrlParams *params) {
if (!raw_query || strlen(raw_query) == 0) {
send_redirect("/");
if (has_user_pref) {
for (int i = 0; i < user_engine_count; i++)
free(user_engines[i]);
free(user_engines);
}
free(locale);
free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
return -1;
}
const SearchEngine *selected_engine = find_enabled_engine(selected_engine_id);
if (!selected_engine)
selected_engine_id = "all";
if (selected_engine &&
!engine_allowed_for_user(selected_engine, user_engines,
user_engine_count, has_user_pref)) {
selected_engine = NULL;
}
if (selected_engine) {
snprintf(selected_engine_id, sizeof(selected_engine_id), "%s",
selected_engine->id);
} else {
snprintf(selected_engine_id, sizeof(selected_engine_id), "all");
}
context_set(&ctx, "selected_engine", selected_engine_id);
char *search_href = build_search_href(raw_query, selected_engine_id, 1);
@ -528,13 +521,6 @@ int results_handler(UrlParams *params) {
infobox_data[i].result = (InfoBox){NULL};
}
if (page == 1) {
for (int i = 0; i < HANDLER_COUNT; i++) {
pthread_create(&infobox_threads[i], NULL, infobox_thread_func,
&infobox_data[i]);
}
}
ScrapeJob jobs[ENGINE_COUNT];
SearchResult *all_results[ENGINE_COUNT];
@ -592,11 +578,8 @@ int results_handler(UrlParams *params) {
snprintf(response, sizeof(response), "<h1>%s</h1>", rate_limit_msg);
send_response(response);
free(request_cache_key);
if (has_user_pref) {
for (int i = 0; i < user_engine_count; i++)
free(user_engines[i]);
free(user_engines);
}
free(locale);
free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
return -1;
}
@ -615,15 +598,14 @@ int results_handler(UrlParams *params) {
}
if (filter_engine_count > 1) {
char ***filter_matrix = NULL;
int *filter_inner_counts = NULL;
int filter_count = 0;
StringMatrix filter_matrix;
string_matrix_init(&filter_matrix);
char *all_href = build_search_href(raw_query, "all", 1);
filter_count = add_link_to_collection(
const char *all_values[LINK_FIELD_COUNT] = {
all_href, "All",
selected_engine ? "engine-filter" : "engine-filter active",
&filter_matrix, &filter_inner_counts, filter_count);
selected_engine ? "engine-filter" : "engine-filter active"};
string_matrix_append(&filter_matrix, all_values, LINK_FIELD_COUNT);
free(all_href);
for (int i = 0; i < ENGINE_COUNT; i++) {
@ -639,22 +621,24 @@ int results_handler(UrlParams *params) {
? "engine-filter active"
: "engine-filter";
filter_count = add_link_to_collection(filter_href, ENGINE_REGISTRY[i].name,
filter_class, &filter_matrix,
&filter_inner_counts, filter_count);
const char *filter_values[LINK_FIELD_COUNT] = {
filter_href, ENGINE_REGISTRY[i].name, filter_class};
string_matrix_append(&filter_matrix, filter_values, LINK_FIELD_COUNT);
free(filter_href);
}
if (filter_count > 0) {
context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix,
filter_count, filter_inner_counts);
for (int i = 0; i < filter_count; i++) {
for (int j = 0; j < LINK_FIELD_COUNT; j++)
free(filter_matrix[i][j]);
free(filter_matrix[i]);
}
free(filter_matrix);
free(filter_inner_counts);
if (filter_matrix.count > 0) {
context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix.rows,
filter_matrix.count,
filter_matrix.field_counts);
}
string_matrix_free(&filter_matrix);
}
if (page == 1) {
for (int i = 0; i < HANDLER_COUNT; i++) {
pthread_create(&infobox_threads[i], NULL, infobox_thread_func,
&infobox_data[i]);
}
}
@ -672,27 +656,12 @@ int results_handler(UrlParams *params) {
for (int i = 0; i < engine_idx; i++) {
if (jobs[i].results_count > 0 && all_results[i][0].url) {
char *redirect_url = strdup(all_results[i][0].url);
for (int j = 0; j < enabled_engine_count; j++) {
for (int k = 0; k < jobs[j].results_count; k++) {
free(all_results[j][k].url);
free(all_results[j][k].title);
free(all_results[j][k].snippet);
}
free(all_results[j]);
}
if (page == 1) {
for (int j = 0; j < HANDLER_COUNT; j++) {
if (infobox_data[j].success) {
free_infobox(&infobox_data[j].result);
}
}
}
free_search_results(all_results, jobs, engine_idx);
if (page == 1)
free_infobox_results(infobox_data);
free(request_cache_key);
if (has_user_pref) {
for (int i = 0; i < user_engine_count; i++)
free(user_engines[i]);
free(user_engines);
}
free(locale);
free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
if (redirect_url) {
send_redirect(redirect_url);
@ -701,54 +670,37 @@ int results_handler(UrlParams *params) {
return 0;
}
}
for (int i = 0; i < enabled_engine_count; i++) {
free(all_results[i]);
}
if (page == 1) {
for (int i = 0; i < HANDLER_COUNT; i++) {
if (infobox_data[i].success) {
free_infobox(&infobox_data[i].result);
}
}
}
free_search_results(all_results, jobs, engine_idx);
if (page == 1)
free_infobox_results(infobox_data);
free(request_cache_key);
if (has_user_pref) {
for (int i = 0; i < user_engine_count; i++)
free(user_engines[i]);
free(user_engines);
}
free(locale);
free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
char no_results_html[128];
snprintf(no_results_html, sizeof(no_results_html), "<h1>%s</h1>", no_results_msg);
snprintf(no_results_html, sizeof(no_results_html), "<h1>%s</h1>",
no_results_msg);
send_response(no_results_html);
return 0;
}
char ***infobox_matrix = NULL;
int *infobox_inner_counts = NULL;
int infobox_count = 0;
StringMatrix infobox_matrix;
string_matrix_init(&infobox_matrix);
if (page == 1) {
for (int i = 0; i < HANDLER_COUNT; i++) {
if (infobox_data[i].success) {
infobox_count =
add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix,
&infobox_inner_counts, infobox_count);
add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix);
}
}
}
if (infobox_count > 0) {
context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix,
infobox_count, infobox_inner_counts);
for (int i = 0; i < infobox_count; i++) {
for (int j = 0; j < INFOBOX_FIELD_COUNT; j++)
free(infobox_matrix[i][j]);
free(infobox_matrix[i]);
}
free(infobox_matrix);
free(infobox_inner_counts);
if (infobox_matrix.count > 0) {
context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix.rows,
infobox_matrix.count,
infobox_matrix.field_counts);
}
string_matrix_free(&infobox_matrix);
int warning_count = 0;
for (int i = 0; i < enabled_engine_count; i++) {
@ -757,35 +709,25 @@ int results_handler(UrlParams *params) {
}
if (warning_count > 0) {
char ***warning_matrix = NULL;
int *warning_inner_counts = NULL;
int warning_index = 0;
StringMatrix warning_matrix;
string_matrix_init(&warning_matrix);
for (int i = 0; i < enabled_engine_count; i++) {
const char *warning_message = warning_message_for_job(&jobs[i], locale);
if (!warning_message)
continue;
warning_index = add_warning_to_collection(
jobs[i].engine->name, warning_message, &warning_matrix,
&warning_inner_counts, warning_index);
add_warning_to_collection(jobs[i].engine->name, warning_message,
&warning_matrix);
}
if (warning_index > 0) {
context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix,
warning_index, warning_inner_counts);
if (warning_matrix.count > 0) {
context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix.rows,
warning_matrix.count,
warning_matrix.field_counts);
}
if (warning_matrix) {
for (int i = 0; i < warning_index; i++) {
free(warning_matrix[i][0]);
free(warning_matrix[i][1]);
free(warning_matrix[i]);
}
free(warning_matrix);
}
if (warning_inner_counts)
free(warning_inner_counts);
string_matrix_free(&warning_matrix);
}
int total_results = 0;
@ -794,33 +736,8 @@ int results_handler(UrlParams *params) {
}
if (total_results > 0) {
char ***results_matrix = (char ***)malloc(sizeof(char **) * total_results);
int *results_inner_counts = (int *)malloc(sizeof(int) * total_results);
if (!results_matrix || !results_inner_counts) {
char *html = render_template("results.html", &ctx);
if (html) {
send_response(html);
free(html);
}
for (int i = 0; i < enabled_engine_count; i++)
free(all_results[i]);
if (page == 1) {
for (int i = 0; i < HANDLER_COUNT; i++) {
if (infobox_data[i].success) {
free_infobox(&infobox_data[i].result);
}
}
}
free(request_cache_key);
if (has_user_pref) {
for (int i = 0; i < user_engine_count; i++)
free(user_engines[i]);
free(user_engines);
}
free_context(&ctx);
return 0;
}
int unique_count = 0;
StringMatrix results_matrix;
string_matrix_init(&results_matrix);
UrlHashTable url_table;
url_hash_init(&url_table);
@ -837,61 +754,45 @@ int results_handler(UrlParams *params) {
url_hash_insert(&url_table, display_url);
results_matrix[unique_count] =
(char **)malloc(sizeof(char *) * RESULT_FIELD_COUNT);
if (!results_matrix[unique_count]) {
free(all_results[i][j].url);
free(all_results[i][j].title);
free(all_results[i][j].snippet);
continue;
}
char *pretty_url = pretty_display_url(display_url);
char *base_url = get_base_url(display_url);
results_matrix[unique_count][0] = strdup(display_url);
results_matrix[unique_count][1] = strdup(pretty_url);
results_matrix[unique_count][2] = all_results[i][j].title
? strdup(all_results[i][j].title)
: strdup("Untitled");
results_matrix[unique_count][3] =
all_results[i][j].snippet ? strdup(all_results[i][j].snippet)
: strdup("");
results_matrix[unique_count][4] = strdup(base_url ? base_url : "");
results_matrix[unique_count][5] = strdup("");
results_inner_counts[unique_count] = RESULT_FIELD_COUNT;
const char *values[RESULT_FIELD_COUNT] = {
display_url,
pretty_url,
all_results[i][j].title ? all_results[i][j].title : "Untitled",
all_results[i][j].snippet ? all_results[i][j].snippet : "",
base_url,
"",
};
int append_result =
string_matrix_append(&results_matrix, values, RESULT_FIELD_COUNT);
free(pretty_url);
free(base_url);
free(all_results[i][j].url);
free(all_results[i][j].title);
free(all_results[i][j].snippet);
unique_count++;
if (append_result != 0) {
continue;
}
}
free(all_results[i]);
}
context_set_array_of_arrays(&ctx, "results", results_matrix, unique_count,
results_inner_counts);
context_set_array_of_arrays(&ctx, "results", results_matrix.rows,
results_matrix.count,
results_matrix.field_counts);
char ***pager_matrix = NULL;
int *pager_inner_counts = NULL;
SearchHrefData href_data = { .query = raw_query, .engine_id = selected_engine_id };
int pager_count = build_pagination(page, search_href_builder,
&href_data, &pager_matrix,
&pager_inner_counts);
StringMatrix pager_matrix;
SearchHrefData href_data = {.query = raw_query,
.engine_id = selected_engine_id};
int pagination_result = string_matrix_build_pagination(
page, search_href_builder, &href_data, &pager_matrix);
if (pager_count > 0) {
context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix,
pager_count, pager_inner_counts);
for (int i = 0; i < pager_count; i++) {
for (int j = 0; j < LINK_FIELD_COUNT; j++)
free(pager_matrix[i][j]);
free(pager_matrix[i]);
}
free(pager_matrix);
free(pager_inner_counts);
if (pagination_result == 0 && pager_matrix.count > 0) {
context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix.rows,
pager_matrix.count,
pager_matrix.field_counts);
}
char *html = render_template("results.html", &ctx);
@ -900,13 +801,8 @@ int results_handler(UrlParams *params) {
free(html);
}
for (int i = 0; i < unique_count; i++) {
for (int j = 0; j < RESULT_FIELD_COUNT; j++)
free(results_matrix[i][j]);
free(results_matrix[i]);
}
free(results_matrix);
free(results_inner_counts);
string_matrix_free(&pager_matrix);
string_matrix_free(&results_matrix);
url_hash_free(&url_table);
} else {
char *html = render_template("results.html", &ctx);
@ -915,26 +811,15 @@ int results_handler(UrlParams *params) {
free(html);
}
for (int i = 0; i < enabled_engine_count; i++) {
free(all_results[i]);
}
free_search_results(all_results, jobs, engine_idx);
}
free(request_cache_key);
if (page == 1) {
for (int i = 0; i < HANDLER_COUNT; i++) {
if (infobox_data[i].success) {
free_infobox(&infobox_data[i].result);
}
}
}
if (page == 1)
free_infobox_results(infobox_data);
free(locale);
if (has_user_pref) {
for (int i = 0; i < user_engine_count; i++)
free(user_engines[i]);
free(user_engines);
}
free_user_engine_list(user_engines, user_engine_count);
free_context(&ctx);
return 0;

16
src/Routes/Settings.c
View file

@ -34,6 +34,14 @@ int settings_handler(UrlParams *params) {
int user_engine_count = 0;
int has_user_pref =
(get_user_engines(&user_engines, &user_engine_count) == 0);
char *default_engine = get_default_search_engine();
if (strcmp(default_engine, "all") != 0 && has_user_pref &&
!user_engines_contains(default_engine, user_engines,
user_engine_count)) {
free(default_engine);
default_engine = strdup("all");
}
int enabled_count = 0;
for (int i = 0; i < ENGINE_COUNT; i++) {
@ -63,11 +71,13 @@ int settings_handler(UrlParams *params) {
}
}
engine_data[idx] = malloc(sizeof(char *) * 3);
engine_data[idx] = malloc(sizeof(char *) * 4);
engine_data[idx][0] = (char *)ENGINE_REGISTRY[i].id;
engine_data[idx][1] = (char *)ENGINE_REGISTRY[i].name;
engine_data[idx][2] = is_selected ? "checked" : "";
engine_inner[idx] = 3;
engine_data[idx][3] =
strcmp(default_engine, ENGINE_REGISTRY[i].id) == 0 ? "selected" : "";
engine_inner[idx] = 4;
idx++;
}
@ -78,6 +88,7 @@ int settings_handler(UrlParams *params) {
context_set(&ctx, "query", query);
context_set(&ctx, "theme", theme);
context_set(&ctx, "locale", locale);
context_set(&ctx, "default_engine", default_engine);
context_set_array_of_arrays(&ctx, "locales", locale_data, locale_count,
inner_counts);
@ -132,6 +143,7 @@ int settings_handler(UrlParams *params) {
send_response(rendered_html);
free(rendered_html);
free(default_engine);
free(theme);
free(locale);
free_context(&ctx);

30
src/Routes/SettingsSave.c
View file

@ -10,6 +10,8 @@ int settings_save_handler(UrlParams *params) {
const char *theme = "";
const char *locale = "";
const char *query = "";
const char *default_engine = "all";
int default_engine_present = 0;
int engines_present = 0;
char selected_ids[ENGINE_COUNT][32];
int selected_count = 0;
@ -22,6 +24,9 @@ int settings_save_handler(UrlParams *params) {
locale = params->params[i].value;
} else if (strcmp(params->params[i].key, "q") == 0) {
query = params->params[i].value;
} else if (strcmp(params->params[i].key, "default_engine") == 0) {
default_engine = params->params[i].value;
default_engine_present = 1;
} else if (strcmp(params->params[i].key, "engines_present") == 0) {
engines_present = 1;
} else if (strncmp(params->params[i].key, "engine_", 7) == 0 &&
@ -58,6 +63,31 @@ int settings_save_handler(UrlParams *params) {
false, false);
}
if (default_engine_present) {
const char *validated_default = "all";
for (int i = 0; i < ENGINE_COUNT; i++) {
if (!ENGINE_REGISTRY[i].enabled ||
strcmp(default_engine, ENGINE_REGISTRY[i].id) != 0)
continue;
int selected = !engines_present;
for (int j = 0; j < selected_count; j++) {
if (strcmp(selected_ids[j], ENGINE_REGISTRY[i].id) == 0) {
selected = 1;
break;
}
}
if (selected)
validated_default = ENGINE_REGISTRY[i].id;
break;
}
set_cookie("default_engine", validated_default,
"Fri, 31 Dec 2038 23:59:59 GMT", "/", false, false);
}
char redirect_url[512];
snprintf(redirect_url, sizeof(redirect_url), "/settings?q=%s", query);
send_redirect(redirect_url);

131
src/Scraping/Scraping.c
View file

@ -1,9 +1,11 @@
#include "Scraping.h"
#include "../Cache/Cache.h"
#include "../Proxy/Proxy.h"
#include "../Utility/XmlHelper.h"
#include "Config.h"
#include <curl/curl.h>
#include <libxml/HTMLparser.h>
#include <libxml/parser.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -24,7 +26,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job,
return response_contains(response, "<title>Startpage Captcha</title>") ||
response_contains(response, "Startpage Captcha") ||
response_contains(response, "/static-pages-assets/page-data/captcha/") ||
response_contains(response,
"/static-pages-assets/page-data/captcha/") ||
response_contains(response, ">Startpage Blocked</title>");
}
@ -39,7 +42,8 @@ static int response_looks_like_results_page(const ScrapeJob *job,
}
if (strcmp(job->engine->name, "Startpage") == 0) {
return response_contains(response, "<title>Startpage Search Results</title>") ||
return response_contains(response,
"<title>Startpage Search Results</title>") ||
response_contains(response, "class=\"w-gl") ||
response_contains(response, "data-testid=\"gl-title-link\"");
}
@ -55,6 +59,11 @@ static int response_looks_like_results_page(const ScrapeJob *job,
response_contains(response, "Mojeek Search");
}
if (strcmp(job->engine->name, "YaCy") == 0) {
return response_contains(response, "<item") ||
response_contains(response, "<rss");
}
return 0;
}
@ -72,18 +81,24 @@ static void classify_job_response(ScrapeJob *job, const char *response,
return;
}
xmlDocPtr doc = htmlReadMemory(response, response_size, NULL, NULL,
HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
HTML_PARSE_NOWARNING);
xmlDocPtr doc;
if (job->engine->is_xml) {
doc = xmlReadMemory(response, response_size, NULL, NULL,
XML_PARSE_RECOVER | XML_PARSE_NOERROR |
XML_PARSE_NOWARNING);
} else {
doc = htmlReadMemory(response, response_size, NULL, NULL,
HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
HTML_PARSE_NOWARNING);
}
if (!doc) {
job->status = SCRAPE_STATUS_FETCH_ERROR;
return;
}
job->results_count =
job->engine->parser(job->engine->name, doc, job->out_results,
job->max_results);
job->results_count = job->engine->parser(job->engine->name, doc,
job->out_results, job->max_results);
xmlFreeDoc(doc);
if (job->results_count > 0) {
@ -156,14 +171,26 @@ void parse_and_cache_response(ScrapeJob *job) {
}
}
void cleanup_job_handle(ScrapeJob *job, CURL *handle) {
static void cleanup_job_results(ScrapeJob *job) {
if (!job || !job->out_results)
return;
xml_result_free(*job->out_results, job->results_count);
*job->out_results = NULL;
job->results_count = 0;
}
static void cleanup_job_handle(ScrapeJob *job, CURL *handle) {
struct curl_slist *headers = NULL;
curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers);
if (handle)
curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers);
if (headers)
curl_slist_free_all(headers);
free(job->response.memory);
job->response.memory = NULL;
job->response.size = 0;
job->response.capacity = 0;
}
void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) {
@ -180,19 +207,25 @@ void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) {
}
int setup_job(ScrapeJob *job, CURLM *multi_handle) {
if (job->handle)
if (job->handle) {
cleanup_job_handle(job, job->handle);
curl_easy_cleanup(job->handle);
if (job->response.memory)
job->handle = NULL;
} else if (job->response.memory) {
free(job->response.memory);
job->response.memory = NULL;
job->response.size = 0;
job->response.capacity = 0;
}
job->results_count = 0;
cleanup_job_results(job);
job->http_status = 0;
job->status = SCRAPE_STATUS_PENDING;
if (check_cache_for_job(job)) {
job->results_count = job->results_count > 0 ? job->results_count : 0;
if (check_cache_for_job(job))
return 0;
}
cleanup_job_results(job);
char *encoded_query = curl_easy_escape(NULL, job->query, 0);
if (!encoded_query) {
@ -200,12 +233,17 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
return -1;
}
for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) {
if (p[0] == '%' && p[1] == '2' && p[2] == '0') {
*p = '+';
memmove(p + 1, p + 3, strlen(p + 3) + 1);
char *read = encoded_query;
char *write = encoded_query;
while (*read) {
if (read[0] == '%' && read[1] == '2' && read[2] == '0') {
*write++ = '+';
read += 3;
} else {
*write++ = *read++;
}
}
*write = '\0';
char *full_url =
build_search_url(job->engine->base_url, job->engine->page_param,
@ -228,6 +266,14 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
job->response.memory = (char *)malloc(INITIAL_BUFFER_SIZE);
job->response.size = 0;
job->response.capacity = INITIAL_BUFFER_SIZE;
if (!job->response.memory) {
curl_easy_cleanup(job->handle);
job->handle = NULL;
job->response.capacity = 0;
free(full_url);
job->status = SCRAPE_STATUS_FETCH_ERROR;
return -1;
}
struct curl_slist *headers =
build_request_headers(job->engine->host_header, job->engine->referer);
@ -236,10 +282,33 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
curl_easy_setopt(job->handle, CURLOPT_PRIVATE, headers);
free(full_url);
curl_multi_add_handle(multi_handle, job->handle);
CURLMcode add_result = curl_multi_add_handle(multi_handle, job->handle);
if (add_result != CURLM_OK) {
cleanup_job_handle(job, job->handle);
curl_easy_cleanup(job->handle);
job->handle = NULL;
job->status = SCRAPE_STATUS_FETCH_ERROR;
return -1;
}
return 0;
}
static void cleanup_unfinished_jobs(CURLM *multi_handle, ScrapeJob *jobs,
int num_jobs) {
for (int i = 0; i < num_jobs; i++) {
if (!jobs[i].handle)
continue;
curl_multi_remove_handle(multi_handle, jobs[i].handle);
cleanup_job_handle(&jobs[i], jobs[i].handle);
curl_easy_cleanup(jobs[i].handle);
jobs[i].handle = NULL;
if (jobs[i].status == SCRAPE_STATUS_PENDING)
jobs[i].status = SCRAPE_STATUS_FETCH_ERROR;
}
}
int handle_responses(CURLM *multi_handle, ScrapeJob *jobs, int num_jobs) {
CURLMsg *msg;
int msgs_left;
@ -279,34 +348,30 @@ int should_retry(ScrapeJob *jobs, int num_jobs) {
int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) {
int retries = 0;
retry:
;
retry:;
CURLM *multi_handle = curl_multi_init();
if (!multi_handle)
return -1;
for (int i = 0; i < num_jobs; i++) {
if (setup_job(&jobs[i], multi_handle) != 0 && jobs[i].handle) {
curl_multi_remove_handle(multi_handle, jobs[i].handle);
curl_easy_cleanup(jobs[i].handle);
jobs[i].handle = NULL;
}
setup_job(&jobs[i], multi_handle);
}
http_delay();
int still_running = 0;
curl_multi_perform(multi_handle, &still_running);
CURLMcode mc = curl_multi_perform(multi_handle, &still_running);
do {
while (mc == CURLM_OK && still_running) {
int numfds = 0;
CURLMcode mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
if (mc != CURLM_OK)
break;
curl_multi_perform(multi_handle, &still_running);
} while (still_running);
mc = curl_multi_perform(multi_handle, &still_running);
}
handle_responses(multi_handle, jobs, num_jobs);
cleanup_unfinished_jobs(multi_handle, jobs, num_jobs);
curl_multi_cleanup(multi_handle);
if (retries < max_proxy_retries && should_retry(jobs, num_jobs)) {

2
src/Scraping/Scraping.h
View file

@ -26,6 +26,7 @@ typedef struct {
int page_base;
ParserFunc parser;
int enabled;
int is_xml;
} SearchEngine;
typedef struct {
@ -59,6 +60,7 @@ typedef struct {
extern SearchEngine ENGINE_REGISTRY[];
extern const int ENGINE_COUNT;
void apply_engines_config(const char *engines_str);
void configure_yacy_engine(const char *instance);
size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
void *userp);

13
src/Scraping/ScrapingHttp.c
View file

@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header,
struct curl_slist *headers = NULL;
char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM];
snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
headers = curl_slist_append(headers, host_buf);
headers = curl_slist_append(headers, ref_buf);
if (host_header && host_header[0] != '\0') {
snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
headers = curl_slist_append(headers, host_buf);
}
if (referer && referer[0] != '\0') {
snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
headers = curl_slist_append(headers, ref_buf);
}
headers = curl_slist_append(
headers,
"Accept: "

116
src/Scraping/ScrapingParsers.c
View file

@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
if (!ctx)
return 0;
xmlXPathObjectPtr obj =
xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
xmlXPathObjectPtr obj = xml_xpath_eval(
ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
free_xpath_objects(ctx, obj);
@ -268,18 +268,17 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
xmlNodePtr result_node = obj->nodesetval->nodeTab[i];
ctx->node = result_node;
xmlXPathObjectPtr link_obj =
xml_xpath_eval(ctx, ".//a[@class='title']");
xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']");
char *url =
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0],
(xmlChar *)"href")
: NULL;
char *title = (link_obj && link_obj->nodesetval &&
link_obj->nodesetval->nodeNr > 0)
? xml_node_content(link_obj->nodesetval->nodeTab[0])
: NULL;
char *title =
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
? xml_node_content(link_obj->nodesetval->nodeTab[0])
: NULL;
xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']");
char *snippet_text =
@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc,
static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
SearchResult **out_results, int max_results);
static int parse_yacy(const char *engine_name, xmlDocPtr doc,
SearchResult **out_results, int max_results) {
(void)engine_name;
int found_count = 0;
xmlXPathContextPtr ctx = create_xpath_context(doc);
if (!ctx)
return 0;
xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item");
if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
free_xpath_objects(ctx, obj);
return 0;
}
int num_items = obj->nodesetval->nodeNr;
*out_results = alloc_results_array(num_items, max_results);
if (!*out_results) {
free_xpath_objects(ctx, obj);
return 0;
}
for (int i = 0; i < num_items && found_count < max_results; i++) {
xmlNodePtr item_node = obj->nodesetval->nodeTab[i];
ctx->node = item_node;
xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title");
char *title = (title_obj && title_obj->nodesetval &&
title_obj->nodesetval->nodeNr > 0)
? xml_node_content(title_obj->nodesetval->nodeTab[0])
: NULL;
xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link");
char *url =
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
? xml_node_content(link_obj->nodesetval->nodeTab[0])
: NULL;
xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description");
char *snippet_text =
(desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0)
? xml_node_content(desc_obj->nodesetval->nodeTab[0])
: NULL;
if (url && title) {
assign_result(&(*out_results)[found_count], url, title, snippet_text, 0);
found_count++;
}
free_xml_node_list(title, url, snippet_text);
if (title_obj)
xmlXPathFreeObject(title_obj);
if (link_obj)
xmlXPathFreeObject(link_obj);
if (desc_obj)
xmlXPathFreeObject(desc_obj);
}
ctx->node = NULL;
free_xpath_objects(ctx, obj);
return found_count;
}
static char yacy_base_url[BUFFER_SIZE_LARGE];
void configure_yacy_engine(const char *instance) {
if (!instance || instance[0] == '\0')
instance = DEFAULT_YACY_INSTANCE;
snprintf(yacy_base_url, sizeof(yacy_base_url),
"%s/"
"yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&"
"nav=all&maximumRecords=%d&query=",
instance, MAX_RESULTS_PER_ENGINE);
for (int i = 0; i < ENGINE_COUNT; i++) {
if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
ENGINE_REGISTRY[i].base_url = yacy_base_url;
break;
}
}
}
SearchEngine ENGINE_REGISTRY[] = {
{.id = "ddg",
.name = "DuckDuckGo Lite",
@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = {
.page_multiplier = 10,
.page_base = 1,
.parser = parse_mojeek,
.enabled = 1}};
.enabled = 1},
{.id = "yacy",
.name = "YaCy",
.base_url = "",
.host_header = NULL,
.referer = NULL,
.page_param = "startRecord",
.page_multiplier = 10,
.page_base = 0,
.parser = parse_yacy,
.enabled = 0,
.is_xml = 1}};
const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine);
@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) {
void apply_engines_config(const char *engines_str) {
if (!engines_str || engines_str[0] == '\0') {
for (int i = 0; i < ENGINE_COUNT; i++) {
if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
continue;
ENGINE_REGISTRY[i].enabled = 1;
}
return;
@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) {
if (strcmp(token, "*") == 0) {
for (int i = 0; i < ENGINE_COUNT; i++) {
if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
continue;
ENGINE_REGISTRY[i].enabled = 1;
}
} else if (token[0] == '-' && token[1] != '\0') {

219
src/Utility/Utility.c
View file

@ -14,7 +14,6 @@ static int themes_initialized = 0;
void init_themes(const char *static_path) {
if (themes_initialized)
return;
themes_initialized = 1;
char themes_dir[512];
snprintf(themes_dir, sizeof(themes_dir), "%s/themes", static_path);
@ -26,20 +25,32 @@ void init_themes(const char *static_path) {
struct dirent *entry;
int capacity = 4;
themes_list = malloc(sizeof(char *) * capacity);
if (!themes_list) {
closedir(dir);
return;
}
themes_count = 0;
while ((entry = readdir(dir)) != NULL) {
size_t len = strlen(entry->d_name);
if (len > 4 && strcmp(entry->d_name + len - 4, ".css") == 0) {
if (themes_count >= capacity) {
capacity *= 2;
themes_list = realloc(themes_list, sizeof(char *) * capacity);
int new_capacity = capacity * 2;
char **new_list = realloc(themes_list, sizeof(char *) * new_capacity);
if (!new_list)
break;
themes_list = new_list;
capacity = new_capacity;
}
themes_list[themes_count] = strndup(entry->d_name, len - 4);
char *theme = strndup(entry->d_name, len - 4);
if (!theme)
break;
themes_list[themes_count] = theme;
themes_count++;
}
}
closedir(dir);
themes_initialized = 1;
for (int i = 0; i < themes_count; i++) {
for (int j = i + 1; j < themes_count; j++) {
@ -206,98 +217,130 @@ int user_engines_contains(const char *engine_id, char **ids, int count) {
return 0;
}
int add_link_to_collection(const char *href, const char *label,
const char *class_name, char ****collection,
int **inner_counts, int current_count) {
char ***old_collection = *collection;
int *old_inner_counts = *inner_counts;
char ***new_collection =
(char ***)malloc(sizeof(char **) * (current_count + 1));
int *new_inner_counts = (int *)malloc(sizeof(int) * (current_count + 1));
char *get_default_search_engine(void) {
char *cookie = get_cookie("default_engine");
if (cookie && (strcmp(cookie, "all") == 0 || is_engine_id_enabled(cookie)))
return cookie;
if (!new_collection || !new_inner_counts) {
free(new_collection);
free(new_inner_counts);
return current_count;
}
if (*collection && current_count > 0) {
memcpy(new_collection, *collection, sizeof(char **) * current_count);
}
if (*inner_counts && current_count > 0) {
memcpy(new_inner_counts, *inner_counts, sizeof(int) * current_count);
}
*collection = new_collection;
*inner_counts = new_inner_counts;
(*collection)[current_count] =
(char **)malloc(sizeof(char *) * LINK_FIELD_COUNT);
if (!(*collection)[current_count]) {
*collection = old_collection;
*inner_counts = old_inner_counts;
free(new_collection);
free(new_inner_counts);
return current_count;
}
(*collection)[current_count][0] = strdup(href ? href : "");
(*collection)[current_count][1] = strdup(label ? label : "");
(*collection)[current_count][2] = strdup(class_name ? class_name : "");
if (!(*collection)[current_count][0] || !(*collection)[current_count][1] ||
!(*collection)[current_count][2]) {
free((*collection)[current_count][0]);
free((*collection)[current_count][1]);
free((*collection)[current_count][2]);
free((*collection)[current_count]);
*collection = old_collection;
*inner_counts = old_inner_counts;
free(new_collection);
free(new_inner_counts);
return current_count;
}
(*inner_counts)[current_count] = LINK_FIELD_COUNT;
free(old_collection);
free(old_inner_counts);
return current_count + 1;
free(cookie);
return strdup("all");
}
int build_pagination(int page, char *(*href_builder)(int page, void *data),
void *data, char ****out_matrix, int **out_inner_counts) {
void free_string_matrix(char ***matrix, int *inner_counts, int row_count) {
if (matrix) {
for (int i = 0; i < row_count; i++) {
int field_count = inner_counts ? inner_counts[i] : 0;
for (int j = 0; j < field_count; j++)
free(matrix[i][j]);
free(matrix[i]);
}
}
free(matrix);
free(inner_counts);
}
void string_matrix_init(StringMatrix *matrix) {
if (matrix)
*matrix = (StringMatrix){0};
}
int string_matrix_append(StringMatrix *matrix, const char *const *values,
int field_count) {
if (!matrix || !values || field_count <= 0)
return -1;
char **row = calloc((size_t)field_count, sizeof(*row));
if (!row)
return -1;
for (int i = 0; i < field_count; i++) {
row[i] = strdup(values[i] ? values[i] : "");
if (!row[i]) {
for (int j = 0; j < i; j++)
free(row[j]);
free(row);
return -1;
}
}
if (matrix->count == matrix->capacity) {
int new_capacity = matrix->capacity ? matrix->capacity * 2 : 8;
char ***new_rows = malloc(sizeof(*new_rows) * (size_t)new_capacity);
int *new_counts = malloc(sizeof(*new_counts) * (size_t)new_capacity);
if (!new_rows || !new_counts) {
free(new_rows);
free(new_counts);
for (int i = 0; i < field_count; i++)
free(row[i]);
free(row);
return -1;
}
if (matrix->count > 0) {
memcpy(new_rows, matrix->rows, sizeof(*new_rows) * matrix->count);
memcpy(new_counts, matrix->field_counts,
sizeof(*new_counts) * matrix->count);
}
free(matrix->rows);
free(matrix->field_counts);
matrix->rows = new_rows;
matrix->field_counts = new_counts;
matrix->capacity = new_capacity;
}
matrix->rows[matrix->count] = row;
matrix->field_counts[matrix->count] = field_count;
matrix->count++;
return 0;
}
void string_matrix_free(StringMatrix *matrix) {
if (!matrix)
return;
free_string_matrix(matrix->rows, matrix->field_counts, matrix->count);
*matrix = (StringMatrix){0};
}
int string_matrix_build_pagination(int page,
char *(*href_builder)(int page, void *data),
void *data, StringMatrix *out_matrix) {
enum { PAGER_WINDOW_SIZE = 5 };
if (!out_matrix || !href_builder)
return -1;
*out_matrix = NULL;
*out_inner_counts = NULL;
int count = 0;
string_matrix_init(out_matrix);
int pager_start = page <= 3 ? 1 : page - 2;
int pager_end = pager_start + PAGER_WINDOW_SIZE - 1;
if (page > 1) {
char *href = href_builder(page - 1, data);
count = add_link_to_collection(href, "←", "pagination-btn prev", out_matrix,
out_inner_counts, count);
for (int i = page > 1 ? pager_start - 1 : pager_start; i <= pager_end + 1;
i++) {
const char *label = NULL;
const char *class_name = "pagination-btn";
char page_label[16];
int target_page = i;
if (page > 1 && i == pager_start - 1) {
label = "←";
class_name = "pagination-btn prev";
target_page = page - 1;
} else if (i == pager_end + 1) {
label = "→";
class_name = "pagination-btn next";
target_page = page + 1;
} else {
snprintf(page_label, sizeof(page_label), "%d", i);
label = page_label;
if (i == page)
class_name = "pagination-btn pagination-current";
}
char *href = href_builder(target_page, data);
const char *values[LINK_FIELD_COUNT] = {href, label, class_name};
int result = string_matrix_append(out_matrix, values, LINK_FIELD_COUNT);
free(href);
if (result != 0) {
string_matrix_free(out_matrix);
return -1;
}
}
for (int i = pager_start; i <= pager_end; i++) {
char label[16];
snprintf(label, sizeof(label), "%d", i);
char *href = href_builder(i, data);
count = add_link_to_collection(
href, label,
i == page ? "pagination-btn pagination-current" : "pagination-btn",
out_matrix, out_inner_counts, count);
free(href);
}
char *href = href_builder(page + 1, data);
count = add_link_to_collection(href, "→", "pagination-btn next", out_matrix,
out_inner_counts, count);
free(href);
return count;
return 0;
}

22
src/Utility/Utility.h
View file

@ -13,6 +13,13 @@
#define LINK_FIELD_COUNT 3
typedef struct {
char ***rows;
int *field_counts;
int count;
int capacity;
} StringMatrix;
int hex_to_int(char c);
char *get_theme(const char *default_theme);
void init_themes(const char *static_path);
@ -23,12 +30,17 @@ char *get_locale(const char *default_locale);
int is_engine_id_enabled(const char *engine_id);
int get_user_engines(char ***out_ids, int *out_count);
int user_engines_contains(const char *engine_id, char **ids, int count);
char *get_default_search_engine(void);
int add_link_to_collection(const char *href, const char *label,
const char *class_name, char ****collection,
int **inner_counts, int current_count);
void free_string_matrix(char ***matrix, int *inner_counts, int row_count);
int build_pagination(int page, char *(*href_builder)(int page, void *data),
void *data, char ****out_matrix, int **out_inner_counts);
void string_matrix_init(StringMatrix *matrix);
int string_matrix_append(StringMatrix *matrix, const char *const *values,
int field_count);
void string_matrix_free(StringMatrix *matrix);
int string_matrix_build_pagination(int page,
char *(*href_builder)(int page, void *data),
void *data, StringMatrix *out_matrix);
#endif

40
static/main.css
View file

@ -822,6 +822,38 @@ header .logo-link:hover {
height:18px;
accent-color:var(--accent);
cursor:pointer;
flex-shrink:0;
margin:0;
}
.settings-engine-list {
margin:0;
}
.settings-engine-options {
display:grid;
gap:8px;
margin-inline:-3px;
}
.settings-engine-option {
display:flex;
align-items:center;
justify-content:space-between;
gap:16px;
padding:12px 14px;
border:1px solid var(--border);
border-radius:8px;
background:var(--bg-main);
color:var(--text-primary);
font-size:0.95rem;
cursor:pointer;
}
.settings-default-engine-field {
margin-top:12px;
padding-top:12px;
border-top:1px solid var(--border);
}
.settings-actions {
@ -888,6 +920,12 @@ header .logo-link:hover {
align-items:stretch;
gap:8px;
}
.settings-engine-options {
margin-inline:0;
}
.settings-engine-option {
padding-inline:7px;
}
.settings-actions {
flex-direction:column;
}
@ -936,5 +974,3 @@ header .logo-link:hover {
flex-direction: column;
}
}

2
templates/opensearch.xml
View file

@ -7,5 +7,5 @@
<Url type="text/html" method="get" template="{{scheme}}://{{domain}}/search?q={searchTerms}"/>
<InputEncoding>UTF-8</InputEncoding>
<OutputEncoding>UTF-8</OutputEncoding>
<moz:SearchForm>{{domain}}/</moz:SearchForm>
<moz:SearchForm>{{scheme}}://{{domain}}/</moz:SearchForm>
</OpenSearchDescription>

31
templates/settings.html
View file

@ -76,16 +76,33 @@
</div>
</section>
{{if has_enabled_engines}}
<section class="settings-section">
<section class="settings-section settings-engines-section">
<h3 class="settings-section-title">{{l("engines_label")}}</h3>
<p class="settings-section-desc">{{l("engines_desc")}}</p>
<input type="hidden" name="engines_present" value="1">
{{for eng in enabled_engines}}
<div class="settings-field">
<label class="settings-label" for="engine_{{eng[0]}}">{{eng[1]}}</label>
<input type="checkbox" id="engine_{{eng[0]}}" name="engine_{{eng[0]}}" value="1" class="settings-checkbox" {{eng[2]}}>
<div class="settings-engine-list">
<div class="settings-engine-options">
{{for eng in enabled_engines}}
<label class="settings-engine-option" for="engine_{{eng[0]}}">
<span>{{eng[1]}}</span>
<input type="checkbox" id="engine_{{eng[0]}}" name="engine_{{eng[0]}}" value="1" class="settings-checkbox" {{eng[2]}}>
</label>
{{endfor}}
</div>
</div>
<div class="settings-field settings-default-engine-field">
<label class="settings-label" for="default_engine">{{l("default_engine_label")}}</label>
<select id="default_engine" name="default_engine" class="settings-select">
<option value="all" {{if default_engine == "all"}}selected{{endif}}>{{l("all_engines_option")}}</option>
{{for eng in enabled_engines}}
{{if eng[2] != ""}}
<option value="{{eng[0]}}" {{eng[3]}}>{{eng[1]}}</option>
{{endif}}
{{endfor}}
</select>
</div>
{{endfor}}
</section>
{{endif}}
<div class="settings-actions">
@ -96,4 +113,4 @@
</div>
</body>
</html>
</html>