Merge branch 'indev'
This commit is contained in:
commit
e352971bc5
21 changed files with 763 additions and 519 deletions
3
Makefile
3
Makefile
|
|
@ -7,8 +7,9 @@ GIT_HASH := $(shell git rev-parse --short HEAD)
|
||||||
GIT_DATE := $(shell git log -1 --format='%ad' --date='format:%y.%m.%d')
|
GIT_DATE := $(shell git log -1 --format='%ad' --date='format:%y.%m.%d')
|
||||||
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
|
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
|
||||||
GIT_REMOTE := $(shell git remote get-url origin)
|
GIT_REMOTE := $(shell git remote get-url origin)
|
||||||
|
GIT_TAG := $(shell git describe --tags --exact-match HEAD 2>/dev/null)
|
||||||
|
|
||||||
VERSION := $(GIT_DATE)+$(GIT_HASH)_$(GIT_BRANCH)
|
VERSION := $(if $(GIT_TAG),$(GIT_TAG),$(GIT_DATE)+$(GIT_HASH)_$(GIT_BRANCH))
|
||||||
|
|
||||||
CFLAGS := -Wall -Wextra -O2 -Isrc -DVERSION='"$(VERSION)"' -DGIT_REMOTE='"$(GIT_REMOTE)"'
|
CFLAGS := -Wall -Wextra -O2 -Isrc -DVERSION='"$(VERSION)"' -DGIT_REMOTE='"$(GIT_REMOTE)"'
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,8 +31,11 @@ port = 8087
|
||||||
[engines]
|
[engines]
|
||||||
# Use * for all engines, or specify comma-separated list (e.g., ddg,yahoo)
|
# Use * for all engines, or specify comma-separated list (e.g., ddg,yahoo)
|
||||||
# Use *,-engine to exclude specific engines (e.g., *,-startpage)
|
# Use *,-engine to exclude specific engines (e.g., *,-startpage)
|
||||||
# Available engines: ddg, startpage, yahoo, mojeek
|
# Available engines: ddg, startpage, yahoo, mojeek, yacy
|
||||||
engines="*"
|
engines="*"
|
||||||
|
# YaCy instance to use for the yacy engine (optional, default: http://127.0.0.1:8090)
|
||||||
|
# Requires "yacy" to be in the engines list above.
|
||||||
|
#yacy_instance = http://127.0.0.1:8090
|
||||||
|
|
||||||
[rate_limit]
|
[rate_limit]
|
||||||
# Rate limit searches per interval
|
# Rate limit searches per interval
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,8 @@ display_language_label = "Display Language"
|
||||||
language_desc = "Choose your preferred language."
|
language_desc = "Choose your preferred language."
|
||||||
engines_label = "Search Engines"
|
engines_label = "Search Engines"
|
||||||
engines_desc = "Choose which search engines to use. Only engines enabled on the server are shown."
|
engines_desc = "Choose which search engines to use. Only engines enabled on the server are shown."
|
||||||
|
default_engine_label = "Default Search Engine"
|
||||||
|
all_engines_option = "All enabled engines"
|
||||||
save_settings_button = "Save Settings"
|
save_settings_button = "Save Settings"
|
||||||
no_results = "No results found"
|
no_results = "No results found"
|
||||||
error_images = "Error fetching images"
|
error_images = "Error fetching images"
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,8 @@ display_language_label = "Display Language"
|
||||||
language_desc = "Choose your preferred language."
|
language_desc = "Choose your preferred language."
|
||||||
engines_label = "Search Engines"
|
engines_label = "Search Engines"
|
||||||
engines_desc = "Choose which search engines to use. Only engines enabled on the server are shown."
|
engines_desc = "Choose which search engines to use. Only engines enabled on the server are shown."
|
||||||
|
default_engine_label = "Default Search Engine"
|
||||||
|
all_engines_option = "All enabled engines"
|
||||||
save_settings_button = "Save Settings"
|
save_settings_button = "Save Settings"
|
||||||
no_results = "No results found"
|
no_results = "No results found"
|
||||||
error_images = "Error fetching images"
|
error_images = "Error fetching images"
|
||||||
|
|
|
||||||
34
locales/fi_fi.ini
Normal file
34
locales/fi_fi.ini
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
[Meta]
|
||||||
|
Id = "fi_fi"
|
||||||
|
Name = "Suomi"
|
||||||
|
Direction = "ltr"
|
||||||
|
|
||||||
|
[Keys]
|
||||||
|
search_placeholder = "Hae netistä..."
|
||||||
|
search_button = "Hae"
|
||||||
|
surprise_me_button = "Kokeilen onneani"
|
||||||
|
all_tab = "Kaikki"
|
||||||
|
images_tab = "Kuvat"
|
||||||
|
settings_tab = "Asetukset"
|
||||||
|
settings_title = "Asetukset"
|
||||||
|
theme_label = "Ulkonäkö"
|
||||||
|
theme_desc = "Valitse haluamasi väriteema."
|
||||||
|
theme_system = "Järjestelmä"
|
||||||
|
theme_light = "Vaalea"
|
||||||
|
theme_dark = "Tumma"
|
||||||
|
language_label = "Kieli"
|
||||||
|
display_language_label = "Näyttökieli"
|
||||||
|
language_desc = "Valitse haluamasi kieli."
|
||||||
|
engines_label = "Hakukoneet"
|
||||||
|
engines_desc = "Valitse mitä hakukoneita käytetään. Vain palvelimella käyttöön otetut hakukoneet näytetään."
|
||||||
|
save_settings_button = "Tallenna asetukset"
|
||||||
|
no_results = "Ei tuloksia"
|
||||||
|
error_images = "Virhe kuvien haussa"
|
||||||
|
rate_limit = "Hidasta hieman! Sinulta on tullut liian monta hakupyyntöä!"
|
||||||
|
warning_fetch_error = "pyyntö epäonnistui ennen kuin OmniSearch ehti lukea hakutuloksia."
|
||||||
|
warning_parse_mismatch = "palautti hakutulokset muodossa jota OmniSearch ei osaa jäsentää."
|
||||||
|
warning_blocked = "palautti captchan tai muun estosivun hakutulosten sijaan."
|
||||||
|
read_more = "Lue Lisää"
|
||||||
|
view_cached = "Välimuistissa"
|
||||||
|
view_image = "Kuva"
|
||||||
|
visit_site = "Sivu"
|
||||||
64
src/Config.c
64
src/Config.c
|
|
@ -3,6 +3,17 @@
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
|
static char *trim(char *value) {
|
||||||
|
while (*value == ' ' || *value == '\t')
|
||||||
|
value++;
|
||||||
|
|
||||||
|
char *end = value + strlen(value);
|
||||||
|
while (end > value && (end[-1] == ' ' || end[-1] == '\t'))
|
||||||
|
end--;
|
||||||
|
*end = '\0';
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
int load_config(const char *filename, Config *config) {
|
int load_config(const char *filename, Config *config) {
|
||||||
FILE *file = fopen(filename, "r");
|
FILE *file = fopen(filename, "r");
|
||||||
if (!file) {
|
if (!file) {
|
||||||
|
|
@ -33,51 +44,31 @@ int load_config(const char *filename, Config *config) {
|
||||||
char *delimiter = strchr(line, '=');
|
char *delimiter = strchr(line, '=');
|
||||||
if (delimiter) {
|
if (delimiter) {
|
||||||
*delimiter = '\0';
|
*delimiter = '\0';
|
||||||
char *key = line;
|
char *key = trim(line);
|
||||||
char *value = delimiter + 1;
|
char *value = trim(delimiter + 1);
|
||||||
|
|
||||||
while (*key == ' ' || *key == '\t')
|
char *value_end = value + strlen(value);
|
||||||
key++;
|
while (value_end > value &&
|
||||||
while (*value == ' ' || *value == '\t')
|
(value_end[-1] == '"' || value_end[-1] == '\''))
|
||||||
value++;
|
*--value_end = '\0';
|
||||||
|
|
||||||
char *key_end = key + strlen(key) - 1;
|
|
||||||
while (key_end > key && (*key_end == ' ' || *key_end == '\t')) {
|
|
||||||
*key_end = '\0';
|
|
||||||
key_end--;
|
|
||||||
}
|
|
||||||
|
|
||||||
char *value_end = value + strlen(value) - 1;
|
|
||||||
while (value_end > value && (*value_end == ' ' || *value_end == '\t' ||
|
|
||||||
*value_end == '"' || *value_end == '\'')) {
|
|
||||||
*value_end = '\0';
|
|
||||||
value_end--;
|
|
||||||
}
|
|
||||||
|
|
||||||
while (*value == ' ' || *value == '\t')
|
|
||||||
value++;
|
|
||||||
while (*value == '"' || *value == '\'')
|
while (*value == '"' || *value == '\'')
|
||||||
value++;
|
value++;
|
||||||
|
|
||||||
if (strcmp(section, "server") == 0) {
|
if (strcmp(section, "server") == 0) {
|
||||||
if (strcmp(key, "host") == 0) {
|
if (strcmp(key, "host") == 0) {
|
||||||
strncpy(config->host, value, sizeof(config->host) - 1);
|
snprintf(config->host, sizeof(config->host), "%s", value);
|
||||||
config->host[sizeof(config->host) - 1] = '\0';
|
|
||||||
} else if (strcmp(key, "port") == 0) {
|
} else if (strcmp(key, "port") == 0) {
|
||||||
config->port = atoi(value);
|
config->port = atoi(value);
|
||||||
} else if (strcmp(key, "locale") == 0) {
|
} else if (strcmp(key, "locale") == 0) {
|
||||||
strncpy(config->default_locale, value,
|
snprintf(config->default_locale, sizeof(config->default_locale), "%s",
|
||||||
sizeof(config->default_locale) - 1);
|
value);
|
||||||
config->default_locale[sizeof(config->default_locale) - 1] = '\0';
|
|
||||||
}
|
}
|
||||||
} else if (strcmp(section, "proxy") == 0) {
|
} else if (strcmp(section, "proxy") == 0) {
|
||||||
if (strcmp(key, "proxy") == 0) {
|
if (strcmp(key, "proxy") == 0) {
|
||||||
strncpy(config->proxy, value, sizeof(config->proxy) - 1);
|
snprintf(config->proxy, sizeof(config->proxy), "%s", value);
|
||||||
config->proxy[sizeof(config->proxy) - 1] = '\0';
|
|
||||||
} else if (strcmp(key, "list_file") == 0) {
|
} else if (strcmp(key, "list_file") == 0) {
|
||||||
strncpy(config->proxy_list_file, value,
|
snprintf(config->proxy_list_file, sizeof(config->proxy_list_file),
|
||||||
sizeof(config->proxy_list_file) - 1);
|
"%s", value);
|
||||||
config->proxy_list_file[sizeof(config->proxy_list_file) - 1] = '\0';
|
|
||||||
} else if (strcmp(key, "max_retries") == 0) {
|
} else if (strcmp(key, "max_retries") == 0) {
|
||||||
config->max_proxy_retries = atoi(value);
|
config->max_proxy_retries = atoi(value);
|
||||||
} else if (strcmp(key, "randomize_username") == 0) {
|
} else if (strcmp(key, "randomize_username") == 0) {
|
||||||
|
|
@ -87,8 +78,7 @@ int load_config(const char *filename, Config *config) {
|
||||||
}
|
}
|
||||||
} else if (strcmp(section, "cache") == 0) {
|
} else if (strcmp(section, "cache") == 0) {
|
||||||
if (strcmp(key, "dir") == 0) {
|
if (strcmp(key, "dir") == 0) {
|
||||||
strncpy(config->cache_dir, value, sizeof(config->cache_dir) - 1);
|
snprintf(config->cache_dir, sizeof(config->cache_dir), "%s", value);
|
||||||
config->cache_dir[sizeof(config->cache_dir) - 1] = '\0';
|
|
||||||
} else if (strcmp(key, "ttl_search") == 0) {
|
} else if (strcmp(key, "ttl_search") == 0) {
|
||||||
config->cache_ttl_search = atoi(value);
|
config->cache_ttl_search = atoi(value);
|
||||||
} else if (strcmp(key, "ttl_infobox") == 0) {
|
} else if (strcmp(key, "ttl_infobox") == 0) {
|
||||||
|
|
@ -98,8 +88,10 @@ int load_config(const char *filename, Config *config) {
|
||||||
}
|
}
|
||||||
} else if (strcmp(section, "engines") == 0) {
|
} else if (strcmp(section, "engines") == 0) {
|
||||||
if (strcmp(key, "engines") == 0) {
|
if (strcmp(key, "engines") == 0) {
|
||||||
strncpy(config->engines, value, sizeof(config->engines) - 1);
|
snprintf(config->engines, sizeof(config->engines), "%s", value);
|
||||||
config->engines[sizeof(config->engines) - 1] = '\0';
|
} else if (strcmp(key, "yacy_instance") == 0) {
|
||||||
|
snprintf(config->yacy_instance, sizeof(config->yacy_instance), "%s",
|
||||||
|
value);
|
||||||
}
|
}
|
||||||
} else if (strcmp(section, "rate_limit") == 0) {
|
} else if (strcmp(section, "rate_limit") == 0) {
|
||||||
if (strcmp(key, "search_requests") == 0) {
|
if (strcmp(key, "search_requests") == 0) {
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@
|
||||||
#define DEFAULT_CACHE_TTL_INFOBOX 86400
|
#define DEFAULT_CACHE_TTL_INFOBOX 86400
|
||||||
#define DEFAULT_CACHE_TTL_IMAGE 604800
|
#define DEFAULT_CACHE_TTL_IMAGE 604800
|
||||||
#define DEFAULT_MAX_PROXY_RETRIES 3
|
#define DEFAULT_MAX_PROXY_RETRIES 3
|
||||||
|
#define DEFAULT_YACY_INSTANCE "http://127.0.0.1:8090"
|
||||||
|
|
||||||
#define BUFFER_SIZE_SMALL 256
|
#define BUFFER_SIZE_SMALL 256
|
||||||
#define BUFFER_SIZE_MEDIUM 512
|
#define BUFFER_SIZE_MEDIUM 512
|
||||||
|
|
@ -46,6 +47,7 @@ typedef struct {
|
||||||
int cache_ttl_infobox;
|
int cache_ttl_infobox;
|
||||||
int cache_ttl_image;
|
int cache_ttl_image;
|
||||||
char engines[512];
|
char engines[512];
|
||||||
|
char yacy_instance[512];
|
||||||
int rate_limit_search_requests;
|
int rate_limit_search_requests;
|
||||||
int rate_limit_search_interval;
|
int rate_limit_search_interval;
|
||||||
int rate_limit_images_requests;
|
int rate_limit_images_requests;
|
||||||
|
|
|
||||||
18
src/Main.c
18
src/Main.c
|
|
@ -78,6 +78,7 @@ int main() {
|
||||||
.cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX,
|
.cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX,
|
||||||
.cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE,
|
.cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE,
|
||||||
.engines = "",
|
.engines = "",
|
||||||
|
.yacy_instance = "",
|
||||||
.rate_limit_search_requests = 0,
|
.rate_limit_search_requests = 0,
|
||||||
.rate_limit_search_interval = 0,
|
.rate_limit_search_interval = 0,
|
||||||
.rate_limit_images_requests = 0,
|
.rate_limit_images_requests = 0,
|
||||||
|
|
@ -101,6 +102,23 @@ int main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
apply_engines_config(cfg.engines);
|
apply_engines_config(cfg.engines);
|
||||||
|
configure_yacy_engine(cfg.yacy_instance);
|
||||||
|
|
||||||
|
if (cfg.yacy_instance[0] != '\0') {
|
||||||
|
int yacy_enabled = 0;
|
||||||
|
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||||
|
if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
|
||||||
|
yacy_enabled = ENGINE_REGISTRY[i].enabled;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!yacy_enabled) {
|
||||||
|
fprintf(
|
||||||
|
stderr,
|
||||||
|
"[INFO] YaCy instance configured but the yacy engine is not "
|
||||||
|
"enabled (add it to the engines list, e.g. engines=\"*,yacy\")\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (cache_init(cfg.cache_dir) != 0) {
|
if (cache_init(cfg.cache_dir) != 0) {
|
||||||
fprintf(stderr,
|
fprintf(stderr,
|
||||||
|
|
|
||||||
|
|
@ -76,14 +76,12 @@ int images_handler(UrlParams *params) {
|
||||||
if (!error_images_msg)
|
if (!error_images_msg)
|
||||||
error_images_msg = "Error fetching images";
|
error_images_msg = "Error fetching images";
|
||||||
|
|
||||||
char ***pager_matrix = NULL;
|
StringMatrix pagination;
|
||||||
int *pager_inner_counts = NULL;
|
string_matrix_build_pagination(page, images_href_builder, (void *)raw_query,
|
||||||
int pager_count =
|
&pagination);
|
||||||
build_pagination(page, images_href_builder, (void *)raw_query,
|
if (pagination.count > 0) {
|
||||||
&pager_matrix, &pager_inner_counts);
|
context_set_array_of_arrays(&ctx, "pagination_links", pagination.rows,
|
||||||
if (pager_count > 0) {
|
pagination.count, pagination.field_counts);
|
||||||
context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix,
|
|
||||||
pager_count, pager_inner_counts);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
char *display_query = url_decode_query(raw_query);
|
char *display_query = url_decode_query(raw_query);
|
||||||
|
|
@ -91,8 +89,9 @@ int images_handler(UrlParams *params) {
|
||||||
|
|
||||||
if (!raw_query || strlen(raw_query) == 0) {
|
if (!raw_query || strlen(raw_query) == 0) {
|
||||||
send_redirect("/");
|
send_redirect("/");
|
||||||
if (display_query)
|
string_matrix_free(&pagination);
|
||||||
free(display_query);
|
free(display_query);
|
||||||
|
free(locale);
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
@ -128,7 +127,9 @@ int images_handler(UrlParams *params) {
|
||||||
snprintf(response, sizeof(response), "<h1>%s</h1>", rate_limit_msg);
|
snprintf(response, sizeof(response), "<h1>%s</h1>", rate_limit_msg);
|
||||||
send_response(response);
|
send_response(response);
|
||||||
free(request_cache_key);
|
free(request_cache_key);
|
||||||
|
string_matrix_free(&pagination);
|
||||||
free(display_query);
|
free(display_query);
|
||||||
|
free(locale);
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
@ -147,37 +148,37 @@ int images_handler(UrlParams *params) {
|
||||||
snprintf(error_html, sizeof(error_html), "<h1>%s</h1>", error_images_msg);
|
snprintf(error_html, sizeof(error_html), "<h1>%s</h1>", error_images_msg);
|
||||||
send_response(error_html);
|
send_response(error_html);
|
||||||
free(request_cache_key);
|
free(request_cache_key);
|
||||||
|
string_matrix_free(&pagination);
|
||||||
free(display_query);
|
free(display_query);
|
||||||
|
free(locale);
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
char ***image_matrix = malloc(sizeof(char **) * result_count);
|
StringMatrix images;
|
||||||
int *inner_counts = malloc(sizeof(int) * result_count);
|
string_matrix_init(&images);
|
||||||
|
|
||||||
if (!image_matrix || !inner_counts) {
|
|
||||||
if (image_matrix)
|
|
||||||
free(image_matrix);
|
|
||||||
if (inner_counts)
|
|
||||||
free(inner_counts);
|
|
||||||
free_image_results(results, result_count);
|
|
||||||
free(request_cache_key);
|
|
||||||
free(display_query);
|
|
||||||
free_context(&ctx);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < result_count; i++) {
|
for (int i = 0; i < result_count; i++) {
|
||||||
image_matrix[i] = malloc(sizeof(char *) * IMAGE_RESULT_FIELDS);
|
const char *values[IMAGE_RESULT_FIELDS] = {
|
||||||
image_matrix[i][0] = strdup(results[i].thumbnail_url);
|
results[i].thumbnail_url,
|
||||||
image_matrix[i][1] = strdup(results[i].title);
|
results[i].title,
|
||||||
image_matrix[i][2] = strdup(results[i].page_url);
|
results[i].page_url,
|
||||||
image_matrix[i][3] = strdup(results[i].full_url);
|
results[i].full_url,
|
||||||
inner_counts[i] = IMAGE_RESULT_FIELDS;
|
};
|
||||||
|
if (string_matrix_append(&images, values, IMAGE_RESULT_FIELDS) != 0) {
|
||||||
|
string_matrix_free(&images);
|
||||||
|
string_matrix_free(&pagination);
|
||||||
|
free_image_results(results, result_count);
|
||||||
|
free(request_cache_key);
|
||||||
|
free(display_query);
|
||||||
|
free(locale);
|
||||||
|
free_context(&ctx);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
context_set_array_of_arrays(&ctx, "images", image_matrix, result_count,
|
context_set_array_of_arrays(&ctx, "images", images.rows, images.count,
|
||||||
inner_counts);
|
images.field_counts);
|
||||||
|
|
||||||
char *rendered = render_template("images.html", &ctx);
|
char *rendered = render_template("images.html", &ctx);
|
||||||
if (rendered) {
|
if (rendered) {
|
||||||
|
|
@ -187,27 +188,13 @@ int images_handler(UrlParams *params) {
|
||||||
send_response("<h1>Error rendering image results</h1>");
|
send_response("<h1>Error rendering image results</h1>");
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < result_count; i++) {
|
string_matrix_free(&images);
|
||||||
for (int j = 0; j < IMAGE_RESULT_FIELDS; j++)
|
string_matrix_free(&pagination);
|
||||||
free(image_matrix[i][j]);
|
|
||||||
free(image_matrix[i]);
|
|
||||||
}
|
|
||||||
free(image_matrix);
|
|
||||||
free(inner_counts);
|
|
||||||
|
|
||||||
if (pager_count > 0) {
|
|
||||||
for (int i = 0; i < pager_count; i++) {
|
|
||||||
for (int j = 0; j < LINK_FIELD_COUNT; j++)
|
|
||||||
free(pager_matrix[i][j]);
|
|
||||||
free(pager_matrix[i]);
|
|
||||||
}
|
|
||||||
free(pager_matrix);
|
|
||||||
free(pager_inner_counts);
|
|
||||||
}
|
|
||||||
|
|
||||||
free_image_results(results, result_count);
|
free_image_results(results, result_count);
|
||||||
free(request_cache_key);
|
free(request_cache_key);
|
||||||
free(display_query);
|
free(display_query);
|
||||||
|
free(locale);
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
||||||
|
|
@ -140,29 +140,31 @@ static InfoBox fetch_currency_wrapper(char *query) {
|
||||||
return fetch_currency_data(query);
|
return fetch_currency_data(query);
|
||||||
}
|
}
|
||||||
char *get_base_url(const char *input) {
|
char *get_base_url(const char *input) {
|
||||||
if (!input) return NULL;
|
if (!input)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
const char *start = input;
|
const char *start = input;
|
||||||
|
|
||||||
const char *protocol_pos = strstr(input, "://");
|
const char *protocol_pos = strstr(input, "://");
|
||||||
if (protocol_pos) {
|
if (protocol_pos) {
|
||||||
start = protocol_pos + 3;
|
start = protocol_pos + 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
const char *end = start;
|
const char *end = start;
|
||||||
while (*end && *end != '/' && *end != '?' && *end != '#') {
|
while (*end && *end != '/' && *end != '?' && *end != '#') {
|
||||||
end++;
|
end++;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t len = end - start;
|
size_t len = end - start;
|
||||||
|
|
||||||
char *domain = (char *)malloc(len + 1);
|
char *domain = (char *)malloc(len + 1);
|
||||||
if (!domain) return NULL;
|
if (!domain)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
strncpy(domain, start, len);
|
strncpy(domain, start, len);
|
||||||
domain[len] = '\0';
|
domain[len] = '\0';
|
||||||
|
|
||||||
return domain;
|
return domain;
|
||||||
}
|
}
|
||||||
static int is_calculator_query(const char *query) {
|
static int is_calculator_query(const char *query) {
|
||||||
if (!query)
|
if (!query)
|
||||||
|
|
@ -249,88 +251,65 @@ static void *infobox_thread_func(void *arg) {
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int add_infobox_to_collection(InfoBox *infobox, char ****collection,
|
static int add_infobox_to_collection(InfoBox *infobox,
|
||||||
int **inner_counts, int current_count) {
|
StringMatrix *collection) {
|
||||||
*collection =
|
const char *values[INFOBOX_FIELD_COUNT] = {
|
||||||
(char ***)realloc(*collection, sizeof(char **) * (current_count + 1));
|
infobox->title, infobox->thumbnail_url, infobox->extract,
|
||||||
*inner_counts =
|
infobox->url, infobox->url,
|
||||||
(int *)realloc(*inner_counts, sizeof(int) * (current_count + 1));
|
};
|
||||||
|
return string_matrix_append(collection, values, INFOBOX_FIELD_COUNT);
|
||||||
(*collection)[current_count] =
|
|
||||||
(char **)malloc(sizeof(char *) * INFOBOX_FIELD_COUNT);
|
|
||||||
(*collection)[current_count][0] =
|
|
||||||
infobox->title ? strdup(infobox->title) : NULL;
|
|
||||||
(*collection)[current_count][1] =
|
|
||||||
infobox->thumbnail_url ? strdup(infobox->thumbnail_url) : NULL;
|
|
||||||
(*collection)[current_count][2] =
|
|
||||||
infobox->extract ? strdup(infobox->extract) : NULL;
|
|
||||||
(*collection)[current_count][3] = infobox->url ? strdup(infobox->url) : NULL;
|
|
||||||
(*collection)[current_count][4] = infobox->url ? strdup(infobox->url) : NULL;
|
|
||||||
(*inner_counts)[current_count] = INFOBOX_FIELD_COUNT;
|
|
||||||
|
|
||||||
return current_count + 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static int add_warning_to_collection(const char *engine_name,
|
static int add_warning_to_collection(const char *engine_name,
|
||||||
const char *warning_message,
|
const char *warning_message,
|
||||||
char ****collection, int **inner_counts,
|
StringMatrix *collection) {
|
||||||
int current_count) {
|
const char *values[] = {engine_name, warning_message};
|
||||||
char ***new_collection =
|
return string_matrix_append(collection, values, 2);
|
||||||
(char ***)malloc(sizeof(char **) * (current_count + 1));
|
|
||||||
int *new_inner_counts =
|
|
||||||
(int *)malloc(sizeof(int) * (current_count + 1));
|
|
||||||
|
|
||||||
if (!new_collection || !new_inner_counts) {
|
|
||||||
free(new_collection);
|
|
||||||
free(new_inner_counts);
|
|
||||||
return current_count;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (*collection && current_count > 0) {
|
|
||||||
memcpy(new_collection, *collection, sizeof(char **) * current_count);
|
|
||||||
}
|
|
||||||
if (*inner_counts && current_count > 0) {
|
|
||||||
memcpy(new_inner_counts, *inner_counts, sizeof(int) * current_count);
|
|
||||||
}
|
|
||||||
|
|
||||||
free(*collection);
|
|
||||||
free(*inner_counts);
|
|
||||||
|
|
||||||
*collection = new_collection;
|
|
||||||
*inner_counts = new_inner_counts;
|
|
||||||
|
|
||||||
(*collection)[current_count] = (char **)malloc(sizeof(char *) * 2);
|
|
||||||
if (!(*collection)[current_count])
|
|
||||||
return current_count;
|
|
||||||
|
|
||||||
(*collection)[current_count][0] = strdup(engine_name ? engine_name : "");
|
|
||||||
(*collection)[current_count][1] =
|
|
||||||
strdup(warning_message ? warning_message : "");
|
|
||||||
|
|
||||||
if (!(*collection)[current_count][0] || !(*collection)[current_count][1]) {
|
|
||||||
free((*collection)[current_count][0]);
|
|
||||||
free((*collection)[current_count][1]);
|
|
||||||
free((*collection)[current_count]);
|
|
||||||
return current_count;
|
|
||||||
}
|
|
||||||
|
|
||||||
(*inner_counts)[current_count] = 2;
|
|
||||||
return current_count + 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static const char *warning_message_for_job(const ScrapeJob *job, const char *locale) {
|
static void free_user_engine_list(char **user_engines, int user_engine_count) {
|
||||||
|
for (int i = 0; i < user_engine_count; i++)
|
||||||
|
free(user_engines[i]);
|
||||||
|
free(user_engines);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void free_search_results(SearchResult **all_results, ScrapeJob *jobs,
|
||||||
|
int job_count) {
|
||||||
|
for (int i = 0; i < job_count; i++) {
|
||||||
|
for (int j = 0; all_results[i] && j < jobs[i].results_count; j++) {
|
||||||
|
free(all_results[i][j].url);
|
||||||
|
free(all_results[i][j].title);
|
||||||
|
free(all_results[i][j].snippet);
|
||||||
|
}
|
||||||
|
free(all_results[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void free_infobox_results(InfoBoxThreadData *data) {
|
||||||
|
for (int i = 0; i < HANDLER_COUNT; i++) {
|
||||||
|
free_infobox(&data[i].result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static const char *warning_message_for_job(const ScrapeJob *job,
|
||||||
|
const char *locale) {
|
||||||
switch (job->status) {
|
switch (job->status) {
|
||||||
case SCRAPE_STATUS_FETCH_ERROR: {
|
case SCRAPE_STATUS_FETCH_ERROR: {
|
||||||
const char *msg = beaker_get_locale_value(locale, "warning_fetch_error");
|
const char *msg = beaker_get_locale_value(locale, "warning_fetch_error");
|
||||||
return msg ? msg : "request failed before OmniSearch could read search results.";
|
return msg ? msg
|
||||||
|
: "request failed before OmniSearch could read search results.";
|
||||||
}
|
}
|
||||||
case SCRAPE_STATUS_PARSE_MISMATCH: {
|
case SCRAPE_STATUS_PARSE_MISMATCH: {
|
||||||
const char *msg = beaker_get_locale_value(locale, "warning_parse_mismatch");
|
const char *msg = beaker_get_locale_value(locale, "warning_parse_mismatch");
|
||||||
return msg ? msg : "returned search results in a format OmniSearch could not parse.";
|
return msg ? msg
|
||||||
|
: "returned search results in a format OmniSearch could not "
|
||||||
|
"parse.";
|
||||||
}
|
}
|
||||||
case SCRAPE_STATUS_BLOCKED: {
|
case SCRAPE_STATUS_BLOCKED: {
|
||||||
const char *msg = beaker_get_locale_value(locale, "warning_blocked");
|
const char *msg = beaker_get_locale_value(locale, "warning_blocked");
|
||||||
return msg ? msg : "returned a captcha or another blocking page instead of search results.";
|
return msg ? msg
|
||||||
|
: "returned a captcha or another blocking page instead of "
|
||||||
|
"search results.";
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
@ -384,8 +363,8 @@ static int engine_allowed_for_user(const SearchEngine *eng, char **user_ids,
|
||||||
static char *build_search_href(const char *query, const char *engine_id,
|
static char *build_search_href(const char *query, const char *engine_id,
|
||||||
int page) {
|
int page) {
|
||||||
const char *safe_query = query ? query : "";
|
const char *safe_query = query ? query : "";
|
||||||
int use_engine = engine_id && engine_id[0] != '\0' &&
|
int use_engine =
|
||||||
!engine_id_matches(engine_id, "all");
|
engine_id && engine_id[0] != '\0' && !engine_id_matches(engine_id, "all");
|
||||||
size_t needed = strlen("/search?q=") + strlen(safe_query) + 1;
|
size_t needed = strlen("/search?q=") + strlen(safe_query) + 1;
|
||||||
|
|
||||||
if (use_engine)
|
if (use_engine)
|
||||||
|
|
@ -437,13 +416,18 @@ int results_handler(UrlParams *params) {
|
||||||
extern Config global_config;
|
extern Config global_config;
|
||||||
TemplateContext ctx = new_context();
|
TemplateContext ctx = new_context();
|
||||||
char *raw_query = "";
|
char *raw_query = "";
|
||||||
const char *selected_engine_id = "all";
|
char selected_engine_id[32];
|
||||||
|
char *default_engine = get_default_search_engine();
|
||||||
|
snprintf(selected_engine_id, sizeof(selected_engine_id), "%s",
|
||||||
|
default_engine);
|
||||||
|
free(default_engine);
|
||||||
int page = 1;
|
int page = 1;
|
||||||
int btnI = 0;
|
int btnI = 0;
|
||||||
|
|
||||||
char **user_engines = NULL;
|
char **user_engines = NULL;
|
||||||
int user_engine_count = 0;
|
int user_engine_count = 0;
|
||||||
int has_user_pref = (get_user_engines(&user_engines, &user_engine_count) == 0);
|
int has_user_pref =
|
||||||
|
(get_user_engines(&user_engines, &user_engine_count) == 0);
|
||||||
|
|
||||||
if (params) {
|
if (params) {
|
||||||
for (int i = 0; i < params->count; i++) {
|
for (int i = 0; i < params->count; i++) {
|
||||||
|
|
@ -454,7 +438,8 @@ int results_handler(UrlParams *params) {
|
||||||
if (parsed > 1)
|
if (parsed > 1)
|
||||||
page = parsed;
|
page = parsed;
|
||||||
} else if (strcmp(params->params[i].key, "engine") == 0) {
|
} else if (strcmp(params->params[i].key, "engine") == 0) {
|
||||||
selected_engine_id = params->params[i].value;
|
snprintf(selected_engine_id, sizeof(selected_engine_id), "%s",
|
||||||
|
params->params[i].value);
|
||||||
} else if (strcmp(params->params[i].key, "btnI") == 0) {
|
} else if (strcmp(params->params[i].key, "btnI") == 0) {
|
||||||
btnI = atoi(params->params[i].value);
|
btnI = atoi(params->params[i].value);
|
||||||
}
|
}
|
||||||
|
|
@ -471,9 +456,11 @@ int results_handler(UrlParams *params) {
|
||||||
beaker_set_locale(&ctx, locale);
|
beaker_set_locale(&ctx, locale);
|
||||||
|
|
||||||
const char *rate_limit_msg = beaker_get_locale_value(locale, "rate_limit");
|
const char *rate_limit_msg = beaker_get_locale_value(locale, "rate_limit");
|
||||||
if (!rate_limit_msg) rate_limit_msg = "Slow down! Too many searches from you!";
|
if (!rate_limit_msg)
|
||||||
|
rate_limit_msg = "Slow down! Too many searches from you!";
|
||||||
const char *no_results_msg = beaker_get_locale_value(locale, "no_results");
|
const char *no_results_msg = beaker_get_locale_value(locale, "no_results");
|
||||||
if (!no_results_msg) no_results_msg = "No results found";
|
if (!no_results_msg)
|
||||||
|
no_results_msg = "No results found";
|
||||||
|
|
||||||
char page_str[16];
|
char page_str[16];
|
||||||
snprintf(page_str, sizeof(page_str), "%d", page);
|
snprintf(page_str, sizeof(page_str), "%d", page);
|
||||||
|
|
@ -491,18 +478,24 @@ int results_handler(UrlParams *params) {
|
||||||
|
|
||||||
if (!raw_query || strlen(raw_query) == 0) {
|
if (!raw_query || strlen(raw_query) == 0) {
|
||||||
send_redirect("/");
|
send_redirect("/");
|
||||||
if (has_user_pref) {
|
free(locale);
|
||||||
for (int i = 0; i < user_engine_count; i++)
|
free_user_engine_list(user_engines, user_engine_count);
|
||||||
free(user_engines[i]);
|
|
||||||
free(user_engines);
|
|
||||||
}
|
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
const SearchEngine *selected_engine = find_enabled_engine(selected_engine_id);
|
const SearchEngine *selected_engine = find_enabled_engine(selected_engine_id);
|
||||||
if (!selected_engine)
|
if (selected_engine &&
|
||||||
selected_engine_id = "all";
|
!engine_allowed_for_user(selected_engine, user_engines,
|
||||||
|
user_engine_count, has_user_pref)) {
|
||||||
|
selected_engine = NULL;
|
||||||
|
}
|
||||||
|
if (selected_engine) {
|
||||||
|
snprintf(selected_engine_id, sizeof(selected_engine_id), "%s",
|
||||||
|
selected_engine->id);
|
||||||
|
} else {
|
||||||
|
snprintf(selected_engine_id, sizeof(selected_engine_id), "all");
|
||||||
|
}
|
||||||
|
|
||||||
context_set(&ctx, "selected_engine", selected_engine_id);
|
context_set(&ctx, "selected_engine", selected_engine_id);
|
||||||
char *search_href = build_search_href(raw_query, selected_engine_id, 1);
|
char *search_href = build_search_href(raw_query, selected_engine_id, 1);
|
||||||
|
|
@ -528,13 +521,6 @@ int results_handler(UrlParams *params) {
|
||||||
infobox_data[i].result = (InfoBox){NULL};
|
infobox_data[i].result = (InfoBox){NULL};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (page == 1) {
|
|
||||||
for (int i = 0; i < HANDLER_COUNT; i++) {
|
|
||||||
pthread_create(&infobox_threads[i], NULL, infobox_thread_func,
|
|
||||||
&infobox_data[i]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ScrapeJob jobs[ENGINE_COUNT];
|
ScrapeJob jobs[ENGINE_COUNT];
|
||||||
SearchResult *all_results[ENGINE_COUNT];
|
SearchResult *all_results[ENGINE_COUNT];
|
||||||
|
|
||||||
|
|
@ -592,11 +578,8 @@ int results_handler(UrlParams *params) {
|
||||||
snprintf(response, sizeof(response), "<h1>%s</h1>", rate_limit_msg);
|
snprintf(response, sizeof(response), "<h1>%s</h1>", rate_limit_msg);
|
||||||
send_response(response);
|
send_response(response);
|
||||||
free(request_cache_key);
|
free(request_cache_key);
|
||||||
if (has_user_pref) {
|
free(locale);
|
||||||
for (int i = 0; i < user_engine_count; i++)
|
free_user_engine_list(user_engines, user_engine_count);
|
||||||
free(user_engines[i]);
|
|
||||||
free(user_engines);
|
|
||||||
}
|
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
@ -615,15 +598,14 @@ int results_handler(UrlParams *params) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (filter_engine_count > 1) {
|
if (filter_engine_count > 1) {
|
||||||
char ***filter_matrix = NULL;
|
StringMatrix filter_matrix;
|
||||||
int *filter_inner_counts = NULL;
|
string_matrix_init(&filter_matrix);
|
||||||
int filter_count = 0;
|
|
||||||
char *all_href = build_search_href(raw_query, "all", 1);
|
char *all_href = build_search_href(raw_query, "all", 1);
|
||||||
|
const char *all_values[LINK_FIELD_COUNT] = {
|
||||||
filter_count = add_link_to_collection(
|
|
||||||
all_href, "All",
|
all_href, "All",
|
||||||
selected_engine ? "engine-filter" : "engine-filter active",
|
selected_engine ? "engine-filter" : "engine-filter active"};
|
||||||
&filter_matrix, &filter_inner_counts, filter_count);
|
|
||||||
|
string_matrix_append(&filter_matrix, all_values, LINK_FIELD_COUNT);
|
||||||
free(all_href);
|
free(all_href);
|
||||||
|
|
||||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||||
|
|
@ -639,22 +621,24 @@ int results_handler(UrlParams *params) {
|
||||||
? "engine-filter active"
|
? "engine-filter active"
|
||||||
: "engine-filter";
|
: "engine-filter";
|
||||||
|
|
||||||
filter_count = add_link_to_collection(filter_href, ENGINE_REGISTRY[i].name,
|
const char *filter_values[LINK_FIELD_COUNT] = {
|
||||||
filter_class, &filter_matrix,
|
filter_href, ENGINE_REGISTRY[i].name, filter_class};
|
||||||
&filter_inner_counts, filter_count);
|
string_matrix_append(&filter_matrix, filter_values, LINK_FIELD_COUNT);
|
||||||
free(filter_href);
|
free(filter_href);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (filter_count > 0) {
|
if (filter_matrix.count > 0) {
|
||||||
context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix,
|
context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix.rows,
|
||||||
filter_count, filter_inner_counts);
|
filter_matrix.count,
|
||||||
for (int i = 0; i < filter_count; i++) {
|
filter_matrix.field_counts);
|
||||||
for (int j = 0; j < LINK_FIELD_COUNT; j++)
|
}
|
||||||
free(filter_matrix[i][j]);
|
string_matrix_free(&filter_matrix);
|
||||||
free(filter_matrix[i]);
|
}
|
||||||
}
|
|
||||||
free(filter_matrix);
|
if (page == 1) {
|
||||||
free(filter_inner_counts);
|
for (int i = 0; i < HANDLER_COUNT; i++) {
|
||||||
|
pthread_create(&infobox_threads[i], NULL, infobox_thread_func,
|
||||||
|
&infobox_data[i]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -672,27 +656,12 @@ int results_handler(UrlParams *params) {
|
||||||
for (int i = 0; i < engine_idx; i++) {
|
for (int i = 0; i < engine_idx; i++) {
|
||||||
if (jobs[i].results_count > 0 && all_results[i][0].url) {
|
if (jobs[i].results_count > 0 && all_results[i][0].url) {
|
||||||
char *redirect_url = strdup(all_results[i][0].url);
|
char *redirect_url = strdup(all_results[i][0].url);
|
||||||
for (int j = 0; j < enabled_engine_count; j++) {
|
free_search_results(all_results, jobs, engine_idx);
|
||||||
for (int k = 0; k < jobs[j].results_count; k++) {
|
if (page == 1)
|
||||||
free(all_results[j][k].url);
|
free_infobox_results(infobox_data);
|
||||||
free(all_results[j][k].title);
|
|
||||||
free(all_results[j][k].snippet);
|
|
||||||
}
|
|
||||||
free(all_results[j]);
|
|
||||||
}
|
|
||||||
if (page == 1) {
|
|
||||||
for (int j = 0; j < HANDLER_COUNT; j++) {
|
|
||||||
if (infobox_data[j].success) {
|
|
||||||
free_infobox(&infobox_data[j].result);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
free(request_cache_key);
|
free(request_cache_key);
|
||||||
if (has_user_pref) {
|
free(locale);
|
||||||
for (int i = 0; i < user_engine_count; i++)
|
free_user_engine_list(user_engines, user_engine_count);
|
||||||
free(user_engines[i]);
|
|
||||||
free(user_engines);
|
|
||||||
}
|
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
if (redirect_url) {
|
if (redirect_url) {
|
||||||
send_redirect(redirect_url);
|
send_redirect(redirect_url);
|
||||||
|
|
@ -701,54 +670,37 @@ int results_handler(UrlParams *params) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (int i = 0; i < enabled_engine_count; i++) {
|
free_search_results(all_results, jobs, engine_idx);
|
||||||
free(all_results[i]);
|
if (page == 1)
|
||||||
}
|
free_infobox_results(infobox_data);
|
||||||
if (page == 1) {
|
|
||||||
for (int i = 0; i < HANDLER_COUNT; i++) {
|
|
||||||
if (infobox_data[i].success) {
|
|
||||||
free_infobox(&infobox_data[i].result);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
free(request_cache_key);
|
free(request_cache_key);
|
||||||
if (has_user_pref) {
|
free(locale);
|
||||||
for (int i = 0; i < user_engine_count; i++)
|
free_user_engine_list(user_engines, user_engine_count);
|
||||||
free(user_engines[i]);
|
|
||||||
free(user_engines);
|
|
||||||
}
|
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
char no_results_html[128];
|
char no_results_html[128];
|
||||||
snprintf(no_results_html, sizeof(no_results_html), "<h1>%s</h1>", no_results_msg);
|
snprintf(no_results_html, sizeof(no_results_html), "<h1>%s</h1>",
|
||||||
|
no_results_msg);
|
||||||
send_response(no_results_html);
|
send_response(no_results_html);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
char ***infobox_matrix = NULL;
|
StringMatrix infobox_matrix;
|
||||||
int *infobox_inner_counts = NULL;
|
string_matrix_init(&infobox_matrix);
|
||||||
int infobox_count = 0;
|
|
||||||
|
|
||||||
if (page == 1) {
|
if (page == 1) {
|
||||||
for (int i = 0; i < HANDLER_COUNT; i++) {
|
for (int i = 0; i < HANDLER_COUNT; i++) {
|
||||||
if (infobox_data[i].success) {
|
if (infobox_data[i].success) {
|
||||||
infobox_count =
|
add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix);
|
||||||
add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix,
|
|
||||||
&infobox_inner_counts, infobox_count);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (infobox_count > 0) {
|
if (infobox_matrix.count > 0) {
|
||||||
context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix,
|
context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix.rows,
|
||||||
infobox_count, infobox_inner_counts);
|
infobox_matrix.count,
|
||||||
for (int i = 0; i < infobox_count; i++) {
|
infobox_matrix.field_counts);
|
||||||
for (int j = 0; j < INFOBOX_FIELD_COUNT; j++)
|
|
||||||
free(infobox_matrix[i][j]);
|
|
||||||
free(infobox_matrix[i]);
|
|
||||||
}
|
|
||||||
free(infobox_matrix);
|
|
||||||
free(infobox_inner_counts);
|
|
||||||
}
|
}
|
||||||
|
string_matrix_free(&infobox_matrix);
|
||||||
|
|
||||||
int warning_count = 0;
|
int warning_count = 0;
|
||||||
for (int i = 0; i < enabled_engine_count; i++) {
|
for (int i = 0; i < enabled_engine_count; i++) {
|
||||||
|
|
@ -757,35 +709,25 @@ int results_handler(UrlParams *params) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (warning_count > 0) {
|
if (warning_count > 0) {
|
||||||
char ***warning_matrix = NULL;
|
StringMatrix warning_matrix;
|
||||||
int *warning_inner_counts = NULL;
|
string_matrix_init(&warning_matrix);
|
||||||
int warning_index = 0;
|
|
||||||
|
|
||||||
for (int i = 0; i < enabled_engine_count; i++) {
|
for (int i = 0; i < enabled_engine_count; i++) {
|
||||||
const char *warning_message = warning_message_for_job(&jobs[i], locale);
|
const char *warning_message = warning_message_for_job(&jobs[i], locale);
|
||||||
if (!warning_message)
|
if (!warning_message)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
warning_index = add_warning_to_collection(
|
add_warning_to_collection(jobs[i].engine->name, warning_message,
|
||||||
jobs[i].engine->name, warning_message, &warning_matrix,
|
&warning_matrix);
|
||||||
&warning_inner_counts, warning_index);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (warning_index > 0) {
|
if (warning_matrix.count > 0) {
|
||||||
context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix,
|
context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix.rows,
|
||||||
warning_index, warning_inner_counts);
|
warning_matrix.count,
|
||||||
|
warning_matrix.field_counts);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (warning_matrix) {
|
string_matrix_free(&warning_matrix);
|
||||||
for (int i = 0; i < warning_index; i++) {
|
|
||||||
free(warning_matrix[i][0]);
|
|
||||||
free(warning_matrix[i][1]);
|
|
||||||
free(warning_matrix[i]);
|
|
||||||
}
|
|
||||||
free(warning_matrix);
|
|
||||||
}
|
|
||||||
if (warning_inner_counts)
|
|
||||||
free(warning_inner_counts);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int total_results = 0;
|
int total_results = 0;
|
||||||
|
|
@ -794,33 +736,8 @@ int results_handler(UrlParams *params) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (total_results > 0) {
|
if (total_results > 0) {
|
||||||
char ***results_matrix = (char ***)malloc(sizeof(char **) * total_results);
|
StringMatrix results_matrix;
|
||||||
int *results_inner_counts = (int *)malloc(sizeof(int) * total_results);
|
string_matrix_init(&results_matrix);
|
||||||
if (!results_matrix || !results_inner_counts) {
|
|
||||||
char *html = render_template("results.html", &ctx);
|
|
||||||
if (html) {
|
|
||||||
send_response(html);
|
|
||||||
free(html);
|
|
||||||
}
|
|
||||||
for (int i = 0; i < enabled_engine_count; i++)
|
|
||||||
free(all_results[i]);
|
|
||||||
if (page == 1) {
|
|
||||||
for (int i = 0; i < HANDLER_COUNT; i++) {
|
|
||||||
if (infobox_data[i].success) {
|
|
||||||
free_infobox(&infobox_data[i].result);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
free(request_cache_key);
|
|
||||||
if (has_user_pref) {
|
|
||||||
for (int i = 0; i < user_engine_count; i++)
|
|
||||||
free(user_engines[i]);
|
|
||||||
free(user_engines);
|
|
||||||
}
|
|
||||||
free_context(&ctx);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
int unique_count = 0;
|
|
||||||
UrlHashTable url_table;
|
UrlHashTable url_table;
|
||||||
url_hash_init(&url_table);
|
url_hash_init(&url_table);
|
||||||
|
|
||||||
|
|
@ -837,61 +754,45 @@ int results_handler(UrlParams *params) {
|
||||||
|
|
||||||
url_hash_insert(&url_table, display_url);
|
url_hash_insert(&url_table, display_url);
|
||||||
|
|
||||||
results_matrix[unique_count] =
|
|
||||||
(char **)malloc(sizeof(char *) * RESULT_FIELD_COUNT);
|
|
||||||
if (!results_matrix[unique_count]) {
|
|
||||||
free(all_results[i][j].url);
|
|
||||||
free(all_results[i][j].title);
|
|
||||||
free(all_results[i][j].snippet);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
char *pretty_url = pretty_display_url(display_url);
|
char *pretty_url = pretty_display_url(display_url);
|
||||||
char *base_url = get_base_url(display_url);
|
char *base_url = get_base_url(display_url);
|
||||||
|
const char *values[RESULT_FIELD_COUNT] = {
|
||||||
results_matrix[unique_count][0] = strdup(display_url);
|
display_url,
|
||||||
results_matrix[unique_count][1] = strdup(pretty_url);
|
pretty_url,
|
||||||
results_matrix[unique_count][2] = all_results[i][j].title
|
all_results[i][j].title ? all_results[i][j].title : "Untitled",
|
||||||
? strdup(all_results[i][j].title)
|
all_results[i][j].snippet ? all_results[i][j].snippet : "",
|
||||||
: strdup("Untitled");
|
base_url,
|
||||||
results_matrix[unique_count][3] =
|
"",
|
||||||
all_results[i][j].snippet ? strdup(all_results[i][j].snippet)
|
};
|
||||||
: strdup("");
|
int append_result =
|
||||||
results_matrix[unique_count][4] = strdup(base_url ? base_url : "");
|
string_matrix_append(&results_matrix, values, RESULT_FIELD_COUNT);
|
||||||
results_matrix[unique_count][5] = strdup("");
|
|
||||||
|
|
||||||
results_inner_counts[unique_count] = RESULT_FIELD_COUNT;
|
|
||||||
|
|
||||||
free(pretty_url);
|
free(pretty_url);
|
||||||
free(base_url);
|
free(base_url);
|
||||||
free(all_results[i][j].url);
|
free(all_results[i][j].url);
|
||||||
free(all_results[i][j].title);
|
free(all_results[i][j].title);
|
||||||
free(all_results[i][j].snippet);
|
free(all_results[i][j].snippet);
|
||||||
|
if (append_result != 0) {
|
||||||
unique_count++;
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
free(all_results[i]);
|
free(all_results[i]);
|
||||||
}
|
}
|
||||||
|
|
||||||
context_set_array_of_arrays(&ctx, "results", results_matrix, unique_count,
|
context_set_array_of_arrays(&ctx, "results", results_matrix.rows,
|
||||||
results_inner_counts);
|
results_matrix.count,
|
||||||
|
results_matrix.field_counts);
|
||||||
|
|
||||||
char ***pager_matrix = NULL;
|
StringMatrix pager_matrix;
|
||||||
int *pager_inner_counts = NULL;
|
SearchHrefData href_data = {.query = raw_query,
|
||||||
SearchHrefData href_data = { .query = raw_query, .engine_id = selected_engine_id };
|
.engine_id = selected_engine_id};
|
||||||
int pager_count = build_pagination(page, search_href_builder,
|
int pagination_result = string_matrix_build_pagination(
|
||||||
&href_data, &pager_matrix,
|
page, search_href_builder, &href_data, &pager_matrix);
|
||||||
&pager_inner_counts);
|
|
||||||
|
|
||||||
if (pager_count > 0) {
|
if (pagination_result == 0 && pager_matrix.count > 0) {
|
||||||
context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix,
|
context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix.rows,
|
||||||
pager_count, pager_inner_counts);
|
pager_matrix.count,
|
||||||
for (int i = 0; i < pager_count; i++) {
|
pager_matrix.field_counts);
|
||||||
for (int j = 0; j < LINK_FIELD_COUNT; j++)
|
|
||||||
free(pager_matrix[i][j]);
|
|
||||||
free(pager_matrix[i]);
|
|
||||||
}
|
|
||||||
free(pager_matrix);
|
|
||||||
free(pager_inner_counts);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
char *html = render_template("results.html", &ctx);
|
char *html = render_template("results.html", &ctx);
|
||||||
|
|
@ -900,13 +801,8 @@ int results_handler(UrlParams *params) {
|
||||||
free(html);
|
free(html);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < unique_count; i++) {
|
string_matrix_free(&pager_matrix);
|
||||||
for (int j = 0; j < RESULT_FIELD_COUNT; j++)
|
string_matrix_free(&results_matrix);
|
||||||
free(results_matrix[i][j]);
|
|
||||||
free(results_matrix[i]);
|
|
||||||
}
|
|
||||||
free(results_matrix);
|
|
||||||
free(results_inner_counts);
|
|
||||||
url_hash_free(&url_table);
|
url_hash_free(&url_table);
|
||||||
} else {
|
} else {
|
||||||
char *html = render_template("results.html", &ctx);
|
char *html = render_template("results.html", &ctx);
|
||||||
|
|
@ -915,26 +811,15 @@ int results_handler(UrlParams *params) {
|
||||||
free(html);
|
free(html);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < enabled_engine_count; i++) {
|
free_search_results(all_results, jobs, engine_idx);
|
||||||
free(all_results[i]);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
free(request_cache_key);
|
free(request_cache_key);
|
||||||
|
|
||||||
if (page == 1) {
|
if (page == 1)
|
||||||
for (int i = 0; i < HANDLER_COUNT; i++) {
|
free_infobox_results(infobox_data);
|
||||||
if (infobox_data[i].success) {
|
|
||||||
free_infobox(&infobox_data[i].result);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
free(locale);
|
free(locale);
|
||||||
if (has_user_pref) {
|
free_user_engine_list(user_engines, user_engine_count);
|
||||||
for (int i = 0; i < user_engine_count; i++)
|
|
||||||
free(user_engines[i]);
|
|
||||||
free(user_engines);
|
|
||||||
}
|
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,14 @@ int settings_handler(UrlParams *params) {
|
||||||
int user_engine_count = 0;
|
int user_engine_count = 0;
|
||||||
int has_user_pref =
|
int has_user_pref =
|
||||||
(get_user_engines(&user_engines, &user_engine_count) == 0);
|
(get_user_engines(&user_engines, &user_engine_count) == 0);
|
||||||
|
char *default_engine = get_default_search_engine();
|
||||||
|
|
||||||
|
if (strcmp(default_engine, "all") != 0 && has_user_pref &&
|
||||||
|
!user_engines_contains(default_engine, user_engines,
|
||||||
|
user_engine_count)) {
|
||||||
|
free(default_engine);
|
||||||
|
default_engine = strdup("all");
|
||||||
|
}
|
||||||
|
|
||||||
int enabled_count = 0;
|
int enabled_count = 0;
|
||||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||||
|
|
@ -63,11 +71,13 @@ int settings_handler(UrlParams *params) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
engine_data[idx] = malloc(sizeof(char *) * 3);
|
engine_data[idx] = malloc(sizeof(char *) * 4);
|
||||||
engine_data[idx][0] = (char *)ENGINE_REGISTRY[i].id;
|
engine_data[idx][0] = (char *)ENGINE_REGISTRY[i].id;
|
||||||
engine_data[idx][1] = (char *)ENGINE_REGISTRY[i].name;
|
engine_data[idx][1] = (char *)ENGINE_REGISTRY[i].name;
|
||||||
engine_data[idx][2] = is_selected ? "checked" : "";
|
engine_data[idx][2] = is_selected ? "checked" : "";
|
||||||
engine_inner[idx] = 3;
|
engine_data[idx][3] =
|
||||||
|
strcmp(default_engine, ENGINE_REGISTRY[i].id) == 0 ? "selected" : "";
|
||||||
|
engine_inner[idx] = 4;
|
||||||
|
|
||||||
idx++;
|
idx++;
|
||||||
}
|
}
|
||||||
|
|
@ -78,6 +88,7 @@ int settings_handler(UrlParams *params) {
|
||||||
context_set(&ctx, "query", query);
|
context_set(&ctx, "query", query);
|
||||||
context_set(&ctx, "theme", theme);
|
context_set(&ctx, "theme", theme);
|
||||||
context_set(&ctx, "locale", locale);
|
context_set(&ctx, "locale", locale);
|
||||||
|
context_set(&ctx, "default_engine", default_engine);
|
||||||
context_set_array_of_arrays(&ctx, "locales", locale_data, locale_count,
|
context_set_array_of_arrays(&ctx, "locales", locale_data, locale_count,
|
||||||
inner_counts);
|
inner_counts);
|
||||||
|
|
||||||
|
|
@ -132,6 +143,7 @@ int settings_handler(UrlParams *params) {
|
||||||
send_response(rendered_html);
|
send_response(rendered_html);
|
||||||
|
|
||||||
free(rendered_html);
|
free(rendered_html);
|
||||||
|
free(default_engine);
|
||||||
free(theme);
|
free(theme);
|
||||||
free(locale);
|
free(locale);
|
||||||
free_context(&ctx);
|
free_context(&ctx);
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,8 @@ int settings_save_handler(UrlParams *params) {
|
||||||
const char *theme = "";
|
const char *theme = "";
|
||||||
const char *locale = "";
|
const char *locale = "";
|
||||||
const char *query = "";
|
const char *query = "";
|
||||||
|
const char *default_engine = "all";
|
||||||
|
int default_engine_present = 0;
|
||||||
int engines_present = 0;
|
int engines_present = 0;
|
||||||
char selected_ids[ENGINE_COUNT][32];
|
char selected_ids[ENGINE_COUNT][32];
|
||||||
int selected_count = 0;
|
int selected_count = 0;
|
||||||
|
|
@ -22,6 +24,9 @@ int settings_save_handler(UrlParams *params) {
|
||||||
locale = params->params[i].value;
|
locale = params->params[i].value;
|
||||||
} else if (strcmp(params->params[i].key, "q") == 0) {
|
} else if (strcmp(params->params[i].key, "q") == 0) {
|
||||||
query = params->params[i].value;
|
query = params->params[i].value;
|
||||||
|
} else if (strcmp(params->params[i].key, "default_engine") == 0) {
|
||||||
|
default_engine = params->params[i].value;
|
||||||
|
default_engine_present = 1;
|
||||||
} else if (strcmp(params->params[i].key, "engines_present") == 0) {
|
} else if (strcmp(params->params[i].key, "engines_present") == 0) {
|
||||||
engines_present = 1;
|
engines_present = 1;
|
||||||
} else if (strncmp(params->params[i].key, "engine_", 7) == 0 &&
|
} else if (strncmp(params->params[i].key, "engine_", 7) == 0 &&
|
||||||
|
|
@ -58,6 +63,31 @@ int settings_save_handler(UrlParams *params) {
|
||||||
false, false);
|
false, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (default_engine_present) {
|
||||||
|
const char *validated_default = "all";
|
||||||
|
|
||||||
|
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||||
|
if (!ENGINE_REGISTRY[i].enabled ||
|
||||||
|
strcmp(default_engine, ENGINE_REGISTRY[i].id) != 0)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
int selected = !engines_present;
|
||||||
|
for (int j = 0; j < selected_count; j++) {
|
||||||
|
if (strcmp(selected_ids[j], ENGINE_REGISTRY[i].id) == 0) {
|
||||||
|
selected = 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (selected)
|
||||||
|
validated_default = ENGINE_REGISTRY[i].id;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
set_cookie("default_engine", validated_default,
|
||||||
|
"Fri, 31 Dec 2038 23:59:59 GMT", "/", false, false);
|
||||||
|
}
|
||||||
|
|
||||||
char redirect_url[512];
|
char redirect_url[512];
|
||||||
snprintf(redirect_url, sizeof(redirect_url), "/settings?q=%s", query);
|
snprintf(redirect_url, sizeof(redirect_url), "/settings?q=%s", query);
|
||||||
send_redirect(redirect_url);
|
send_redirect(redirect_url);
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,11 @@
|
||||||
#include "Scraping.h"
|
#include "Scraping.h"
|
||||||
#include "../Cache/Cache.h"
|
#include "../Cache/Cache.h"
|
||||||
#include "../Proxy/Proxy.h"
|
#include "../Proxy/Proxy.h"
|
||||||
|
#include "../Utility/XmlHelper.h"
|
||||||
#include "Config.h"
|
#include "Config.h"
|
||||||
#include <curl/curl.h>
|
#include <curl/curl.h>
|
||||||
#include <libxml/HTMLparser.h>
|
#include <libxml/HTMLparser.h>
|
||||||
|
#include <libxml/parser.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
@ -24,7 +26,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job,
|
||||||
|
|
||||||
return response_contains(response, "<title>Startpage Captcha</title>") ||
|
return response_contains(response, "<title>Startpage Captcha</title>") ||
|
||||||
response_contains(response, "Startpage Captcha") ||
|
response_contains(response, "Startpage Captcha") ||
|
||||||
response_contains(response, "/static-pages-assets/page-data/captcha/") ||
|
response_contains(response,
|
||||||
|
"/static-pages-assets/page-data/captcha/") ||
|
||||||
response_contains(response, ">Startpage Blocked</title>");
|
response_contains(response, ">Startpage Blocked</title>");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -39,7 +42,8 @@ static int response_looks_like_results_page(const ScrapeJob *job,
|
||||||
}
|
}
|
||||||
|
|
||||||
if (strcmp(job->engine->name, "Startpage") == 0) {
|
if (strcmp(job->engine->name, "Startpage") == 0) {
|
||||||
return response_contains(response, "<title>Startpage Search Results</title>") ||
|
return response_contains(response,
|
||||||
|
"<title>Startpage Search Results</title>") ||
|
||||||
response_contains(response, "class=\"w-gl") ||
|
response_contains(response, "class=\"w-gl") ||
|
||||||
response_contains(response, "data-testid=\"gl-title-link\"");
|
response_contains(response, "data-testid=\"gl-title-link\"");
|
||||||
}
|
}
|
||||||
|
|
@ -55,6 +59,11 @@ static int response_looks_like_results_page(const ScrapeJob *job,
|
||||||
response_contains(response, "Mojeek Search");
|
response_contains(response, "Mojeek Search");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (strcmp(job->engine->name, "YaCy") == 0) {
|
||||||
|
return response_contains(response, "<item") ||
|
||||||
|
response_contains(response, "<rss");
|
||||||
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -72,18 +81,24 @@ static void classify_job_response(ScrapeJob *job, const char *response,
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
xmlDocPtr doc = htmlReadMemory(response, response_size, NULL, NULL,
|
xmlDocPtr doc;
|
||||||
HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
|
if (job->engine->is_xml) {
|
||||||
HTML_PARSE_NOWARNING);
|
doc = xmlReadMemory(response, response_size, NULL, NULL,
|
||||||
|
XML_PARSE_RECOVER | XML_PARSE_NOERROR |
|
||||||
|
XML_PARSE_NOWARNING);
|
||||||
|
} else {
|
||||||
|
doc = htmlReadMemory(response, response_size, NULL, NULL,
|
||||||
|
HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
|
||||||
|
HTML_PARSE_NOWARNING);
|
||||||
|
}
|
||||||
|
|
||||||
if (!doc) {
|
if (!doc) {
|
||||||
job->status = SCRAPE_STATUS_FETCH_ERROR;
|
job->status = SCRAPE_STATUS_FETCH_ERROR;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
job->results_count =
|
job->results_count = job->engine->parser(job->engine->name, doc,
|
||||||
job->engine->parser(job->engine->name, doc, job->out_results,
|
job->out_results, job->max_results);
|
||||||
job->max_results);
|
|
||||||
xmlFreeDoc(doc);
|
xmlFreeDoc(doc);
|
||||||
|
|
||||||
if (job->results_count > 0) {
|
if (job->results_count > 0) {
|
||||||
|
|
@ -156,14 +171,26 @@ void parse_and_cache_response(ScrapeJob *job) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void cleanup_job_handle(ScrapeJob *job, CURL *handle) {
|
static void cleanup_job_results(ScrapeJob *job) {
|
||||||
|
if (!job || !job->out_results)
|
||||||
|
return;
|
||||||
|
|
||||||
|
xml_result_free(*job->out_results, job->results_count);
|
||||||
|
*job->out_results = NULL;
|
||||||
|
job->results_count = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void cleanup_job_handle(ScrapeJob *job, CURL *handle) {
|
||||||
struct curl_slist *headers = NULL;
|
struct curl_slist *headers = NULL;
|
||||||
curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers);
|
if (handle)
|
||||||
|
curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers);
|
||||||
if (headers)
|
if (headers)
|
||||||
curl_slist_free_all(headers);
|
curl_slist_free_all(headers);
|
||||||
|
|
||||||
free(job->response.memory);
|
free(job->response.memory);
|
||||||
job->response.memory = NULL;
|
job->response.memory = NULL;
|
||||||
|
job->response.size = 0;
|
||||||
|
job->response.capacity = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) {
|
void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) {
|
||||||
|
|
@ -180,19 +207,25 @@ void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) {
|
||||||
}
|
}
|
||||||
|
|
||||||
int setup_job(ScrapeJob *job, CURLM *multi_handle) {
|
int setup_job(ScrapeJob *job, CURLM *multi_handle) {
|
||||||
if (job->handle)
|
if (job->handle) {
|
||||||
|
cleanup_job_handle(job, job->handle);
|
||||||
curl_easy_cleanup(job->handle);
|
curl_easy_cleanup(job->handle);
|
||||||
if (job->response.memory)
|
job->handle = NULL;
|
||||||
|
} else if (job->response.memory) {
|
||||||
free(job->response.memory);
|
free(job->response.memory);
|
||||||
|
job->response.memory = NULL;
|
||||||
|
job->response.size = 0;
|
||||||
|
job->response.capacity = 0;
|
||||||
|
}
|
||||||
|
|
||||||
job->results_count = 0;
|
cleanup_job_results(job);
|
||||||
job->http_status = 0;
|
job->http_status = 0;
|
||||||
job->status = SCRAPE_STATUS_PENDING;
|
job->status = SCRAPE_STATUS_PENDING;
|
||||||
|
|
||||||
if (check_cache_for_job(job)) {
|
if (check_cache_for_job(job))
|
||||||
job->results_count = job->results_count > 0 ? job->results_count : 0;
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
|
||||||
|
cleanup_job_results(job);
|
||||||
|
|
||||||
char *encoded_query = curl_easy_escape(NULL, job->query, 0);
|
char *encoded_query = curl_easy_escape(NULL, job->query, 0);
|
||||||
if (!encoded_query) {
|
if (!encoded_query) {
|
||||||
|
|
@ -200,12 +233,17 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) {
|
char *read = encoded_query;
|
||||||
if (p[0] == '%' && p[1] == '2' && p[2] == '0') {
|
char *write = encoded_query;
|
||||||
*p = '+';
|
while (*read) {
|
||||||
memmove(p + 1, p + 3, strlen(p + 3) + 1);
|
if (read[0] == '%' && read[1] == '2' && read[2] == '0') {
|
||||||
|
*write++ = '+';
|
||||||
|
read += 3;
|
||||||
|
} else {
|
||||||
|
*write++ = *read++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
*write = '\0';
|
||||||
|
|
||||||
char *full_url =
|
char *full_url =
|
||||||
build_search_url(job->engine->base_url, job->engine->page_param,
|
build_search_url(job->engine->base_url, job->engine->page_param,
|
||||||
|
|
@ -228,6 +266,14 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
|
||||||
job->response.memory = (char *)malloc(INITIAL_BUFFER_SIZE);
|
job->response.memory = (char *)malloc(INITIAL_BUFFER_SIZE);
|
||||||
job->response.size = 0;
|
job->response.size = 0;
|
||||||
job->response.capacity = INITIAL_BUFFER_SIZE;
|
job->response.capacity = INITIAL_BUFFER_SIZE;
|
||||||
|
if (!job->response.memory) {
|
||||||
|
curl_easy_cleanup(job->handle);
|
||||||
|
job->handle = NULL;
|
||||||
|
job->response.capacity = 0;
|
||||||
|
free(full_url);
|
||||||
|
job->status = SCRAPE_STATUS_FETCH_ERROR;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
struct curl_slist *headers =
|
struct curl_slist *headers =
|
||||||
build_request_headers(job->engine->host_header, job->engine->referer);
|
build_request_headers(job->engine->host_header, job->engine->referer);
|
||||||
|
|
@ -236,10 +282,33 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
|
||||||
curl_easy_setopt(job->handle, CURLOPT_PRIVATE, headers);
|
curl_easy_setopt(job->handle, CURLOPT_PRIVATE, headers);
|
||||||
|
|
||||||
free(full_url);
|
free(full_url);
|
||||||
curl_multi_add_handle(multi_handle, job->handle);
|
CURLMcode add_result = curl_multi_add_handle(multi_handle, job->handle);
|
||||||
|
if (add_result != CURLM_OK) {
|
||||||
|
cleanup_job_handle(job, job->handle);
|
||||||
|
curl_easy_cleanup(job->handle);
|
||||||
|
job->handle = NULL;
|
||||||
|
job->status = SCRAPE_STATUS_FETCH_ERROR;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void cleanup_unfinished_jobs(CURLM *multi_handle, ScrapeJob *jobs,
|
||||||
|
int num_jobs) {
|
||||||
|
for (int i = 0; i < num_jobs; i++) {
|
||||||
|
if (!jobs[i].handle)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
curl_multi_remove_handle(multi_handle, jobs[i].handle);
|
||||||
|
cleanup_job_handle(&jobs[i], jobs[i].handle);
|
||||||
|
curl_easy_cleanup(jobs[i].handle);
|
||||||
|
jobs[i].handle = NULL;
|
||||||
|
|
||||||
|
if (jobs[i].status == SCRAPE_STATUS_PENDING)
|
||||||
|
jobs[i].status = SCRAPE_STATUS_FETCH_ERROR;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
int handle_responses(CURLM *multi_handle, ScrapeJob *jobs, int num_jobs) {
|
int handle_responses(CURLM *multi_handle, ScrapeJob *jobs, int num_jobs) {
|
||||||
CURLMsg *msg;
|
CURLMsg *msg;
|
||||||
int msgs_left;
|
int msgs_left;
|
||||||
|
|
@ -279,34 +348,30 @@ int should_retry(ScrapeJob *jobs, int num_jobs) {
|
||||||
int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) {
|
int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) {
|
||||||
int retries = 0;
|
int retries = 0;
|
||||||
|
|
||||||
retry:
|
retry:;
|
||||||
;
|
|
||||||
CURLM *multi_handle = curl_multi_init();
|
CURLM *multi_handle = curl_multi_init();
|
||||||
if (!multi_handle)
|
if (!multi_handle)
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
for (int i = 0; i < num_jobs; i++) {
|
for (int i = 0; i < num_jobs; i++) {
|
||||||
if (setup_job(&jobs[i], multi_handle) != 0 && jobs[i].handle) {
|
setup_job(&jobs[i], multi_handle);
|
||||||
curl_multi_remove_handle(multi_handle, jobs[i].handle);
|
|
||||||
curl_easy_cleanup(jobs[i].handle);
|
|
||||||
jobs[i].handle = NULL;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
http_delay();
|
http_delay();
|
||||||
|
|
||||||
int still_running = 0;
|
int still_running = 0;
|
||||||
curl_multi_perform(multi_handle, &still_running);
|
CURLMcode mc = curl_multi_perform(multi_handle, &still_running);
|
||||||
|
|
||||||
do {
|
while (mc == CURLM_OK && still_running) {
|
||||||
int numfds = 0;
|
int numfds = 0;
|
||||||
CURLMcode mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
|
mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
|
||||||
if (mc != CURLM_OK)
|
if (mc != CURLM_OK)
|
||||||
break;
|
break;
|
||||||
curl_multi_perform(multi_handle, &still_running);
|
mc = curl_multi_perform(multi_handle, &still_running);
|
||||||
} while (still_running);
|
}
|
||||||
|
|
||||||
handle_responses(multi_handle, jobs, num_jobs);
|
handle_responses(multi_handle, jobs, num_jobs);
|
||||||
|
cleanup_unfinished_jobs(multi_handle, jobs, num_jobs);
|
||||||
curl_multi_cleanup(multi_handle);
|
curl_multi_cleanup(multi_handle);
|
||||||
|
|
||||||
if (retries < max_proxy_retries && should_retry(jobs, num_jobs)) {
|
if (retries < max_proxy_retries && should_retry(jobs, num_jobs)) {
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,7 @@ typedef struct {
|
||||||
int page_base;
|
int page_base;
|
||||||
ParserFunc parser;
|
ParserFunc parser;
|
||||||
int enabled;
|
int enabled;
|
||||||
|
int is_xml;
|
||||||
} SearchEngine;
|
} SearchEngine;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
|
|
@ -59,6 +60,7 @@ typedef struct {
|
||||||
extern SearchEngine ENGINE_REGISTRY[];
|
extern SearchEngine ENGINE_REGISTRY[];
|
||||||
extern const int ENGINE_COUNT;
|
extern const int ENGINE_COUNT;
|
||||||
void apply_engines_config(const char *engines_str);
|
void apply_engines_config(const char *engines_str);
|
||||||
|
void configure_yacy_engine(const char *instance);
|
||||||
|
|
||||||
size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
|
size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
|
||||||
void *userp);
|
void *userp);
|
||||||
|
|
|
||||||
|
|
@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header,
|
||||||
struct curl_slist *headers = NULL;
|
struct curl_slist *headers = NULL;
|
||||||
char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM];
|
char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM];
|
||||||
|
|
||||||
snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
|
if (host_header && host_header[0] != '\0') {
|
||||||
snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
|
snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
|
||||||
|
headers = curl_slist_append(headers, host_buf);
|
||||||
headers = curl_slist_append(headers, host_buf);
|
}
|
||||||
headers = curl_slist_append(headers, ref_buf);
|
if (referer && referer[0] != '\0') {
|
||||||
|
snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
|
||||||
|
headers = curl_slist_append(headers, ref_buf);
|
||||||
|
}
|
||||||
headers = curl_slist_append(
|
headers = curl_slist_append(
|
||||||
headers,
|
headers,
|
||||||
"Accept: "
|
"Accept: "
|
||||||
|
|
|
||||||
|
|
@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
|
||||||
if (!ctx)
|
if (!ctx)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
xmlXPathObjectPtr obj =
|
xmlXPathObjectPtr obj = xml_xpath_eval(
|
||||||
xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
|
ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
|
||||||
|
|
||||||
if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
|
if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
|
||||||
free_xpath_objects(ctx, obj);
|
free_xpath_objects(ctx, obj);
|
||||||
|
|
@ -268,18 +268,17 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
|
||||||
xmlNodePtr result_node = obj->nodesetval->nodeTab[i];
|
xmlNodePtr result_node = obj->nodesetval->nodeTab[i];
|
||||||
ctx->node = result_node;
|
ctx->node = result_node;
|
||||||
|
|
||||||
xmlXPathObjectPtr link_obj =
|
xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']");
|
||||||
xml_xpath_eval(ctx, ".//a[@class='title']");
|
|
||||||
char *url =
|
char *url =
|
||||||
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
|
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
|
||||||
? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0],
|
? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0],
|
||||||
(xmlChar *)"href")
|
(xmlChar *)"href")
|
||||||
: NULL;
|
: NULL;
|
||||||
|
|
||||||
char *title = (link_obj && link_obj->nodesetval &&
|
char *title =
|
||||||
link_obj->nodesetval->nodeNr > 0)
|
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
|
||||||
? xml_node_content(link_obj->nodesetval->nodeTab[0])
|
? xml_node_content(link_obj->nodesetval->nodeTab[0])
|
||||||
: NULL;
|
: NULL;
|
||||||
|
|
||||||
xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']");
|
xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']");
|
||||||
char *snippet_text =
|
char *snippet_text =
|
||||||
|
|
@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc,
|
||||||
static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
|
static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
|
||||||
SearchResult **out_results, int max_results);
|
SearchResult **out_results, int max_results);
|
||||||
|
|
||||||
|
static int parse_yacy(const char *engine_name, xmlDocPtr doc,
|
||||||
|
SearchResult **out_results, int max_results) {
|
||||||
|
(void)engine_name;
|
||||||
|
int found_count = 0;
|
||||||
|
|
||||||
|
xmlXPathContextPtr ctx = create_xpath_context(doc);
|
||||||
|
if (!ctx)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item");
|
||||||
|
|
||||||
|
if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
|
||||||
|
free_xpath_objects(ctx, obj);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int num_items = obj->nodesetval->nodeNr;
|
||||||
|
*out_results = alloc_results_array(num_items, max_results);
|
||||||
|
if (!*out_results) {
|
||||||
|
free_xpath_objects(ctx, obj);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < num_items && found_count < max_results; i++) {
|
||||||
|
xmlNodePtr item_node = obj->nodesetval->nodeTab[i];
|
||||||
|
ctx->node = item_node;
|
||||||
|
|
||||||
|
xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title");
|
||||||
|
char *title = (title_obj && title_obj->nodesetval &&
|
||||||
|
title_obj->nodesetval->nodeNr > 0)
|
||||||
|
? xml_node_content(title_obj->nodesetval->nodeTab[0])
|
||||||
|
: NULL;
|
||||||
|
|
||||||
|
xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link");
|
||||||
|
char *url =
|
||||||
|
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
|
||||||
|
? xml_node_content(link_obj->nodesetval->nodeTab[0])
|
||||||
|
: NULL;
|
||||||
|
|
||||||
|
xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description");
|
||||||
|
char *snippet_text =
|
||||||
|
(desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0)
|
||||||
|
? xml_node_content(desc_obj->nodesetval->nodeTab[0])
|
||||||
|
: NULL;
|
||||||
|
|
||||||
|
if (url && title) {
|
||||||
|
assign_result(&(*out_results)[found_count], url, title, snippet_text, 0);
|
||||||
|
found_count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
free_xml_node_list(title, url, snippet_text);
|
||||||
|
if (title_obj)
|
||||||
|
xmlXPathFreeObject(title_obj);
|
||||||
|
if (link_obj)
|
||||||
|
xmlXPathFreeObject(link_obj);
|
||||||
|
if (desc_obj)
|
||||||
|
xmlXPathFreeObject(desc_obj);
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx->node = NULL;
|
||||||
|
free_xpath_objects(ctx, obj);
|
||||||
|
return found_count;
|
||||||
|
}
|
||||||
|
|
||||||
|
static char yacy_base_url[BUFFER_SIZE_LARGE];
|
||||||
|
|
||||||
|
void configure_yacy_engine(const char *instance) {
|
||||||
|
if (!instance || instance[0] == '\0')
|
||||||
|
instance = DEFAULT_YACY_INSTANCE;
|
||||||
|
|
||||||
|
snprintf(yacy_base_url, sizeof(yacy_base_url),
|
||||||
|
"%s/"
|
||||||
|
"yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&"
|
||||||
|
"nav=all&maximumRecords=%d&query=",
|
||||||
|
instance, MAX_RESULTS_PER_ENGINE);
|
||||||
|
|
||||||
|
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||||
|
if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
|
||||||
|
ENGINE_REGISTRY[i].base_url = yacy_base_url;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
SearchEngine ENGINE_REGISTRY[] = {
|
SearchEngine ENGINE_REGISTRY[] = {
|
||||||
{.id = "ddg",
|
{.id = "ddg",
|
||||||
.name = "DuckDuckGo Lite",
|
.name = "DuckDuckGo Lite",
|
||||||
|
|
@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = {
|
||||||
.page_multiplier = 10,
|
.page_multiplier = 10,
|
||||||
.page_base = 1,
|
.page_base = 1,
|
||||||
.parser = parse_mojeek,
|
.parser = parse_mojeek,
|
||||||
.enabled = 1}};
|
.enabled = 1},
|
||||||
|
{.id = "yacy",
|
||||||
|
.name = "YaCy",
|
||||||
|
.base_url = "",
|
||||||
|
.host_header = NULL,
|
||||||
|
.referer = NULL,
|
||||||
|
.page_param = "startRecord",
|
||||||
|
.page_multiplier = 10,
|
||||||
|
.page_base = 0,
|
||||||
|
.parser = parse_yacy,
|
||||||
|
.enabled = 0,
|
||||||
|
.is_xml = 1}};
|
||||||
|
|
||||||
const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine);
|
const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine);
|
||||||
|
|
||||||
|
|
@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) {
|
||||||
void apply_engines_config(const char *engines_str) {
|
void apply_engines_config(const char *engines_str) {
|
||||||
if (!engines_str || engines_str[0] == '\0') {
|
if (!engines_str || engines_str[0] == '\0') {
|
||||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||||
|
if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
|
||||||
|
continue;
|
||||||
ENGINE_REGISTRY[i].enabled = 1;
|
ENGINE_REGISTRY[i].enabled = 1;
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
|
|
@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) {
|
||||||
|
|
||||||
if (strcmp(token, "*") == 0) {
|
if (strcmp(token, "*") == 0) {
|
||||||
for (int i = 0; i < ENGINE_COUNT; i++) {
|
for (int i = 0; i < ENGINE_COUNT; i++) {
|
||||||
|
if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
|
||||||
|
continue;
|
||||||
ENGINE_REGISTRY[i].enabled = 1;
|
ENGINE_REGISTRY[i].enabled = 1;
|
||||||
}
|
}
|
||||||
} else if (token[0] == '-' && token[1] != '\0') {
|
} else if (token[0] == '-' && token[1] != '\0') {
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ static int themes_initialized = 0;
|
||||||
void init_themes(const char *static_path) {
|
void init_themes(const char *static_path) {
|
||||||
if (themes_initialized)
|
if (themes_initialized)
|
||||||
return;
|
return;
|
||||||
themes_initialized = 1;
|
|
||||||
|
|
||||||
char themes_dir[512];
|
char themes_dir[512];
|
||||||
snprintf(themes_dir, sizeof(themes_dir), "%s/themes", static_path);
|
snprintf(themes_dir, sizeof(themes_dir), "%s/themes", static_path);
|
||||||
|
|
@ -26,20 +25,32 @@ void init_themes(const char *static_path) {
|
||||||
struct dirent *entry;
|
struct dirent *entry;
|
||||||
int capacity = 4;
|
int capacity = 4;
|
||||||
themes_list = malloc(sizeof(char *) * capacity);
|
themes_list = malloc(sizeof(char *) * capacity);
|
||||||
|
if (!themes_list) {
|
||||||
|
closedir(dir);
|
||||||
|
return;
|
||||||
|
}
|
||||||
themes_count = 0;
|
themes_count = 0;
|
||||||
|
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
size_t len = strlen(entry->d_name);
|
size_t len = strlen(entry->d_name);
|
||||||
if (len > 4 && strcmp(entry->d_name + len - 4, ".css") == 0) {
|
if (len > 4 && strcmp(entry->d_name + len - 4, ".css") == 0) {
|
||||||
if (themes_count >= capacity) {
|
if (themes_count >= capacity) {
|
||||||
capacity *= 2;
|
int new_capacity = capacity * 2;
|
||||||
themes_list = realloc(themes_list, sizeof(char *) * capacity);
|
char **new_list = realloc(themes_list, sizeof(char *) * new_capacity);
|
||||||
|
if (!new_list)
|
||||||
|
break;
|
||||||
|
themes_list = new_list;
|
||||||
|
capacity = new_capacity;
|
||||||
}
|
}
|
||||||
themes_list[themes_count] = strndup(entry->d_name, len - 4);
|
char *theme = strndup(entry->d_name, len - 4);
|
||||||
|
if (!theme)
|
||||||
|
break;
|
||||||
|
themes_list[themes_count] = theme;
|
||||||
themes_count++;
|
themes_count++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
closedir(dir);
|
closedir(dir);
|
||||||
|
themes_initialized = 1;
|
||||||
|
|
||||||
for (int i = 0; i < themes_count; i++) {
|
for (int i = 0; i < themes_count; i++) {
|
||||||
for (int j = i + 1; j < themes_count; j++) {
|
for (int j = i + 1; j < themes_count; j++) {
|
||||||
|
|
@ -206,98 +217,130 @@ int user_engines_contains(const char *engine_id, char **ids, int count) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
int add_link_to_collection(const char *href, const char *label,
|
char *get_default_search_engine(void) {
|
||||||
const char *class_name, char ****collection,
|
char *cookie = get_cookie("default_engine");
|
||||||
int **inner_counts, int current_count) {
|
if (cookie && (strcmp(cookie, "all") == 0 || is_engine_id_enabled(cookie)))
|
||||||
char ***old_collection = *collection;
|
return cookie;
|
||||||
int *old_inner_counts = *inner_counts;
|
|
||||||
char ***new_collection =
|
|
||||||
(char ***)malloc(sizeof(char **) * (current_count + 1));
|
|
||||||
int *new_inner_counts = (int *)malloc(sizeof(int) * (current_count + 1));
|
|
||||||
|
|
||||||
if (!new_collection || !new_inner_counts) {
|
free(cookie);
|
||||||
free(new_collection);
|
return strdup("all");
|
||||||
free(new_inner_counts);
|
|
||||||
return current_count;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (*collection && current_count > 0) {
|
|
||||||
memcpy(new_collection, *collection, sizeof(char **) * current_count);
|
|
||||||
}
|
|
||||||
if (*inner_counts && current_count > 0) {
|
|
||||||
memcpy(new_inner_counts, *inner_counts, sizeof(int) * current_count);
|
|
||||||
}
|
|
||||||
|
|
||||||
*collection = new_collection;
|
|
||||||
*inner_counts = new_inner_counts;
|
|
||||||
|
|
||||||
(*collection)[current_count] =
|
|
||||||
(char **)malloc(sizeof(char *) * LINK_FIELD_COUNT);
|
|
||||||
if (!(*collection)[current_count]) {
|
|
||||||
*collection = old_collection;
|
|
||||||
*inner_counts = old_inner_counts;
|
|
||||||
free(new_collection);
|
|
||||||
free(new_inner_counts);
|
|
||||||
return current_count;
|
|
||||||
}
|
|
||||||
|
|
||||||
(*collection)[current_count][0] = strdup(href ? href : "");
|
|
||||||
(*collection)[current_count][1] = strdup(label ? label : "");
|
|
||||||
(*collection)[current_count][2] = strdup(class_name ? class_name : "");
|
|
||||||
|
|
||||||
if (!(*collection)[current_count][0] || !(*collection)[current_count][1] ||
|
|
||||||
!(*collection)[current_count][2]) {
|
|
||||||
free((*collection)[current_count][0]);
|
|
||||||
free((*collection)[current_count][1]);
|
|
||||||
free((*collection)[current_count][2]);
|
|
||||||
free((*collection)[current_count]);
|
|
||||||
*collection = old_collection;
|
|
||||||
*inner_counts = old_inner_counts;
|
|
||||||
free(new_collection);
|
|
||||||
free(new_inner_counts);
|
|
||||||
return current_count;
|
|
||||||
}
|
|
||||||
|
|
||||||
(*inner_counts)[current_count] = LINK_FIELD_COUNT;
|
|
||||||
|
|
||||||
free(old_collection);
|
|
||||||
free(old_inner_counts);
|
|
||||||
return current_count + 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int build_pagination(int page, char *(*href_builder)(int page, void *data),
|
void free_string_matrix(char ***matrix, int *inner_counts, int row_count) {
|
||||||
void *data, char ****out_matrix, int **out_inner_counts) {
|
if (matrix) {
|
||||||
|
for (int i = 0; i < row_count; i++) {
|
||||||
|
int field_count = inner_counts ? inner_counts[i] : 0;
|
||||||
|
for (int j = 0; j < field_count; j++)
|
||||||
|
free(matrix[i][j]);
|
||||||
|
free(matrix[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(matrix);
|
||||||
|
free(inner_counts);
|
||||||
|
}
|
||||||
|
|
||||||
|
void string_matrix_init(StringMatrix *matrix) {
|
||||||
|
if (matrix)
|
||||||
|
*matrix = (StringMatrix){0};
|
||||||
|
}
|
||||||
|
|
||||||
|
int string_matrix_append(StringMatrix *matrix, const char *const *values,
|
||||||
|
int field_count) {
|
||||||
|
if (!matrix || !values || field_count <= 0)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
char **row = calloc((size_t)field_count, sizeof(*row));
|
||||||
|
if (!row)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
for (int i = 0; i < field_count; i++) {
|
||||||
|
row[i] = strdup(values[i] ? values[i] : "");
|
||||||
|
if (!row[i]) {
|
||||||
|
for (int j = 0; j < i; j++)
|
||||||
|
free(row[j]);
|
||||||
|
free(row);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (matrix->count == matrix->capacity) {
|
||||||
|
int new_capacity = matrix->capacity ? matrix->capacity * 2 : 8;
|
||||||
|
char ***new_rows = malloc(sizeof(*new_rows) * (size_t)new_capacity);
|
||||||
|
int *new_counts = malloc(sizeof(*new_counts) * (size_t)new_capacity);
|
||||||
|
if (!new_rows || !new_counts) {
|
||||||
|
free(new_rows);
|
||||||
|
free(new_counts);
|
||||||
|
for (int i = 0; i < field_count; i++)
|
||||||
|
free(row[i]);
|
||||||
|
free(row);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (matrix->count > 0) {
|
||||||
|
memcpy(new_rows, matrix->rows, sizeof(*new_rows) * matrix->count);
|
||||||
|
memcpy(new_counts, matrix->field_counts,
|
||||||
|
sizeof(*new_counts) * matrix->count);
|
||||||
|
}
|
||||||
|
free(matrix->rows);
|
||||||
|
free(matrix->field_counts);
|
||||||
|
matrix->rows = new_rows;
|
||||||
|
matrix->field_counts = new_counts;
|
||||||
|
matrix->capacity = new_capacity;
|
||||||
|
}
|
||||||
|
|
||||||
|
matrix->rows[matrix->count] = row;
|
||||||
|
matrix->field_counts[matrix->count] = field_count;
|
||||||
|
matrix->count++;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void string_matrix_free(StringMatrix *matrix) {
|
||||||
|
if (!matrix)
|
||||||
|
return;
|
||||||
|
free_string_matrix(matrix->rows, matrix->field_counts, matrix->count);
|
||||||
|
*matrix = (StringMatrix){0};
|
||||||
|
}
|
||||||
|
|
||||||
|
int string_matrix_build_pagination(int page,
|
||||||
|
char *(*href_builder)(int page, void *data),
|
||||||
|
void *data, StringMatrix *out_matrix) {
|
||||||
enum { PAGER_WINDOW_SIZE = 5 };
|
enum { PAGER_WINDOW_SIZE = 5 };
|
||||||
|
if (!out_matrix || !href_builder)
|
||||||
|
return -1;
|
||||||
|
|
||||||
*out_matrix = NULL;
|
string_matrix_init(out_matrix);
|
||||||
*out_inner_counts = NULL;
|
|
||||||
int count = 0;
|
|
||||||
|
|
||||||
int pager_start = page <= 3 ? 1 : page - 2;
|
int pager_start = page <= 3 ? 1 : page - 2;
|
||||||
int pager_end = pager_start + PAGER_WINDOW_SIZE - 1;
|
int pager_end = pager_start + PAGER_WINDOW_SIZE - 1;
|
||||||
|
|
||||||
if (page > 1) {
|
for (int i = page > 1 ? pager_start - 1 : pager_start; i <= pager_end + 1;
|
||||||
char *href = href_builder(page - 1, data);
|
i++) {
|
||||||
count = add_link_to_collection(href, "←", "pagination-btn prev", out_matrix,
|
const char *label = NULL;
|
||||||
out_inner_counts, count);
|
const char *class_name = "pagination-btn";
|
||||||
|
char page_label[16];
|
||||||
|
int target_page = i;
|
||||||
|
|
||||||
|
if (page > 1 && i == pager_start - 1) {
|
||||||
|
label = "←";
|
||||||
|
class_name = "pagination-btn prev";
|
||||||
|
target_page = page - 1;
|
||||||
|
} else if (i == pager_end + 1) {
|
||||||
|
label = "→";
|
||||||
|
class_name = "pagination-btn next";
|
||||||
|
target_page = page + 1;
|
||||||
|
} else {
|
||||||
|
snprintf(page_label, sizeof(page_label), "%d", i);
|
||||||
|
label = page_label;
|
||||||
|
if (i == page)
|
||||||
|
class_name = "pagination-btn pagination-current";
|
||||||
|
}
|
||||||
|
|
||||||
|
char *href = href_builder(target_page, data);
|
||||||
|
const char *values[LINK_FIELD_COUNT] = {href, label, class_name};
|
||||||
|
int result = string_matrix_append(out_matrix, values, LINK_FIELD_COUNT);
|
||||||
free(href);
|
free(href);
|
||||||
|
if (result != 0) {
|
||||||
|
string_matrix_free(out_matrix);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
return 0;
|
||||||
for (int i = pager_start; i <= pager_end; i++) {
|
|
||||||
char label[16];
|
|
||||||
snprintf(label, sizeof(label), "%d", i);
|
|
||||||
char *href = href_builder(i, data);
|
|
||||||
count = add_link_to_collection(
|
|
||||||
href, label,
|
|
||||||
i == page ? "pagination-btn pagination-current" : "pagination-btn",
|
|
||||||
out_matrix, out_inner_counts, count);
|
|
||||||
free(href);
|
|
||||||
}
|
|
||||||
|
|
||||||
char *href = href_builder(page + 1, data);
|
|
||||||
count = add_link_to_collection(href, "→", "pagination-btn next", out_matrix,
|
|
||||||
out_inner_counts, count);
|
|
||||||
free(href);
|
|
||||||
|
|
||||||
return count;
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,13 @@
|
||||||
|
|
||||||
#define LINK_FIELD_COUNT 3
|
#define LINK_FIELD_COUNT 3
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char ***rows;
|
||||||
|
int *field_counts;
|
||||||
|
int count;
|
||||||
|
int capacity;
|
||||||
|
} StringMatrix;
|
||||||
|
|
||||||
int hex_to_int(char c);
|
int hex_to_int(char c);
|
||||||
char *get_theme(const char *default_theme);
|
char *get_theme(const char *default_theme);
|
||||||
void init_themes(const char *static_path);
|
void init_themes(const char *static_path);
|
||||||
|
|
@ -23,12 +30,17 @@ char *get_locale(const char *default_locale);
|
||||||
int is_engine_id_enabled(const char *engine_id);
|
int is_engine_id_enabled(const char *engine_id);
|
||||||
int get_user_engines(char ***out_ids, int *out_count);
|
int get_user_engines(char ***out_ids, int *out_count);
|
||||||
int user_engines_contains(const char *engine_id, char **ids, int count);
|
int user_engines_contains(const char *engine_id, char **ids, int count);
|
||||||
|
char *get_default_search_engine(void);
|
||||||
|
|
||||||
int add_link_to_collection(const char *href, const char *label,
|
void free_string_matrix(char ***matrix, int *inner_counts, int row_count);
|
||||||
const char *class_name, char ****collection,
|
|
||||||
int **inner_counts, int current_count);
|
|
||||||
|
|
||||||
int build_pagination(int page, char *(*href_builder)(int page, void *data),
|
void string_matrix_init(StringMatrix *matrix);
|
||||||
void *data, char ****out_matrix, int **out_inner_counts);
|
int string_matrix_append(StringMatrix *matrix, const char *const *values,
|
||||||
|
int field_count);
|
||||||
|
void string_matrix_free(StringMatrix *matrix);
|
||||||
|
|
||||||
|
int string_matrix_build_pagination(int page,
|
||||||
|
char *(*href_builder)(int page, void *data),
|
||||||
|
void *data, StringMatrix *out_matrix);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
||||||
|
|
@ -822,6 +822,38 @@ header .logo-link:hover {
|
||||||
height:18px;
|
height:18px;
|
||||||
accent-color:var(--accent);
|
accent-color:var(--accent);
|
||||||
cursor:pointer;
|
cursor:pointer;
|
||||||
|
flex-shrink:0;
|
||||||
|
margin:0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.settings-engine-list {
|
||||||
|
margin:0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.settings-engine-options {
|
||||||
|
display:grid;
|
||||||
|
gap:8px;
|
||||||
|
margin-inline:-3px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.settings-engine-option {
|
||||||
|
display:flex;
|
||||||
|
align-items:center;
|
||||||
|
justify-content:space-between;
|
||||||
|
gap:16px;
|
||||||
|
padding:12px 14px;
|
||||||
|
border:1px solid var(--border);
|
||||||
|
border-radius:8px;
|
||||||
|
background:var(--bg-main);
|
||||||
|
color:var(--text-primary);
|
||||||
|
font-size:0.95rem;
|
||||||
|
cursor:pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.settings-default-engine-field {
|
||||||
|
margin-top:12px;
|
||||||
|
padding-top:12px;
|
||||||
|
border-top:1px solid var(--border);
|
||||||
}
|
}
|
||||||
|
|
||||||
.settings-actions {
|
.settings-actions {
|
||||||
|
|
@ -888,6 +920,12 @@ header .logo-link:hover {
|
||||||
align-items:stretch;
|
align-items:stretch;
|
||||||
gap:8px;
|
gap:8px;
|
||||||
}
|
}
|
||||||
|
.settings-engine-options {
|
||||||
|
margin-inline:0;
|
||||||
|
}
|
||||||
|
.settings-engine-option {
|
||||||
|
padding-inline:7px;
|
||||||
|
}
|
||||||
.settings-actions {
|
.settings-actions {
|
||||||
flex-direction:column;
|
flex-direction:column;
|
||||||
}
|
}
|
||||||
|
|
@ -936,5 +974,3 @@ header .logo-link:hover {
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,5 +7,5 @@
|
||||||
<Url type="text/html" method="get" template="{{scheme}}://{{domain}}/search?q={searchTerms}"/>
|
<Url type="text/html" method="get" template="{{scheme}}://{{domain}}/search?q={searchTerms}"/>
|
||||||
<InputEncoding>UTF-8</InputEncoding>
|
<InputEncoding>UTF-8</InputEncoding>
|
||||||
<OutputEncoding>UTF-8</OutputEncoding>
|
<OutputEncoding>UTF-8</OutputEncoding>
|
||||||
<moz:SearchForm>{{domain}}/</moz:SearchForm>
|
<moz:SearchForm>{{scheme}}://{{domain}}/</moz:SearchForm>
|
||||||
</OpenSearchDescription>
|
</OpenSearchDescription>
|
||||||
|
|
|
||||||
|
|
@ -76,16 +76,33 @@
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
{{if has_enabled_engines}}
|
{{if has_enabled_engines}}
|
||||||
<section class="settings-section">
|
<section class="settings-section settings-engines-section">
|
||||||
<h3 class="settings-section-title">{{l("engines_label")}}</h3>
|
<h3 class="settings-section-title">{{l("engines_label")}}</h3>
|
||||||
<p class="settings-section-desc">{{l("engines_desc")}}</p>
|
<p class="settings-section-desc">{{l("engines_desc")}}</p>
|
||||||
<input type="hidden" name="engines_present" value="1">
|
<input type="hidden" name="engines_present" value="1">
|
||||||
{{for eng in enabled_engines}}
|
|
||||||
<div class="settings-field">
|
<div class="settings-engine-list">
|
||||||
<label class="settings-label" for="engine_{{eng[0]}}">{{eng[1]}}</label>
|
<div class="settings-engine-options">
|
||||||
<input type="checkbox" id="engine_{{eng[0]}}" name="engine_{{eng[0]}}" value="1" class="settings-checkbox" {{eng[2]}}>
|
{{for eng in enabled_engines}}
|
||||||
|
<label class="settings-engine-option" for="engine_{{eng[0]}}">
|
||||||
|
<span>{{eng[1]}}</span>
|
||||||
|
<input type="checkbox" id="engine_{{eng[0]}}" name="engine_{{eng[0]}}" value="1" class="settings-checkbox" {{eng[2]}}>
|
||||||
|
</label>
|
||||||
|
{{endfor}}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="settings-field settings-default-engine-field">
|
||||||
|
<label class="settings-label" for="default_engine">{{l("default_engine_label")}}</label>
|
||||||
|
<select id="default_engine" name="default_engine" class="settings-select">
|
||||||
|
<option value="all" {{if default_engine == "all"}}selected{{endif}}>{{l("all_engines_option")}}</option>
|
||||||
|
{{for eng in enabled_engines}}
|
||||||
|
{{if eng[2] != ""}}
|
||||||
|
<option value="{{eng[0]}}" {{eng[3]}}>{{eng[1]}}</option>
|
||||||
|
{{endif}}
|
||||||
|
{{endfor}}
|
||||||
|
</select>
|
||||||
</div>
|
</div>
|
||||||
{{endfor}}
|
|
||||||
</section>
|
</section>
|
||||||
{{endif}}
|
{{endif}}
|
||||||
<div class="settings-actions">
|
<div class="settings-actions">
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue