electria/omnisearch
Watch
1
0
Fork
You've already forked omnisearch
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: add yacy support

Browse source
This commit is contained in:
frosty 2026-08-03 23:21:41 -04:00
commit 7b587d82c8
8 changed files with 170 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

5
example-config.ini
View file

@ -31,8 +31,11 @@ port = 8087
[engines]
# Use * for all engines, or specify comma-separated list (e.g., ddg,yahoo)
# Use *,-engine to exclude specific engines (e.g., *,-startpage)
# Available engines: ddg, startpage, yahoo, mojeek
# Available engines: ddg, startpage, yahoo, mojeek, yacy
engines="*"
# YaCy instance to use for the yacy engine (optional, default: http://127.0.0.1:8090)
# Requires "yacy" to be in the engines list above.
#yacy_instance = http://127.0.0.1:8090
[rate_limit]
# Rate limit searches per interval

4
src/Config.c
View file

@ -100,6 +100,10 @@ int load_config(const char *filename, Config *config) {
if (strcmp(key, "engines") == 0) {
strncpy(config->engines, value, sizeof(config->engines) - 1);
config->engines[sizeof(config->engines) - 1] = '\0';
} else if (strcmp(key, "yacy_instance") == 0) {
strncpy(config->yacy_instance, value,
sizeof(config->yacy_instance) - 1);
config->yacy_instance[sizeof(config->yacy_instance) - 1] = '\0';
}
} else if (strcmp(section, "rate_limit") == 0) {
if (strcmp(key, "search_requests") == 0) {

2
src/Config.h
View file

@ -8,6 +8,7 @@
#define DEFAULT_CACHE_TTL_INFOBOX 86400
#define DEFAULT_CACHE_TTL_IMAGE 604800
#define DEFAULT_MAX_PROXY_RETRIES 3
#define DEFAULT_YACY_INSTANCE "http://127.0.0.1:8090"
#define BUFFER_SIZE_SMALL 256
#define BUFFER_SIZE_MEDIUM 512
@ -46,6 +47,7 @@ typedef struct {
int cache_ttl_infobox;
int cache_ttl_image;
char engines[512];
char yacy_instance[512];
int rate_limit_search_requests;
int rate_limit_search_interval;
int rate_limit_images_requests;

18
src/Main.c
View file

@ -78,6 +78,7 @@ int main() {
.cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX,
.cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE,
.engines = "",
.yacy_instance = "",
.rate_limit_search_requests = 0,
.rate_limit_search_interval = 0,
.rate_limit_images_requests = 0,
@ -101,6 +102,23 @@ int main() {
}
apply_engines_config(cfg.engines);
configure_yacy_engine(cfg.yacy_instance);
if (cfg.yacy_instance[0] != '\0') {
int yacy_enabled = 0;
for (int i = 0; i < ENGINE_COUNT; i++) {
if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
yacy_enabled = ENGINE_REGISTRY[i].enabled;
break;
}
}
if (!yacy_enabled) {
fprintf(
stderr,
"[INFO] YaCy instance configured but the yacy engine is not "
"enabled (add it to the engines list, e.g. engines=\"*,yacy\")\n");
}
}
if (cache_init(cfg.cache_dir) != 0) {
fprintf(stderr,

36
src/Scraping/Scraping.c
View file

@ -4,6 +4,7 @@
#include "Config.h"
#include <curl/curl.h>
#include <libxml/HTMLparser.h>
#include <libxml/parser.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -24,7 +25,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job,
return response_contains(response, "<title>Startpage Captcha</title>") ||
response_contains(response, "Startpage Captcha") ||
response_contains(response, "/static-pages-assets/page-data/captcha/") ||
response_contains(response,
"/static-pages-assets/page-data/captcha/") ||
response_contains(response, ">Startpage Blocked</title>");
}
@ -39,7 +41,8 @@ static int response_looks_like_results_page(const ScrapeJob *job,
}
if (strcmp(job->engine->name, "Startpage") == 0) {
return response_contains(response, "<title>Startpage Search Results</title>") ||
return response_contains(response,
"<title>Startpage Search Results</title>") ||
response_contains(response, "class=\"w-gl") ||
response_contains(response, "data-testid=\"gl-title-link\"");
}
@ -55,6 +58,11 @@ static int response_looks_like_results_page(const ScrapeJob *job,
response_contains(response, "Mojeek Search");
}
if (strcmp(job->engine->name, "YaCy") == 0) {
return response_contains(response, "<item") ||
response_contains(response, "<rss");
}
return 0;
}
@ -72,18 +80,24 @@ static void classify_job_response(ScrapeJob *job, const char *response,
return;
}
xmlDocPtr doc = htmlReadMemory(response, response_size, NULL, NULL,
HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
HTML_PARSE_NOWARNING);
xmlDocPtr doc;
if (job->engine->is_xml) {
doc = xmlReadMemory(response, response_size, NULL, NULL,
XML_PARSE_RECOVER | XML_PARSE_NOERROR |
XML_PARSE_NOWARNING);
} else {
doc = htmlReadMemory(response, response_size, NULL, NULL,
HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
HTML_PARSE_NOWARNING);
}
if (!doc) {
job->status = SCRAPE_STATUS_FETCH_ERROR;
return;
}
job->results_count =
job->engine->parser(job->engine->name, doc, job->out_results,
job->max_results);
job->results_count = job->engine->parser(job->engine->name, doc,
job->out_results, job->max_results);
xmlFreeDoc(doc);
if (job->results_count > 0) {
@ -200,7 +214,8 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
return -1;
}
for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) {
for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query;
p--) {
if (p[0] == '%' && p[1] == '2' && p[2] == '0') {
*p = '+';
memmove(p + 1, p + 3, strlen(p + 3) + 1);
@ -279,8 +294,7 @@ int should_retry(ScrapeJob *jobs, int num_jobs) {
int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) {
int retries = 0;
retry:
;
retry:;
CURLM *multi_handle = curl_multi_init();
if (!multi_handle)
return -1;

2
src/Scraping/Scraping.h
View file

@ -26,6 +26,7 @@ typedef struct {
int page_base;
ParserFunc parser;
int enabled;
int is_xml;
} SearchEngine;
typedef struct {
@ -59,6 +60,7 @@ typedef struct {
extern SearchEngine ENGINE_REGISTRY[];
extern const int ENGINE_COUNT;
void apply_engines_config(const char *engines_str);
void configure_yacy_engine(const char *instance);
size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
void *userp);

13
src/Scraping/ScrapingHttp.c
View file

@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header,
struct curl_slist *headers = NULL;
char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM];
snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
headers = curl_slist_append(headers, host_buf);
headers = curl_slist_append(headers, ref_buf);
if (host_header && host_header[0] != '\0') {
snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
headers = curl_slist_append(headers, host_buf);
}
if (referer && referer[0] != '\0') {
snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
headers = curl_slist_append(headers, ref_buf);
}
headers = curl_slist_append(
headers,
"Accept: "

116
src/Scraping/ScrapingParsers.c
View file

@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
if (!ctx)
return 0;
xmlXPathObjectPtr obj =
xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
xmlXPathObjectPtr obj = xml_xpath_eval(
ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
free_xpath_objects(ctx, obj);
@ -268,18 +268,17 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
xmlNodePtr result_node = obj->nodesetval->nodeTab[i];
ctx->node = result_node;
xmlXPathObjectPtr link_obj =
xml_xpath_eval(ctx, ".//a[@class='title']");
xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']");
char *url =
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0],
(xmlChar *)"href")
: NULL;
char *title = (link_obj && link_obj->nodesetval &&
link_obj->nodesetval->nodeNr > 0)
? xml_node_content(link_obj->nodesetval->nodeTab[0])
: NULL;
char *title =
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
? xml_node_content(link_obj->nodesetval->nodeTab[0])
: NULL;
xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']");
char *snippet_text =
@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc,
static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
SearchResult **out_results, int max_results);
static int parse_yacy(const char *engine_name, xmlDocPtr doc,
SearchResult **out_results, int max_results) {
(void)engine_name;
int found_count = 0;
xmlXPathContextPtr ctx = create_xpath_context(doc);
if (!ctx)
return 0;
xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item");
if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
free_xpath_objects(ctx, obj);
return 0;
}
int num_items = obj->nodesetval->nodeNr;
*out_results = alloc_results_array(num_items, max_results);
if (!*out_results) {
free_xpath_objects(ctx, obj);
return 0;
}
for (int i = 0; i < num_items && found_count < max_results; i++) {
xmlNodePtr item_node = obj->nodesetval->nodeTab[i];
ctx->node = item_node;
xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title");
char *title = (title_obj && title_obj->nodesetval &&
title_obj->nodesetval->nodeNr > 0)
? xml_node_content(title_obj->nodesetval->nodeTab[0])
: NULL;
xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link");
char *url =
(link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
? xml_node_content(link_obj->nodesetval->nodeTab[0])
: NULL;
xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description");
char *snippet_text =
(desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0)
? xml_node_content(desc_obj->nodesetval->nodeTab[0])
: NULL;
if (url && title) {
assign_result(&(*out_results)[found_count], url, title, snippet_text, 0);
found_count++;
}
free_xml_node_list(title, url, snippet_text);
if (title_obj)
xmlXPathFreeObject(title_obj);
if (link_obj)
xmlXPathFreeObject(link_obj);
if (desc_obj)
xmlXPathFreeObject(desc_obj);
}
ctx->node = NULL;
free_xpath_objects(ctx, obj);
return found_count;
}
static char yacy_base_url[BUFFER_SIZE_LARGE];
void configure_yacy_engine(const char *instance) {
if (!instance || instance[0] == '\0')
instance = DEFAULT_YACY_INSTANCE;
snprintf(yacy_base_url, sizeof(yacy_base_url),
"%s/"
"yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&"
"nav=all&maximumRecords=%d&query=",
instance, MAX_RESULTS_PER_ENGINE);
for (int i = 0; i < ENGINE_COUNT; i++) {
if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
ENGINE_REGISTRY[i].base_url = yacy_base_url;
break;
}
}
}
SearchEngine ENGINE_REGISTRY[] = {
{.id = "ddg",
.name = "DuckDuckGo Lite",
@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = {
.page_multiplier = 10,
.page_base = 1,
.parser = parse_mojeek,
.enabled = 1}};
.enabled = 1},
{.id = "yacy",
.name = "YaCy",
.base_url = "",
.host_header = NULL,
.referer = NULL,
.page_param = "startRecord",
.page_multiplier = 10,
.page_base = 0,
.parser = parse_yacy,
.enabled = 0,
.is_xml = 1}};
const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine);
@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) {
void apply_engines_config(const char *engines_str) {
if (!engines_str || engines_str[0] == '\0') {
for (int i = 0; i < ENGINE_COUNT; i++) {
if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
continue;
ENGINE_REGISTRY[i].enabled = 1;
}
return;
@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) {
if (strcmp(token, "*") == 0) {
for (int i = 0; i < ENGINE_COUNT; i++) {
if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
continue;
ENGINE_REGISTRY[i].enabled = 1;
}
} else if (token[0] == '-' && token[1] != '\0') {