feat: add yacy support

example-config.ini

@@ -31,8 +31,11 @@ port = 8087
 [engines]
 # Use * for all engines, or specify comma-separated list (e.g., ddg,yahoo)
 # Use *,-engine to exclude specific engines (e.g., *,-startpage)
-# Available engines: ddg, startpage, yahoo, mojeek
+# Available engines: ddg, startpage, yahoo, mojeek, yacy
 engines="*"
+# YaCy instance to use for the yacy engine (optional, default: http://127.0.0.1:8090)
+# Requires "yacy" to be in the engines list above.
+#yacy_instance = http://127.0.0.1:8090
 
 [rate_limit]
 # Rate limit searches per interval

src/Config.c

@@ -100,6 +100,10 @@ int load_config(const char *filename, Config *config) {
         if (strcmp(key, "engines") == 0) {
           strncpy(config->engines, value, sizeof(config->engines) - 1);
           config->engines[sizeof(config->engines) - 1] = '\0';
+        } else if (strcmp(key, "yacy_instance") == 0) {
+          strncpy(config->yacy_instance, value,
+                  sizeof(config->yacy_instance) - 1);
+          config->yacy_instance[sizeof(config->yacy_instance) - 1] = '\0';
         }
       } else if (strcmp(section, "rate_limit") == 0) {
         if (strcmp(key, "search_requests") == 0) {

src/Config.h

@@ -8,6 +8,7 @@
 #define DEFAULT_CACHE_TTL_INFOBOX 86400
 #define DEFAULT_CACHE_TTL_IMAGE 604800
 #define DEFAULT_MAX_PROXY_RETRIES 3
+#define DEFAULT_YACY_INSTANCE "http://127.0.0.1:8090"
 
 #define BUFFER_SIZE_SMALL 256
 #define BUFFER_SIZE_MEDIUM 512
@@ -46,6 +47,7 @@ typedef struct {
   int cache_ttl_infobox;
   int cache_ttl_image;
   char engines[512];
+  char yacy_instance[512];
   int rate_limit_search_requests;
   int rate_limit_search_interval;
   int rate_limit_images_requests;

src/Main.c

@@ -78,6 +78,7 @@ int main() {
                 .cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX,
                 .cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE,
                 .engines = "",
+                .yacy_instance = "",
                 .rate_limit_search_requests = 0,
                 .rate_limit_search_interval = 0,
                 .rate_limit_images_requests = 0,
@@ -101,6 +102,23 @@ int main() {
   }
 
   apply_engines_config(cfg.engines);
+  configure_yacy_engine(cfg.yacy_instance);
+
+  if (cfg.yacy_instance[0] != '\0') {
+    int yacy_enabled = 0;
+    for (int i = 0; i < ENGINE_COUNT; i++) {
+      if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
+        yacy_enabled = ENGINE_REGISTRY[i].enabled;
+        break;
+      }
+    }
+    if (!yacy_enabled) {
+      fprintf(
+          stderr,
+          "[INFO] YaCy instance configured but the yacy engine is not "
+          "enabled (add it to the engines list, e.g. engines=\"*,yacy\")\n");
+    }
+  }
 
   if (cache_init(cfg.cache_dir) != 0) {
     fprintf(stderr,

src/Scraping/Scraping.c

@@ -4,6 +4,7 @@
 #include "Config.h"
 #include <curl/curl.h>
 #include <libxml/HTMLparser.h>
+#include <libxml/parser.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -24,7 +25,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job,
 
   return response_contains(response, "<title>Startpage Captcha</title>") ||
          response_contains(response, "Startpage Captcha") ||
-         response_contains(response, "/static-pages-assets/page-data/captcha/") ||
+         response_contains(response,
+                           "/static-pages-assets/page-data/captcha/") ||
          response_contains(response, ">Startpage Blocked</title>");
 }
 
@@ -39,7 +41,8 @@ static int response_looks_like_results_page(const ScrapeJob *job,
   }
 
   if (strcmp(job->engine->name, "Startpage") == 0) {
-    return response_contains(response, "<title>Startpage Search Results</title>") ||
+    return response_contains(response,
+                             "<title>Startpage Search Results</title>") ||
            response_contains(response, "class=\"w-gl") ||
            response_contains(response, "data-testid=\"gl-title-link\"");
   }
@@ -55,6 +58,11 @@ static int response_looks_like_results_page(const ScrapeJob *job,
            response_contains(response, "Mojeek Search");
   }
 
+  if (strcmp(job->engine->name, "YaCy") == 0) {
+    return response_contains(response, "<item") ||
+           response_contains(response, "<rss");
+  }
+
   return 0;
 }
 
@@ -72,18 +80,24 @@ static void classify_job_response(ScrapeJob *job, const char *response,
     return;
   }
 
-  xmlDocPtr doc = htmlReadMemory(response, response_size, NULL, NULL,
+  xmlDocPtr doc;
+  if (job->engine->is_xml) {
+    doc = xmlReadMemory(response, response_size, NULL, NULL,
+                        XML_PARSE_RECOVER | XML_PARSE_NOERROR |
+                            XML_PARSE_NOWARNING);
+  } else {
+    doc = htmlReadMemory(response, response_size, NULL, NULL,
                          HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |
                              HTML_PARSE_NOWARNING);
+  }
 
   if (!doc) {
     job->status = SCRAPE_STATUS_FETCH_ERROR;
     return;
   }
 
-  job->results_count =
+  job->results_count = job->engine->parser(job->engine->name, doc,
-      job->engine->parser(job->engine->name, doc, job->out_results,
+                                           job->out_results, job->max_results);
-                          job->max_results);
   xmlFreeDoc(doc);
 
   if (job->results_count > 0) {
@@ -200,7 +214,8 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) {
     return -1;
   }
 
-  for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) {
+  for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query;
+       p--) {
     if (p[0] == '%' && p[1] == '2' && p[2] == '0') {
       *p = '+';
       memmove(p + 1, p + 3, strlen(p + 3) + 1);
@@ -279,8 +294,7 @@ int should_retry(ScrapeJob *jobs, int num_jobs) {
 int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) {
   int retries = 0;
 
-retry:
+retry:;
-  ;
   CURLM *multi_handle = curl_multi_init();
   if (!multi_handle)
     return -1;

src/Scraping/Scraping.h

@@ -26,6 +26,7 @@ typedef struct {
   int page_base;
   ParserFunc parser;
   int enabled;
+  int is_xml;
 } SearchEngine;
 
 typedef struct {
@@ -59,6 +60,7 @@ typedef struct {
 extern SearchEngine ENGINE_REGISTRY[];
 extern const int ENGINE_COUNT;
 void apply_engines_config(const char *engines_str);
+void configure_yacy_engine(const char *instance);
 
 size_t write_memory_callback(void *contents, size_t size, size_t nmemb,
                              void *userp);

src/Scraping/ScrapingHttp.c

@@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header,
   struct curl_slist *headers = NULL;
   char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM];
 
+  if (host_header && host_header[0] != '\0') {
     snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header);
-  snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
-
     headers = curl_slist_append(headers, host_buf);
+  }
+  if (referer && referer[0] != '\0') {
+    snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer);
     headers = curl_slist_append(headers, ref_buf);
+  }
   headers = curl_slist_append(
       headers,
       "Accept: "

src/Scraping/ScrapingParsers.c

@@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
   if (!ctx)
     return 0;
 
-  xmlXPathObjectPtr obj =
+  xmlXPathObjectPtr obj = xml_xpath_eval(
-      xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
+      ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]");
 
   if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
     free_xpath_objects(ctx, obj);
@@ -268,16 +268,15 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
     xmlNodePtr result_node = obj->nodesetval->nodeTab[i];
     ctx->node = result_node;
 
-    xmlXPathObjectPtr link_obj =
+    xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']");
-        xml_xpath_eval(ctx, ".//a[@class='title']");
     char *url =
         (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
             ? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0],
                                  (xmlChar *)"href")
             : NULL;
 
-    char *title = (link_obj && link_obj->nodesetval &&
+    char *title =
-                   link_obj->nodesetval->nodeNr > 0)
+        (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
             ? xml_node_content(link_obj->nodesetval->nodeTab[0])
             : NULL;
 
@@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc,
 static int parse_mojeek(const char *engine_name, xmlDocPtr doc,
                         SearchResult **out_results, int max_results);
 
+static int parse_yacy(const char *engine_name, xmlDocPtr doc,
+                      SearchResult **out_results, int max_results) {
+  (void)engine_name;
+  int found_count = 0;
+
+  xmlXPathContextPtr ctx = create_xpath_context(doc);
+  if (!ctx)
+    return 0;
+
+  xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item");
+
+  if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) {
+    free_xpath_objects(ctx, obj);
+    return 0;
+  }
+
+  int num_items = obj->nodesetval->nodeNr;
+  *out_results = alloc_results_array(num_items, max_results);
+  if (!*out_results) {
+    free_xpath_objects(ctx, obj);
+    return 0;
+  }
+
+  for (int i = 0; i < num_items && found_count < max_results; i++) {
+    xmlNodePtr item_node = obj->nodesetval->nodeTab[i];
+    ctx->node = item_node;
+
+    xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title");
+    char *title = (title_obj && title_obj->nodesetval &&
+                   title_obj->nodesetval->nodeNr > 0)
+                      ? xml_node_content(title_obj->nodesetval->nodeTab[0])
+                      : NULL;
+
+    xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link");
+    char *url =
+        (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0)
+            ? xml_node_content(link_obj->nodesetval->nodeTab[0])
+            : NULL;
+
+    xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description");
+    char *snippet_text =
+        (desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0)
+            ? xml_node_content(desc_obj->nodesetval->nodeTab[0])
+            : NULL;
+
+    if (url && title) {
+      assign_result(&(*out_results)[found_count], url, title, snippet_text, 0);
+      found_count++;
+    }
+
+    free_xml_node_list(title, url, snippet_text);
+    if (title_obj)
+      xmlXPathFreeObject(title_obj);
+    if (link_obj)
+      xmlXPathFreeObject(link_obj);
+    if (desc_obj)
+      xmlXPathFreeObject(desc_obj);
+  }
+
+  ctx->node = NULL;
+  free_xpath_objects(ctx, obj);
+  return found_count;
+}
+
+static char yacy_base_url[BUFFER_SIZE_LARGE];
+
+void configure_yacy_engine(const char *instance) {
+  if (!instance || instance[0] == '\0')
+    instance = DEFAULT_YACY_INSTANCE;
+
+  snprintf(yacy_base_url, sizeof(yacy_base_url),
+           "%s/"
+           "yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&"
+           "nav=all&maximumRecords=%d&query=",
+           instance, MAX_RESULTS_PER_ENGINE);
+
+  for (int i = 0; i < ENGINE_COUNT; i++) {
+    if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) {
+      ENGINE_REGISTRY[i].base_url = yacy_base_url;
+      break;
+    }
+  }
+}
+
 SearchEngine ENGINE_REGISTRY[] = {
     {.id = "ddg",
      .name = "DuckDuckGo Lite",
@@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = {
      .page_multiplier = 10,
      .page_base = 1,
      .parser = parse_mojeek,
-     .enabled = 1}};
+     .enabled = 1},
+    {.id = "yacy",
+     .name = "YaCy",
+     .base_url = "",
+     .host_header = NULL,
+     .referer = NULL,
+     .page_param = "startRecord",
+     .page_multiplier = 10,
+     .page_base = 0,
+     .parser = parse_yacy,
+     .enabled = 0,
+     .is_xml = 1}};
 
 const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine);
 
@@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) {
 void apply_engines_config(const char *engines_str) {
   if (!engines_str || engines_str[0] == '\0') {
     for (int i = 0; i < ENGINE_COUNT; i++) {
+      if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
+        continue;
       ENGINE_REGISTRY[i].enabled = 1;
     }
     return;
@@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) {
 
     if (strcmp(token, "*") == 0) {
       for (int i = 0; i < ENGINE_COUNT; i++) {
+        if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy"))
+          continue;
         ENGINE_REGISTRY[i].enabled = 1;
       }
     } else if (token[0] == '-' && token[1] != '\0') {