diff --git a/.gitignore b/.gitignore index a10140e..b544c16 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /bin /obj -config.ini \ No newline at end of file +config.ini + +/result diff --git a/Dockerfile b/Dockerfile index 11aac36..6bc7e49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN apk add --no-cache \ WORKDIR /app COPY . /app -RUN git clone https://git.bwaaa.monster/beaker /tmp/beaker && \ +RUN git clone https://git.bwaaa.space/beaker /tmp/beaker && \ cd /tmp/beaker && \ make && make install && \ rm -rf /tmp/beaker diff --git a/Makefile b/Makefile index e57c991..af6b741 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,9 @@ GIT_HASH := $(shell git rev-parse --short HEAD) GIT_DATE := $(shell git log -1 --format='%ad' --date='format:%y.%m.%d') GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) GIT_REMOTE := $(shell git remote get-url origin) +GIT_TAG := $(shell git describe --tags --exact-match HEAD 2>/dev/null) -VERSION := $(GIT_DATE)+$(GIT_HASH)_$(GIT_BRANCH) +VERSION := $(if $(GIT_TAG),$(GIT_TAG),$(GIT_DATE)+$(GIT_HASH)_$(GIT_BRANCH)) CFLAGS := -Wall -Wextra -O2 -Isrc -DVERSION='"$(VERSION)"' -DGIT_REMOTE='"$(GIT_REMOTE)"' diff --git a/README.md b/README.md index 7798229..a66d0e7 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Create a config.ini, there is an example included in the root. Or if you install ## Dependencies - libxml2 - libcurl (may be replaced in the future with curl-impersonate) -- beaker [(source)](https://git.bwaaa.monster/beaker/) +- beaker [(source)](https://git.bwaaa.space/beaker/) ## First Setup Depending on your system, you may first need to install libcurl and libxml2. @@ -47,6 +47,32 @@ On Alpine, `shadow` is needed for the user creation process during the install. # xbps-install -S libxml2-devel libcurl-devel ``` +### NixOS +Add the flake to your inputs and import the module. That is all you need. +Here's an example of using the modules in a flake: +```nix +# flake.nix +{ + inputs = { + omnisearch = { + url = "git+https://git.bwaaa.monster/omnisearch"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = { self, nixpkgs, omnisearch, ... }: { + nixosConfigurations.mySystem = nixpkgs.lib.nixosSystem { + modules = [ + omnisearch.nixosModules.default + { + services.omnisearch.enable = true; + } + ]; + }; + }; +} +``` + ### macOS (Homebrew) ``` $ brew install libxml2 curl openssl pkg-config @@ -61,7 +87,7 @@ On macOS the build uses `pkg-config` to discover `libxml2`, `libcurl` and OpenSS Install libbeaker: ``` -$ git clone https://git.bwaaa.monster/beaker +$ git clone https://git.bwaaa.space/beaker $ cd beaker $ make # make install @@ -71,7 +97,7 @@ On macOS `beaker` installs to `/usr/local/` by default and installs `libbeaker.d And then install omnisearch: ``` -$ git clone https://git.bwaaa.monster/omnisearch +$ git clone https://git.bwaaa.space/omnisearch $ cd omnisearch $ make # make install- @@ -89,7 +115,7 @@ You need Docker or Podman and Docker Compose installed on your system. Run the container: ``` -$ git clone https://git.bwaaa.monster/omnisearch +$ git clone https://git.bwaaa.space/omnisearch $ cd omnisearch $ docker compose up -d --build ``` @@ -113,6 +139,6 @@ $ git merge origin/master If there are conflicts in the files you modified, resolve them manually. You should also make the changes to the cloned repository, and then run the install command again if you installed omnisearch. Changes made directly to the configuration/assets folder will be overwritten on reinstall. ## Contribution -Generate a patch with ```git format-patch HEAD~1``` and email to [gabriel@bwaaa.monster](mailto:gabriel@bwaaa.monster), beginning the subject line with [PATCH omnisearch] +Generate a patch with ```git format-patch HEAD~1``` and email to [frostinator33@airmail.cc](mailto:frostinator33@airmail.cc), beginning the subject line with [PATCH omnisearch] *If you are sending a revised version of a previous patch, please use [PATCH omnisearch v2, v3, etc].* diff --git a/example-config.ini b/example-config.ini index 2760c53..55413a8 100644 --- a/example-config.ini +++ b/example-config.ini @@ -31,8 +31,11 @@ port = 8087 [engines] # Use * for all engines, or specify comma-separated list (e.g., ddg,yahoo) # Use *,-engine to exclude specific engines (e.g., *,-startpage) -# Available engines: ddg, startpage, yahoo, mojeek +# Available engines: ddg, startpage, yahoo, mojeek, yacy engines="*" +# YaCy instance to use for the yacy engine (optional, default: http://127.0.0.1:8090) +# Requires "yacy" to be in the engines list above. +#yacy_instance = http://127.0.0.1:8090 [rate_limit] # Rate limit searches per interval diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..b9619a1 --- /dev/null +++ b/flake.lock @@ -0,0 +1,43 @@ +{ + "nodes": { + "beaker-src": { + "flake": false, + "locked": { + "lastModified": 1790704008, + "narHash": "sha256-Al/BE7uVRSmUGF/t1m0pWxF9jNgzdCVUrc5ZHkcPiQg=", + "ref": "refs/heads/master", + "rev": "10be7f581fb828e74f064ea7ad4fb0534ee8540a", + "revCount": 46, + "shallow": false, + "type": "git", + "url": "https://git.bwaaa.space/beaker" + }, + "original": { + "shallow": false, + "type": "git", + "url": "https://git.bwaaa.space/beaker" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1791456040, + "narHash": "sha256-gRZ/s8WQJ2CsXdhcDVQH34qxuL0MuMCOs3aBiK1Prsw=", + "rev": "e7439b6b14ad3cc35d05608ebca9bce01a25f5f8", + "type": "tarball", + "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1087755.e7439b6b14ad/nixexprs.tar.zst?lastModified=1791456040&rev=e7439b6b14ad3cc35d05608ebca9bce01a25f5f8" + }, + "original": { + "type": "tarball", + "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.zst" + } + }, + "root": { + "inputs": { + "beaker-src": "beaker-src", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..ef87c1d --- /dev/null +++ b/flake.nix @@ -0,0 +1,76 @@ +{ + inputs = { + nixpkgs.url = "https://channels.nixos.org/nixos-unstable/nixexprs.tar.zst"; + beaker-src = { + url = "git+https://git.bwaaa.space/beaker?shallow=0"; + flake = false; + }; + }; + + outputs = + { + self, + nixpkgs, + beaker-src, + }: + let + supportedSystems = [ + "x86_64-linux" + "aarch64-linux" + ]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + in + { + formatter = forAllSystems (system: nixpkgs.legacyPackages.${system}.nixfmt); + packages = forAllSystems ( + system: + let + pkgs = import nixpkgs { inherit system; }; + + beaker = pkgs.stdenv.mkDerivation { + pname = "beaker"; + version = "git"; + src = beaker-src; + makeFlags = [ + "INSTALL_PREFIX=$(out)/" + "LDCONFIG=true" + ]; + }; + in + { + default = pkgs.stdenv.mkDerivation { + pname = "omnisearch"; + version = "git"; + src = ./.; + + buildInputs = [ + pkgs.libxml2.dev + pkgs.curl.dev + pkgs.openssl + beaker + ]; + + preBuild = '' + makeFlagsArray+=( + "PREFIX=$out" + "CFLAGS=-Wall -Wextra -O2 -Isrc -I${pkgs.libxml2.dev}/include/libxml2 -I${beaker}/include/beaker.h" + "LIBS=-lbeaker -lcurl -lxml2 -lpthread -lm -lssl -lcrypto" + ) + ''; + + installPhase = '' + mkdir -p $out/bin $out/share/omnisearch + install -Dm755 bin/omnisearch $out/bin/omnisearch + cp -r templates static locales -t $out/share/omnisearch/ + ''; + + meta = { + description = "Lightweight metasearch engine in C"; + platforms = pkgs.lib.platforms.linux; + }; + }; + } + ); + nixosModules.default = import ./module.nix self; + }; +} diff --git a/locales/en_gb.ini b/locales/en_gb.ini index db919a3..c3b1dc8 100644 --- a/locales/en_gb.ini +++ b/locales/en_gb.ini @@ -21,6 +21,8 @@ display_language_label = "Display Language" language_desc = "Choose your preferred language." engines_label = "Search Engines" engines_desc = "Choose which search engines to use. Only engines enabled on the server are shown." +default_engine_label = "Default Search Engine" +all_engines_option = "All enabled engines" save_settings_button = "Save Settings" no_results = "No results found" error_images = "Error fetching images" diff --git a/locales/en_us.ini b/locales/en_us.ini index 422f220..0764147 100644 --- a/locales/en_us.ini +++ b/locales/en_us.ini @@ -21,6 +21,8 @@ display_language_label = "Display Language" language_desc = "Choose your preferred language." engines_label = "Search Engines" engines_desc = "Choose which search engines to use. Only engines enabled on the server are shown." +default_engine_label = "Default Search Engine" +all_engines_option = "All enabled engines" save_settings_button = "Save Settings" no_results = "No results found" error_images = "Error fetching images" diff --git a/locales/fi_fi.ini b/locales/fi_fi.ini new file mode 100644 index 0000000..a921af7 --- /dev/null +++ b/locales/fi_fi.ini @@ -0,0 +1,36 @@ +[Meta] +Id = "fi_fi" +Name = "Suomi" +Direction = "ltr" + +[Keys] +search_placeholder = "Hae netistä..." +search_button = "Hae" +surprise_me_button = "Kokeilen onneani" +all_tab = "Kaikki" +images_tab = "Kuvat" +settings_tab = "Asetukset" +settings_title = "Asetukset" +theme_label = "Ulkonäkö" +theme_desc = "Valitse haluamasi väriteema." +theme_system = "Järjestelmä" +theme_light = "Vaalea" +theme_dark = "Tumma" +language_label = "Kieli" +display_language_label = "Näyttökieli" +language_desc = "Valitse haluamasi kieli." +engines_label = "Hakukoneet" +engines_desc = "Valitse mitä hakukoneita käytetään. Vain palvelimella käyttöön otetut hakukoneet näytetään." +default_engine_label = "Oletushakukone" +all_engines_option = "Kaikki valitut hakukoneet" +save_settings_button = "Tallenna asetukset" +no_results = "Ei tuloksia" +error_images = "Virhe kuvien haussa" +rate_limit = "Hidasta hieman! Sinulta on tullut liian monta hakupyyntöä!" +warning_fetch_error = "pyyntö epäonnistui ennen kuin OmniSearch ehti lukea hakutuloksia." +warning_parse_mismatch = "palautti hakutulokset muodossa jota OmniSearch ei osaa jäsentää." +warning_blocked = "palautti captchan tai muun estosivun hakutulosten sijaan." +read_more = "Lue Lisää" +view_cached = "Välimuistissa" +view_image = "Kuva" +visit_site = "Sivu" diff --git a/module.nix b/module.nix new file mode 100644 index 0000000..c0c7825 --- /dev/null +++ b/module.nix @@ -0,0 +1,142 @@ +self: + +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.omnisearch; + pkg = cfg.package; + + finalConfigFile = + if cfg.configFile != null then + cfg.configFile + else + pkgs.writeText "omnisearch.ini" '' + [server] + host = ${cfg.settings.server.host} + port = ${toString cfg.settings.server.port} + domain = ${cfg.settings.server.domain} + ${lib.optionalString (cfg.settings.server.locale != null) "locale = ${cfg.settings.server.locale}"} + + [proxy] + ${lib.optionalString (cfg.settings.proxy.proxy != null) "proxy = \"${cfg.settings.proxy.proxy}\""} + ${lib.optionalString ( + cfg.settings.proxy.list_file != null + ) "list_file = ${cfg.settings.proxy.list_file}"} + max_retries = ${toString cfg.settings.proxy.max_retries} + randomize_username = ${lib.boolToString cfg.settings.proxy.randomize_username} + randomize_password = ${lib.boolToString cfg.settings.proxy.randomize_password} + + [cache] + dir = ${cfg.settings.cache.dir} + ttl_search = ${toString cfg.settings.cache.ttl_search} + ttl_infobox = ${toString cfg.settings.cache.ttl_infobox} + ''; +in +{ + options.services.omnisearch = { + enable = lib.mkEnableOption "OmniSearch metasearch engine"; + + package = lib.mkOption { + type = lib.types.package; + default = self.packages.${pkgs.stdenv.hostPlatform.system}.default; + description = "The omnisearch package to use."; + }; + + configFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "Path to a custom config.ini. Overrides 'settings'."; + }; + + settings = { + server = { + host = lib.mkOption { + type = lib.types.str; + default = "0.0.0.0"; + }; + port = lib.mkOption { + type = lib.types.port; + default = 8087; + }; + domain = lib.mkOption { + type = lib.types.str; + default = "http://localhost:${toString cfg.settings.server.port}"; + }; + locale = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + }; + }; + proxy = { + proxy = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + }; + list_file = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + }; + max_retries = lib.mkOption { + type = lib.types.int; + default = 3; + }; + randomize_username = lib.mkOption { + type = lib.types.bool; + default = true; + }; + randomize_password = lib.mkOption { + type = lib.types.bool; + default = true; + }; + }; + cache = { + dir = lib.mkOption { + type = lib.types.str; + default = "/var/cache/omnisearch"; + }; + ttl_search = lib.mkOption { + type = lib.types.int; + default = 3600; + }; + ttl_infobox = lib.mkOption { + type = lib.types.int; + default = 86400; + }; + }; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.omnisearch = { + description = "OmniSearch Service"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + ExecStart = "${pkg}/bin/omnisearch"; + + WorkingDirectory = "/var/lib/omnisearch"; + StateDirectory = "omnisearch"; + CacheDirectory = "omnisearch"; + + BindReadOnlyPaths = [ + "${pkg}/share/omnisearch/templates:/var/lib/omnisearch/templates" + "${pkg}/share/omnisearch/static:/var/lib/omnisearch/static" + "${pkg}/share/omnisearch/locales:/var/lib/omnisearch/locales" + "${finalConfigFile}:/var/lib/omnisearch/config.ini" + ]; + + DynamicUser = true; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + Restart = "always"; + }; + }; + }; +} diff --git a/src/Config.c b/src/Config.c index 9883d45..aef7504 100644 --- a/src/Config.c +++ b/src/Config.c @@ -1,120 +1,189 @@ #include "Config.h" + +#include +#include +#include #include #include #include +#include + +typedef enum { + CONFIG_STRING, + CONFIG_INTEGER, + CONFIG_BOOLEAN, +} ConfigValueType; + +typedef struct { + const char *section; + const char *key; + ConfigValueType type; + size_t offset; + size_t size; +} ConfigOption; + +#define STRING_OPTION(section_name, key_name, field) \ + {section_name, key_name, CONFIG_STRING, offsetof(Config, field), \ + sizeof(((Config *)0)->field)} +#define INTEGER_OPTION(section_name, key_name, field) \ + {section_name, key_name, CONFIG_INTEGER, offsetof(Config, field), 0} +#define BOOLEAN_OPTION(section_name, key_name, field) \ + {section_name, key_name, CONFIG_BOOLEAN, offsetof(Config, field), 0} + +static const ConfigOption CONFIG_OPTIONS[] = { + STRING_OPTION("server", "host", host), + INTEGER_OPTION("server", "port", port), + STRING_OPTION("server", "locale", default_locale), + STRING_OPTION("proxy", "proxy", proxy), + STRING_OPTION("proxy", "list_file", proxy_list_file), + INTEGER_OPTION("proxy", "max_retries", max_proxy_retries), + BOOLEAN_OPTION("proxy", "randomize_username", randomize_username), + BOOLEAN_OPTION("proxy", "randomize_password", randomize_password), + STRING_OPTION("cache", "dir", cache_dir), + INTEGER_OPTION("cache", "ttl_search", cache_ttl_search), + INTEGER_OPTION("cache", "ttl_infobox", cache_ttl_infobox), + INTEGER_OPTION("cache", "ttl_image", cache_ttl_image), + STRING_OPTION("engines", "engines", engines), + STRING_OPTION("engines", "yacy_instance", yacy_instance), + INTEGER_OPTION("rate_limit", "search_requests", rate_limit_search_requests), + INTEGER_OPTION("rate_limit", "search_interval", rate_limit_search_interval), + INTEGER_OPTION("rate_limit", "images_requests", rate_limit_images_requests), + INTEGER_OPTION("rate_limit", "images_interval", rate_limit_images_interval), +}; + +void config_set_defaults(Config *config) { + if (!config) { + return; + } + + *config = (Config){ + .host = DEFAULT_HOST, + .port = DEFAULT_PORT, + .default_locale = "en_gb", + .max_proxy_retries = DEFAULT_MAX_PROXY_RETRIES, + .cache_dir = DEFAULT_CACHE_DIR, + .cache_ttl_search = DEFAULT_CACHE_TTL_SEARCH, + .cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX, + .cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE, + }; +} + +static char *trim(char *value) { + while (*value == ' ' || *value == '\t') { + value++; + } + + char *end = value + strlen(value); + while (end > value && (end[-1] == ' ' || end[-1] == '\t')) { + end--; + } + *end = '\0'; + return value; +} + +static char *unquote(char *value) { + size_t length = strlen(value); + if (length >= 2 && (value[0] == '"' || value[0] == '\'') && + value[length - 1] == value[0]) { + value[length - 1] = '\0'; + return value + 1; + } + return value; +} + +static int parse_integer(const char *value, int *result) { + char *end; + errno = 0; + long parsed = strtol(value, &end, 10); + + if (value == end || *end != '\0' || errno == ERANGE || parsed < INT_MIN || + parsed > INT_MAX) { + return -1; + } + + *result = (int)parsed; + return 0; +} + +static int parse_boolean(const char *value, int *result) { + if (strcmp(value, "1") == 0 || strcasecmp(value, "true") == 0 || + strcasecmp(value, "yes") == 0 || strcasecmp(value, "on") == 0) { + *result = 1; + return 0; + } + if (strcmp(value, "0") == 0 || strcasecmp(value, "false") == 0 || + strcasecmp(value, "no") == 0 || strcasecmp(value, "off") == 0) { + *result = 0; + return 0; + } + return -1; +} + +static void set_option(Config *config, const char *section, const char *key, + const char *value) { + size_t option_count = sizeof(CONFIG_OPTIONS) / sizeof(CONFIG_OPTIONS[0]); + + for (size_t i = 0; i < option_count; i++) { + const ConfigOption *option = &CONFIG_OPTIONS[i]; + if (strcmp(section, option->section) != 0 || + strcmp(key, option->key) != 0) { + continue; + } + + void *destination = (char *)config + option->offset; + if (option->type == CONFIG_STRING) { + snprintf(destination, option->size, "%s", value); + } else if (option->type == CONFIG_INTEGER) { + parse_integer(value, destination); + } else { + parse_boolean(value, destination); + } + return; + } +} int load_config(const char *filename, Config *config) { + if (!filename || !config) { + return -1; + } + FILE *file = fopen(filename, "r"); if (!file) { return -1; } - char line[512]; + char line[1024]; char section[64] = ""; while (fgets(line, sizeof(line), file)) { - line[strcspn(line, "\r\n")] = 0; + line[strcspn(line, "\r\n")] = '\0'; + char *content = trim(line); - if (line[0] == '\0' || line[0] == '#' || line[0] == ';') { + if (*content == '\0' || *content == '#' || *content == ';') { continue; } - if (line[0] == '[') { - char *end = strchr(line, ']'); + if (*content == '[') { + char *end = strchr(content + 1, ']'); if (end) { *end = '\0'; - snprintf(section, sizeof(section), "%.*s", (int)(sizeof(section) - 1), - line + 1); - section[sizeof(section) - 1] = '\0'; + snprintf(section, sizeof(section), "%s", trim(content + 1)); } continue; } - char *delimiter = strchr(line, '='); - if (delimiter) { - *delimiter = '\0'; - char *key = line; - char *value = delimiter + 1; - - while (*key == ' ' || *key == '\t') - key++; - while (*value == ' ' || *value == '\t') - value++; - - char *key_end = key + strlen(key) - 1; - while (key_end > key && (*key_end == ' ' || *key_end == '\t')) { - *key_end = '\0'; - key_end--; - } - - char *value_end = value + strlen(value) - 1; - while (value_end > value && (*value_end == ' ' || *value_end == '\t' || - *value_end == '"' || *value_end == '\'')) { - *value_end = '\0'; - value_end--; - } - - while (*value == ' ' || *value == '\t') - value++; - while (*value == '"' || *value == '\'') - value++; - - if (strcmp(section, "server") == 0) { - if (strcmp(key, "host") == 0) { - strncpy(config->host, value, sizeof(config->host) - 1); - config->host[sizeof(config->host) - 1] = '\0'; - } else if (strcmp(key, "port") == 0) { - config->port = atoi(value); - } else if (strcmp(key, "locale") == 0) { - strncpy(config->default_locale, value, - sizeof(config->default_locale) - 1); - config->default_locale[sizeof(config->default_locale) - 1] = '\0'; - } - } else if (strcmp(section, "proxy") == 0) { - if (strcmp(key, "proxy") == 0) { - strncpy(config->proxy, value, sizeof(config->proxy) - 1); - config->proxy[sizeof(config->proxy) - 1] = '\0'; - } else if (strcmp(key, "list_file") == 0) { - strncpy(config->proxy_list_file, value, - sizeof(config->proxy_list_file) - 1); - config->proxy_list_file[sizeof(config->proxy_list_file) - 1] = '\0'; - } else if (strcmp(key, "max_retries") == 0) { - config->max_proxy_retries = atoi(value); - } else if (strcmp(key, "randomize_username") == 0) { - config->randomize_username = atoi(value); - } else if (strcmp(key, "randomize_password") == 0) { - config->randomize_password = atoi(value); - } - } else if (strcmp(section, "cache") == 0) { - if (strcmp(key, "dir") == 0) { - strncpy(config->cache_dir, value, sizeof(config->cache_dir) - 1); - config->cache_dir[sizeof(config->cache_dir) - 1] = '\0'; - } else if (strcmp(key, "ttl_search") == 0) { - config->cache_ttl_search = atoi(value); - } else if (strcmp(key, "ttl_infobox") == 0) { - config->cache_ttl_infobox = atoi(value); - } else if (strcmp(key, "ttl_image") == 0) { - config->cache_ttl_image = atoi(value); - } - } else if (strcmp(section, "engines") == 0) { - if (strcmp(key, "engines") == 0) { - strncpy(config->engines, value, sizeof(config->engines) - 1); - config->engines[sizeof(config->engines) - 1] = '\0'; - } - } else if (strcmp(section, "rate_limit") == 0) { - if (strcmp(key, "search_requests") == 0) { - config->rate_limit_search_requests = atoi(value); - } else if (strcmp(key, "search_interval") == 0) { - config->rate_limit_search_interval = atoi(value); - } else if (strcmp(key, "images_requests") == 0) { - config->rate_limit_images_requests = atoi(value); - } else if (strcmp(key, "images_interval") == 0) { - config->rate_limit_images_interval = atoi(value); - } - } + char *delimiter = strchr(content, '='); + if (!delimiter) { + continue; } + + *delimiter = '\0'; + char *key = trim(content); + char *value = unquote(trim(delimiter + 1)); + set_option(config, section, key, value); } + int result = ferror(file) ? -1 : 0; fclose(file); - return 0; + return result; } diff --git a/src/Config.h b/src/Config.h index 25bd978..17ce1d9 100644 --- a/src/Config.h +++ b/src/Config.h @@ -8,6 +8,7 @@ #define DEFAULT_CACHE_TTL_INFOBOX 86400 #define DEFAULT_CACHE_TTL_IMAGE 604800 #define DEFAULT_MAX_PROXY_RETRIES 3 +#define DEFAULT_YACY_INSTANCE "http://127.0.0.1:8090" #define BUFFER_SIZE_SMALL 256 #define BUFFER_SIZE_MEDIUM 512 @@ -46,12 +47,14 @@ typedef struct { int cache_ttl_infobox; int cache_ttl_image; char engines[512]; + char yacy_instance[512]; int rate_limit_search_requests; int rate_limit_search_interval; int rate_limit_images_requests; int rate_limit_images_interval; } Config; +void config_set_defaults(Config *config); int load_config(const char *filename, Config *config); #endif diff --git a/src/Main.c b/src/Main.c index d7ff185..8016c65 100644 --- a/src/Main.c +++ b/src/Main.c @@ -12,6 +12,7 @@ #include "Routes/Home.h" #include "Routes/ImageProxy.h" #include "Routes/Images.h" +#include "Routes/Info.h" #include "Routes/Search.h" #include "Routes/Settings.h" #include "Routes/SettingsSave.h" @@ -65,23 +66,8 @@ int main() { curl_global_init(CURL_GLOBAL_DEFAULT); - Config cfg = {.host = DEFAULT_HOST, - .port = DEFAULT_PORT, - .default_locale = "en_gb", - .proxy = "", - .proxy_list_file = "", - .max_proxy_retries = DEFAULT_MAX_PROXY_RETRIES, - .randomize_username = 0, - .randomize_password = 0, - .cache_dir = DEFAULT_CACHE_DIR, - .cache_ttl_search = DEFAULT_CACHE_TTL_SEARCH, - .cache_ttl_infobox = DEFAULT_CACHE_TTL_INFOBOX, - .cache_ttl_image = DEFAULT_CACHE_TTL_IMAGE, - .engines = "", - .rate_limit_search_requests = 0, - .rate_limit_search_interval = 0, - .rate_limit_images_requests = 0, - .rate_limit_images_interval = 0}; + Config cfg; + config_set_defaults(&cfg); if (load_config("config.ini", &cfg) != 0) { fprintf(stderr, "[WARN] Could not load config file, using defaults\n"); @@ -101,6 +87,23 @@ int main() { } apply_engines_config(cfg.engines); + configure_yacy_engine(cfg.yacy_instance); + + if (cfg.yacy_instance[0] != '\0') { + int yacy_enabled = 0; + for (int i = 0; i < ENGINE_COUNT; i++) { + if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) { + yacy_enabled = ENGINE_REGISTRY[i].enabled; + break; + } + } + if (!yacy_enabled) { + fprintf( + stderr, + "[INFO] YaCy instance configured but the yacy engine is not " + "enabled (add it to the engines list, e.g. engines=\"*,yacy\")\n"); + } + } if (cache_init(cfg.cache_dir) != 0) { fprintf(stderr, @@ -136,6 +139,7 @@ int main() { set_handler("/search", results_handler); set_handler("/images", images_handler); set_handler("/proxy", image_proxy_handler); + set_handler("/info", info_handler); set_handler("/settings", settings_handler); set_handler("/save_settings", settings_save_handler); diff --git a/src/Routes/Images.c b/src/Routes/Images.c index 40ab88f..1ab875e 100644 --- a/src/Routes/Images.c +++ b/src/Routes/Images.c @@ -76,14 +76,12 @@ int images_handler(UrlParams *params) { if (!error_images_msg) error_images_msg = "Error fetching images"; - char ***pager_matrix = NULL; - int *pager_inner_counts = NULL; - int pager_count = - build_pagination(page, images_href_builder, (void *)raw_query, - &pager_matrix, &pager_inner_counts); - if (pager_count > 0) { - context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix, - pager_count, pager_inner_counts); + StringMatrix pagination; + string_matrix_build_pagination(page, images_href_builder, (void *)raw_query, + &pagination); + if (pagination.count > 0) { + context_set_array_of_arrays(&ctx, "pagination_links", pagination.rows, + pagination.count, pagination.field_counts); } char *display_query = url_decode_query(raw_query); @@ -91,8 +89,9 @@ int images_handler(UrlParams *params) { if (!raw_query || strlen(raw_query) == 0) { send_redirect("/"); - if (display_query) - free(display_query); + string_matrix_free(&pagination); + free(display_query); + free(locale); free_context(&ctx); return -1; } @@ -128,7 +127,9 @@ int images_handler(UrlParams *params) { snprintf(response, sizeof(response), "

%s

", rate_limit_msg); send_response(response); free(request_cache_key); + string_matrix_free(&pagination); free(display_query); + free(locale); free_context(&ctx); return -1; } @@ -147,37 +148,37 @@ int images_handler(UrlParams *params) { snprintf(error_html, sizeof(error_html), "

%s

", error_images_msg); send_response(error_html); free(request_cache_key); + string_matrix_free(&pagination); free(display_query); + free(locale); free_context(&ctx); return -1; } - char ***image_matrix = malloc(sizeof(char **) * result_count); - int *inner_counts = malloc(sizeof(int) * result_count); - - if (!image_matrix || !inner_counts) { - if (image_matrix) - free(image_matrix); - if (inner_counts) - free(inner_counts); - free_image_results(results, result_count); - free(request_cache_key); - free(display_query); - free_context(&ctx); - return -1; - } + StringMatrix images; + string_matrix_init(&images); for (int i = 0; i < result_count; i++) { - image_matrix[i] = malloc(sizeof(char *) * IMAGE_RESULT_FIELDS); - image_matrix[i][0] = strdup(results[i].thumbnail_url); - image_matrix[i][1] = strdup(results[i].title); - image_matrix[i][2] = strdup(results[i].page_url); - image_matrix[i][3] = strdup(results[i].full_url); - inner_counts[i] = IMAGE_RESULT_FIELDS; + const char *values[IMAGE_RESULT_FIELDS] = { + results[i].thumbnail_url, + results[i].title, + results[i].page_url, + results[i].full_url, + }; + if (string_matrix_append(&images, values, IMAGE_RESULT_FIELDS) != 0) { + string_matrix_free(&images); + string_matrix_free(&pagination); + free_image_results(results, result_count); + free(request_cache_key); + free(display_query); + free(locale); + free_context(&ctx); + return -1; + } } - context_set_array_of_arrays(&ctx, "images", image_matrix, result_count, - inner_counts); + context_set_array_of_arrays(&ctx, "images", images.rows, images.count, + images.field_counts); char *rendered = render_template("images.html", &ctx); if (rendered) { @@ -187,27 +188,13 @@ int images_handler(UrlParams *params) { send_response("

Error rendering image results

"); } - for (int i = 0; i < result_count; i++) { - for (int j = 0; j < IMAGE_RESULT_FIELDS; j++) - free(image_matrix[i][j]); - free(image_matrix[i]); - } - free(image_matrix); - free(inner_counts); - - if (pager_count > 0) { - for (int i = 0; i < pager_count; i++) { - for (int j = 0; j < LINK_FIELD_COUNT; j++) - free(pager_matrix[i][j]); - free(pager_matrix[i]); - } - free(pager_matrix); - free(pager_inner_counts); - } + string_matrix_free(&images); + string_matrix_free(&pagination); free_image_results(results, result_count); free(request_cache_key); free(display_query); + free(locale); free_context(&ctx); return 0; diff --git a/src/Routes/Info.c b/src/Routes/Info.c new file mode 100644 index 0000000..cb5df29 --- /dev/null +++ b/src/Routes/Info.c @@ -0,0 +1,69 @@ +#include "Info.h" +#include "../Config.h" +#include "../Scraping/Scraping.h" +#include "../Utility/Utility.h" +#include +#include +#include +#include + +static int append_json(char *buf, size_t size, size_t *used, const char *format, + ...) { + if (*used >= size) { + return -1; + } + + va_list args; + va_start(args, format); + int written = vsnprintf(buf + *used, size - *used, format, args); + va_end(args); + + if (written < 0 || (size_t)written >= size - *used) { + return -1; + } + + *used += (size_t)written; + return 0; +} + +int info_handler(UrlParams *params) { + (void)params; + extern Config global_config; + + char json[INITIAL_BUFFER_SIZE]; + size_t used = 0; + + if (append_json(json, sizeof(json), &used, "{\"version\":\"%s\",", VERSION) != + 0 || + append_json(json, sizeof(json), &used, "\"git_remote\":\"%s\",", + GIT_REMOTE) != 0 || + append_json(json, sizeof(json), &used, "\"locale\":\"%s\",", + global_config.default_locale) != 0 || + append_json(json, sizeof(json), &used, "\"engines\":[") != 0) { + send_response("Failed to build instance info"); + return -1; + } + + int first = 1; + for (int i = 0; i < ENGINE_COUNT; i++) { + if (!ENGINE_REGISTRY[i].enabled) { + continue; + } + if (append_json(json, sizeof(json), &used, + "%s{\"id\":\"%s\",\"name\":\"%s\"}", first ? "" : ",", + ENGINE_REGISTRY[i].id, ENGINE_REGISTRY[i].name) != 0) { + send_response("Failed to build instance info"); + return -1; + } + first = 0; + } + + if (append_json(json, sizeof(json), &used, "]}") != 0) { + send_response("Failed to build instance info"); + return -1; + } + + serve_data(json, used, "application/json"); + + return 0; +} diff --git a/src/Routes/Info.h b/src/Routes/Info.h new file mode 100644 index 0000000..fee803c --- /dev/null +++ b/src/Routes/Info.h @@ -0,0 +1,8 @@ +#ifndef INFO_H +#define INFO_H + +#include + +int info_handler(UrlParams *params); + +#endif diff --git a/src/Routes/Search.c b/src/Routes/Search.c index b580c10..0aa146a 100644 --- a/src/Routes/Search.c +++ b/src/Routes/Search.c @@ -140,29 +140,31 @@ static InfoBox fetch_currency_wrapper(char *query) { return fetch_currency_data(query); } char *get_base_url(const char *input) { - if (!input) return NULL; + if (!input) + return NULL; - const char *start = input; + const char *start = input; - const char *protocol_pos = strstr(input, "://"); - if (protocol_pos) { - start = protocol_pos + 3; - } + const char *protocol_pos = strstr(input, "://"); + if (protocol_pos) { + start = protocol_pos + 3; + } - const char *end = start; - while (*end && *end != '/' && *end != '?' && *end != '#') { - end++; - } + const char *end = start; + while (*end && *end != '/' && *end != '?' && *end != '#') { + end++; + } - size_t len = end - start; + size_t len = end - start; - char *domain = (char *)malloc(len + 1); - if (!domain) return NULL; + char *domain = (char *)malloc(len + 1); + if (!domain) + return NULL; - strncpy(domain, start, len); - domain[len] = '\0'; + strncpy(domain, start, len); + domain[len] = '\0'; - return domain; + return domain; } static int is_calculator_query(const char *query) { if (!query) @@ -249,88 +251,65 @@ static void *infobox_thread_func(void *arg) { return NULL; } -static int add_infobox_to_collection(InfoBox *infobox, char ****collection, - int **inner_counts, int current_count) { - *collection = - (char ***)realloc(*collection, sizeof(char **) * (current_count + 1)); - *inner_counts = - (int *)realloc(*inner_counts, sizeof(int) * (current_count + 1)); - - (*collection)[current_count] = - (char **)malloc(sizeof(char *) * INFOBOX_FIELD_COUNT); - (*collection)[current_count][0] = - infobox->title ? strdup(infobox->title) : NULL; - (*collection)[current_count][1] = - infobox->thumbnail_url ? strdup(infobox->thumbnail_url) : NULL; - (*collection)[current_count][2] = - infobox->extract ? strdup(infobox->extract) : NULL; - (*collection)[current_count][3] = infobox->url ? strdup(infobox->url) : NULL; - (*collection)[current_count][4] = infobox->url ? strdup(infobox->url) : NULL; - (*inner_counts)[current_count] = INFOBOX_FIELD_COUNT; - - return current_count + 1; +static int add_infobox_to_collection(InfoBox *infobox, + StringMatrix *collection) { + const char *values[INFOBOX_FIELD_COUNT] = { + infobox->title, infobox->thumbnail_url, infobox->extract, + infobox->url, infobox->url, + }; + return string_matrix_append(collection, values, INFOBOX_FIELD_COUNT); } static int add_warning_to_collection(const char *engine_name, const char *warning_message, - char ****collection, int **inner_counts, - int current_count) { - char ***new_collection = - (char ***)malloc(sizeof(char **) * (current_count + 1)); - int *new_inner_counts = - (int *)malloc(sizeof(int) * (current_count + 1)); - - if (!new_collection || !new_inner_counts) { - free(new_collection); - free(new_inner_counts); - return current_count; - } - - if (*collection && current_count > 0) { - memcpy(new_collection, *collection, sizeof(char **) * current_count); - } - if (*inner_counts && current_count > 0) { - memcpy(new_inner_counts, *inner_counts, sizeof(int) * current_count); - } - - free(*collection); - free(*inner_counts); - - *collection = new_collection; - *inner_counts = new_inner_counts; - - (*collection)[current_count] = (char **)malloc(sizeof(char *) * 2); - if (!(*collection)[current_count]) - return current_count; - - (*collection)[current_count][0] = strdup(engine_name ? engine_name : ""); - (*collection)[current_count][1] = - strdup(warning_message ? warning_message : ""); - - if (!(*collection)[current_count][0] || !(*collection)[current_count][1]) { - free((*collection)[current_count][0]); - free((*collection)[current_count][1]); - free((*collection)[current_count]); - return current_count; - } - - (*inner_counts)[current_count] = 2; - return current_count + 1; + StringMatrix *collection) { + const char *values[] = {engine_name, warning_message}; + return string_matrix_append(collection, values, 2); } -static const char *warning_message_for_job(const ScrapeJob *job, const char *locale) { +static void free_user_engine_list(char **user_engines, int user_engine_count) { + for (int i = 0; i < user_engine_count; i++) + free(user_engines[i]); + free(user_engines); +} + +static void free_search_results(SearchResult **all_results, ScrapeJob *jobs, + int job_count) { + for (int i = 0; i < job_count; i++) { + for (int j = 0; all_results[i] && j < jobs[i].results_count; j++) { + free(all_results[i][j].url); + free(all_results[i][j].title); + free(all_results[i][j].snippet); + } + free(all_results[i]); + } +} + +static void free_infobox_results(InfoBoxThreadData *data) { + for (int i = 0; i < HANDLER_COUNT; i++) { + free_infobox(&data[i].result); + } +} + +static const char *warning_message_for_job(const ScrapeJob *job, + const char *locale) { switch (job->status) { case SCRAPE_STATUS_FETCH_ERROR: { const char *msg = beaker_get_locale_value(locale, "warning_fetch_error"); - return msg ? msg : "request failed before OmniSearch could read search results."; + return msg ? msg + : "request failed before OmniSearch could read search results."; } case SCRAPE_STATUS_PARSE_MISMATCH: { const char *msg = beaker_get_locale_value(locale, "warning_parse_mismatch"); - return msg ? msg : "returned search results in a format OmniSearch could not parse."; + return msg ? msg + : "returned search results in a format OmniSearch could not " + "parse."; } case SCRAPE_STATUS_BLOCKED: { const char *msg = beaker_get_locale_value(locale, "warning_blocked"); - return msg ? msg : "returned a captcha or another blocking page instead of search results."; + return msg ? msg + : "returned a captcha or another blocking page instead of " + "search results."; } default: return NULL; @@ -384,8 +363,8 @@ static int engine_allowed_for_user(const SearchEngine *eng, char **user_ids, static char *build_search_href(const char *query, const char *engine_id, int page) { const char *safe_query = query ? query : ""; - int use_engine = engine_id && engine_id[0] != '\0' && - !engine_id_matches(engine_id, "all"); + int use_engine = + engine_id && engine_id[0] != '\0' && !engine_id_matches(engine_id, "all"); size_t needed = strlen("/search?q=") + strlen(safe_query) + 1; if (use_engine) @@ -437,13 +416,18 @@ int results_handler(UrlParams *params) { extern Config global_config; TemplateContext ctx = new_context(); char *raw_query = ""; - const char *selected_engine_id = "all"; + char selected_engine_id[32]; + char *default_engine = get_default_search_engine(); + snprintf(selected_engine_id, sizeof(selected_engine_id), "%s", + default_engine); + free(default_engine); int page = 1; int btnI = 0; char **user_engines = NULL; int user_engine_count = 0; - int has_user_pref = (get_user_engines(&user_engines, &user_engine_count) == 0); + int has_user_pref = + (get_user_engines(&user_engines, &user_engine_count) == 0); if (params) { for (int i = 0; i < params->count; i++) { @@ -454,7 +438,8 @@ int results_handler(UrlParams *params) { if (parsed > 1) page = parsed; } else if (strcmp(params->params[i].key, "engine") == 0) { - selected_engine_id = params->params[i].value; + snprintf(selected_engine_id, sizeof(selected_engine_id), "%s", + params->params[i].value); } else if (strcmp(params->params[i].key, "btnI") == 0) { btnI = atoi(params->params[i].value); } @@ -471,9 +456,11 @@ int results_handler(UrlParams *params) { beaker_set_locale(&ctx, locale); const char *rate_limit_msg = beaker_get_locale_value(locale, "rate_limit"); - if (!rate_limit_msg) rate_limit_msg = "Slow down! Too many searches from you!"; + if (!rate_limit_msg) + rate_limit_msg = "Slow down! Too many searches from you!"; const char *no_results_msg = beaker_get_locale_value(locale, "no_results"); - if (!no_results_msg) no_results_msg = "No results found"; + if (!no_results_msg) + no_results_msg = "No results found"; char page_str[16]; snprintf(page_str, sizeof(page_str), "%d", page); @@ -491,18 +478,24 @@ int results_handler(UrlParams *params) { if (!raw_query || strlen(raw_query) == 0) { send_redirect("/"); - if (has_user_pref) { - for (int i = 0; i < user_engine_count; i++) - free(user_engines[i]); - free(user_engines); - } + free(locale); + free_user_engine_list(user_engines, user_engine_count); free_context(&ctx); return -1; } const SearchEngine *selected_engine = find_enabled_engine(selected_engine_id); - if (!selected_engine) - selected_engine_id = "all"; + if (selected_engine && + !engine_allowed_for_user(selected_engine, user_engines, user_engine_count, + has_user_pref)) { + selected_engine = NULL; + } + if (selected_engine) { + snprintf(selected_engine_id, sizeof(selected_engine_id), "%s", + selected_engine->id); + } else { + snprintf(selected_engine_id, sizeof(selected_engine_id), "all"); + } context_set(&ctx, "selected_engine", selected_engine_id); char *search_href = build_search_href(raw_query, selected_engine_id, 1); @@ -528,13 +521,6 @@ int results_handler(UrlParams *params) { infobox_data[i].result = (InfoBox){NULL}; } - if (page == 1) { - for (int i = 0; i < HANDLER_COUNT; i++) { - pthread_create(&infobox_threads[i], NULL, infobox_thread_func, - &infobox_data[i]); - } - } - ScrapeJob jobs[ENGINE_COUNT]; SearchResult *all_results[ENGINE_COUNT]; @@ -592,11 +578,8 @@ int results_handler(UrlParams *params) { snprintf(response, sizeof(response), "

%s

", rate_limit_msg); send_response(response); free(request_cache_key); - if (has_user_pref) { - for (int i = 0; i < user_engine_count; i++) - free(user_engines[i]); - free(user_engines); - } + free(locale); + free_user_engine_list(user_engines, user_engine_count); free_context(&ctx); return -1; } @@ -615,15 +598,14 @@ int results_handler(UrlParams *params) { } if (filter_engine_count > 1) { - char ***filter_matrix = NULL; - int *filter_inner_counts = NULL; - int filter_count = 0; + StringMatrix filter_matrix; + string_matrix_init(&filter_matrix); char *all_href = build_search_href(raw_query, "all", 1); - - filter_count = add_link_to_collection( + const char *all_values[LINK_FIELD_COUNT] = { all_href, "All", - selected_engine ? "engine-filter" : "engine-filter active", - &filter_matrix, &filter_inner_counts, filter_count); + selected_engine ? "engine-filter" : "engine-filter active"}; + + string_matrix_append(&filter_matrix, all_values, LINK_FIELD_COUNT); free(all_href); for (int i = 0; i < ENGINE_COUNT; i++) { @@ -639,22 +621,24 @@ int results_handler(UrlParams *params) { ? "engine-filter active" : "engine-filter"; - filter_count = add_link_to_collection(filter_href, ENGINE_REGISTRY[i].name, - filter_class, &filter_matrix, - &filter_inner_counts, filter_count); + const char *filter_values[LINK_FIELD_COUNT] = { + filter_href, ENGINE_REGISTRY[i].name, filter_class}; + string_matrix_append(&filter_matrix, filter_values, LINK_FIELD_COUNT); free(filter_href); } - if (filter_count > 0) { - context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix, - filter_count, filter_inner_counts); - for (int i = 0; i < filter_count; i++) { - for (int j = 0; j < LINK_FIELD_COUNT; j++) - free(filter_matrix[i][j]); - free(filter_matrix[i]); - } - free(filter_matrix); - free(filter_inner_counts); + if (filter_matrix.count > 0) { + context_set_array_of_arrays(&ctx, "engine_filters", filter_matrix.rows, + filter_matrix.count, + filter_matrix.field_counts); + } + string_matrix_free(&filter_matrix); + } + + if (page == 1) { + for (int i = 0; i < HANDLER_COUNT; i++) { + pthread_create(&infobox_threads[i], NULL, infobox_thread_func, + &infobox_data[i]); } } @@ -672,27 +656,12 @@ int results_handler(UrlParams *params) { for (int i = 0; i < engine_idx; i++) { if (jobs[i].results_count > 0 && all_results[i][0].url) { char *redirect_url = strdup(all_results[i][0].url); - for (int j = 0; j < enabled_engine_count; j++) { - for (int k = 0; k < jobs[j].results_count; k++) { - free(all_results[j][k].url); - free(all_results[j][k].title); - free(all_results[j][k].snippet); - } - free(all_results[j]); - } - if (page == 1) { - for (int j = 0; j < HANDLER_COUNT; j++) { - if (infobox_data[j].success) { - free_infobox(&infobox_data[j].result); - } - } - } + free_search_results(all_results, jobs, engine_idx); + if (page == 1) + free_infobox_results(infobox_data); free(request_cache_key); - if (has_user_pref) { - for (int i = 0; i < user_engine_count; i++) - free(user_engines[i]); - free(user_engines); - } + free(locale); + free_user_engine_list(user_engines, user_engine_count); free_context(&ctx); if (redirect_url) { send_redirect(redirect_url); @@ -701,54 +670,37 @@ int results_handler(UrlParams *params) { return 0; } } - for (int i = 0; i < enabled_engine_count; i++) { - free(all_results[i]); - } - if (page == 1) { - for (int i = 0; i < HANDLER_COUNT; i++) { - if (infobox_data[i].success) { - free_infobox(&infobox_data[i].result); - } - } - } + free_search_results(all_results, jobs, engine_idx); + if (page == 1) + free_infobox_results(infobox_data); free(request_cache_key); - if (has_user_pref) { - for (int i = 0; i < user_engine_count; i++) - free(user_engines[i]); - free(user_engines); - } + free(locale); + free_user_engine_list(user_engines, user_engine_count); free_context(&ctx); char no_results_html[128]; - snprintf(no_results_html, sizeof(no_results_html), "

%s

", no_results_msg); + snprintf(no_results_html, sizeof(no_results_html), "

%s

", + no_results_msg); send_response(no_results_html); return 0; } - - char ***infobox_matrix = NULL; - int *infobox_inner_counts = NULL; - int infobox_count = 0; + + StringMatrix infobox_matrix; + string_matrix_init(&infobox_matrix); if (page == 1) { for (int i = 0; i < HANDLER_COUNT; i++) { if (infobox_data[i].success) { - infobox_count = - add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix, - &infobox_inner_counts, infobox_count); + add_infobox_to_collection(&infobox_data[i].result, &infobox_matrix); } } } - if (infobox_count > 0) { - context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix, - infobox_count, infobox_inner_counts); - for (int i = 0; i < infobox_count; i++) { - for (int j = 0; j < INFOBOX_FIELD_COUNT; j++) - free(infobox_matrix[i][j]); - free(infobox_matrix[i]); - } - free(infobox_matrix); - free(infobox_inner_counts); + if (infobox_matrix.count > 0) { + context_set_array_of_arrays(&ctx, "infoboxes", infobox_matrix.rows, + infobox_matrix.count, + infobox_matrix.field_counts); } + string_matrix_free(&infobox_matrix); int warning_count = 0; for (int i = 0; i < enabled_engine_count; i++) { @@ -757,35 +709,33 @@ int results_handler(UrlParams *params) { } if (warning_count > 0) { - char ***warning_matrix = NULL; - int *warning_inner_counts = NULL; - int warning_index = 0; + StringMatrix warning_matrix; + string_matrix_init(&warning_matrix); for (int i = 0; i < enabled_engine_count; i++) { const char *warning_message = warning_message_for_job(&jobs[i], locale); if (!warning_message) continue; - warning_index = add_warning_to_collection( - jobs[i].engine->name, warning_message, &warning_matrix, - &warning_inner_counts, warning_index); - } - - if (warning_index > 0) { - context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix, - warning_index, warning_inner_counts); - } - - if (warning_matrix) { - for (int i = 0; i < warning_index; i++) { - free(warning_matrix[i][0]); - free(warning_matrix[i][1]); - free(warning_matrix[i]); + if (jobs[i].http_status >= 400) { + char warning_with_status[BUFFER_SIZE_LARGE]; + snprintf(warning_with_status, sizeof(warning_with_status), + "%s (HTTP %ld)", warning_message, jobs[i].http_status); + add_warning_to_collection(jobs[i].engine->name, warning_with_status, + &warning_matrix); + } else { + add_warning_to_collection(jobs[i].engine->name, warning_message, + &warning_matrix); } - free(warning_matrix); } - if (warning_inner_counts) - free(warning_inner_counts); + + if (warning_matrix.count > 0) { + context_set_array_of_arrays(&ctx, "engine_warnings", warning_matrix.rows, + warning_matrix.count, + warning_matrix.field_counts); + } + + string_matrix_free(&warning_matrix); } int total_results = 0; @@ -794,33 +744,8 @@ int results_handler(UrlParams *params) { } if (total_results > 0) { - char ***results_matrix = (char ***)malloc(sizeof(char **) * total_results); - int *results_inner_counts = (int *)malloc(sizeof(int) * total_results); - if (!results_matrix || !results_inner_counts) { - char *html = render_template("results.html", &ctx); - if (html) { - send_response(html); - free(html); - } - for (int i = 0; i < enabled_engine_count; i++) - free(all_results[i]); - if (page == 1) { - for (int i = 0; i < HANDLER_COUNT; i++) { - if (infobox_data[i].success) { - free_infobox(&infobox_data[i].result); - } - } - } - free(request_cache_key); - if (has_user_pref) { - for (int i = 0; i < user_engine_count; i++) - free(user_engines[i]); - free(user_engines); - } - free_context(&ctx); - return 0; - } - int unique_count = 0; + StringMatrix results_matrix; + string_matrix_init(&results_matrix); UrlHashTable url_table; url_hash_init(&url_table); @@ -837,61 +762,45 @@ int results_handler(UrlParams *params) { url_hash_insert(&url_table, display_url); - results_matrix[unique_count] = - (char **)malloc(sizeof(char *) * RESULT_FIELD_COUNT); - if (!results_matrix[unique_count]) { - free(all_results[i][j].url); - free(all_results[i][j].title); - free(all_results[i][j].snippet); - continue; - } char *pretty_url = pretty_display_url(display_url); char *base_url = get_base_url(display_url); - - results_matrix[unique_count][0] = strdup(display_url); - results_matrix[unique_count][1] = strdup(pretty_url); - results_matrix[unique_count][2] = all_results[i][j].title - ? strdup(all_results[i][j].title) - : strdup("Untitled"); - results_matrix[unique_count][3] = - all_results[i][j].snippet ? strdup(all_results[i][j].snippet) - : strdup(""); - results_matrix[unique_count][4] = strdup(base_url ? base_url : ""); - results_matrix[unique_count][5] = strdup(""); - - results_inner_counts[unique_count] = RESULT_FIELD_COUNT; + const char *values[RESULT_FIELD_COUNT] = { + display_url, + pretty_url, + all_results[i][j].title ? all_results[i][j].title : "Untitled", + all_results[i][j].snippet ? all_results[i][j].snippet : "", + base_url, + "", + }; + int append_result = + string_matrix_append(&results_matrix, values, RESULT_FIELD_COUNT); free(pretty_url); free(base_url); free(all_results[i][j].url); free(all_results[i][j].title); free(all_results[i][j].snippet); - - unique_count++; + if (append_result != 0) { + continue; + } } free(all_results[i]); } - context_set_array_of_arrays(&ctx, "results", results_matrix, unique_count, - results_inner_counts); + context_set_array_of_arrays(&ctx, "results", results_matrix.rows, + results_matrix.count, + results_matrix.field_counts); - char ***pager_matrix = NULL; - int *pager_inner_counts = NULL; - SearchHrefData href_data = { .query = raw_query, .engine_id = selected_engine_id }; - int pager_count = build_pagination(page, search_href_builder, - &href_data, &pager_matrix, - &pager_inner_counts); + StringMatrix pager_matrix; + SearchHrefData href_data = {.query = raw_query, + .engine_id = selected_engine_id}; + int pagination_result = string_matrix_build_pagination( + page, search_href_builder, &href_data, &pager_matrix); - if (pager_count > 0) { - context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix, - pager_count, pager_inner_counts); - for (int i = 0; i < pager_count; i++) { - for (int j = 0; j < LINK_FIELD_COUNT; j++) - free(pager_matrix[i][j]); - free(pager_matrix[i]); - } - free(pager_matrix); - free(pager_inner_counts); + if (pagination_result == 0 && pager_matrix.count > 0) { + context_set_array_of_arrays(&ctx, "pagination_links", pager_matrix.rows, + pager_matrix.count, + pager_matrix.field_counts); } char *html = render_template("results.html", &ctx); @@ -900,13 +809,8 @@ int results_handler(UrlParams *params) { free(html); } - for (int i = 0; i < unique_count; i++) { - for (int j = 0; j < RESULT_FIELD_COUNT; j++) - free(results_matrix[i][j]); - free(results_matrix[i]); - } - free(results_matrix); - free(results_inner_counts); + string_matrix_free(&pager_matrix); + string_matrix_free(&results_matrix); url_hash_free(&url_table); } else { char *html = render_template("results.html", &ctx); @@ -915,26 +819,15 @@ int results_handler(UrlParams *params) { free(html); } - for (int i = 0; i < enabled_engine_count; i++) { - free(all_results[i]); - } + free_search_results(all_results, jobs, engine_idx); } free(request_cache_key); - if (page == 1) { - for (int i = 0; i < HANDLER_COUNT; i++) { - if (infobox_data[i].success) { - free_infobox(&infobox_data[i].result); - } - } - } + if (page == 1) + free_infobox_results(infobox_data); free(locale); - if (has_user_pref) { - for (int i = 0; i < user_engine_count; i++) - free(user_engines[i]); - free(user_engines); - } + free_user_engine_list(user_engines, user_engine_count); free_context(&ctx); return 0; diff --git a/src/Routes/Settings.c b/src/Routes/Settings.c index eb3072b..05325e8 100644 --- a/src/Routes/Settings.c +++ b/src/Routes/Settings.c @@ -34,6 +34,14 @@ int settings_handler(UrlParams *params) { int user_engine_count = 0; int has_user_pref = (get_user_engines(&user_engines, &user_engine_count) == 0); + char *default_engine = get_default_search_engine(); + + if (strcmp(default_engine, "all") != 0 && has_user_pref && + !user_engines_contains(default_engine, user_engines, + user_engine_count)) { + free(default_engine); + default_engine = strdup("all"); + } int enabled_count = 0; for (int i = 0; i < ENGINE_COUNT; i++) { @@ -63,11 +71,13 @@ int settings_handler(UrlParams *params) { } } - engine_data[idx] = malloc(sizeof(char *) * 3); + engine_data[idx] = malloc(sizeof(char *) * 4); engine_data[idx][0] = (char *)ENGINE_REGISTRY[i].id; engine_data[idx][1] = (char *)ENGINE_REGISTRY[i].name; engine_data[idx][2] = is_selected ? "checked" : ""; - engine_inner[idx] = 3; + engine_data[idx][3] = + strcmp(default_engine, ENGINE_REGISTRY[i].id) == 0 ? "selected" : ""; + engine_inner[idx] = 4; idx++; } @@ -78,6 +88,7 @@ int settings_handler(UrlParams *params) { context_set(&ctx, "query", query); context_set(&ctx, "theme", theme); context_set(&ctx, "locale", locale); + context_set(&ctx, "default_engine", default_engine); context_set_array_of_arrays(&ctx, "locales", locale_data, locale_count, inner_counts); @@ -132,6 +143,7 @@ int settings_handler(UrlParams *params) { send_response(rendered_html); free(rendered_html); + free(default_engine); free(theme); free(locale); free_context(&ctx); diff --git a/src/Routes/SettingsSave.c b/src/Routes/SettingsSave.c index cacff50..3cbb1d6 100644 --- a/src/Routes/SettingsSave.c +++ b/src/Routes/SettingsSave.c @@ -10,6 +10,8 @@ int settings_save_handler(UrlParams *params) { const char *theme = ""; const char *locale = ""; const char *query = ""; + const char *default_engine = "all"; + int default_engine_present = 0; int engines_present = 0; char selected_ids[ENGINE_COUNT][32]; int selected_count = 0; @@ -22,6 +24,9 @@ int settings_save_handler(UrlParams *params) { locale = params->params[i].value; } else if (strcmp(params->params[i].key, "q") == 0) { query = params->params[i].value; + } else if (strcmp(params->params[i].key, "default_engine") == 0) { + default_engine = params->params[i].value; + default_engine_present = 1; } else if (strcmp(params->params[i].key, "engines_present") == 0) { engines_present = 1; } else if (strncmp(params->params[i].key, "engine_", 7) == 0 && @@ -58,6 +63,31 @@ int settings_save_handler(UrlParams *params) { false, false); } + if (default_engine_present) { + const char *validated_default = "all"; + + for (int i = 0; i < ENGINE_COUNT; i++) { + if (!ENGINE_REGISTRY[i].enabled || + strcmp(default_engine, ENGINE_REGISTRY[i].id) != 0) + continue; + + int selected = !engines_present; + for (int j = 0; j < selected_count; j++) { + if (strcmp(selected_ids[j], ENGINE_REGISTRY[i].id) == 0) { + selected = 1; + break; + } + } + + if (selected) + validated_default = ENGINE_REGISTRY[i].id; + break; + } + + set_cookie("default_engine", validated_default, + "Fri, 31 Dec 2038 23:59:59 GMT", "/", false, false); + } + char redirect_url[512]; snprintf(redirect_url, sizeof(redirect_url), "/settings?q=%s", query); send_redirect(redirect_url); diff --git a/src/Scraping/Scraping.c b/src/Scraping/Scraping.c index b81c216..6d94450 100644 --- a/src/Scraping/Scraping.c +++ b/src/Scraping/Scraping.c @@ -1,9 +1,11 @@ #include "Scraping.h" #include "../Cache/Cache.h" #include "../Proxy/Proxy.h" +#include "../Utility/XmlHelper.h" #include "Config.h" #include #include +#include #include #include #include @@ -24,7 +26,8 @@ static int response_is_startpage_captcha(const ScrapeJob *job, return response_contains(response, "Startpage Captcha") || response_contains(response, "Startpage Captcha") || - response_contains(response, "/static-pages-assets/page-data/captcha/") || + response_contains(response, + "/static-pages-assets/page-data/captcha/") || response_contains(response, ">Startpage Blocked"); } @@ -39,7 +42,8 @@ static int response_looks_like_results_page(const ScrapeJob *job, } if (strcmp(job->engine->name, "Startpage") == 0) { - return response_contains(response, "Startpage Search Results") || + return response_contains(response, + "Startpage Search Results") || response_contains(response, "class=\"w-gl") || response_contains(response, "data-testid=\"gl-title-link\""); } @@ -55,6 +59,11 @@ static int response_looks_like_results_page(const ScrapeJob *job, response_contains(response, "Mojeek Search"); } + if (strcmp(job->engine->name, "YaCy") == 0) { + return response_contains(response, "engine->is_xml) { + doc = xmlReadMemory(response, response_size, NULL, NULL, + XML_PARSE_RECOVER | XML_PARSE_NOERROR | + XML_PARSE_NOWARNING); + } else { + doc = htmlReadMemory(response, response_size, NULL, NULL, + HTML_PARSE_RECOVER | HTML_PARSE_NOERROR | + HTML_PARSE_NOWARNING); + } if (!doc) { job->status = SCRAPE_STATUS_FETCH_ERROR; return; } - job->results_count = - job->engine->parser(job->engine->name, doc, job->out_results, - job->max_results); + job->results_count = job->engine->parser(job->engine->name, doc, + job->out_results, job->max_results); xmlFreeDoc(doc); if (job->results_count > 0) { @@ -156,14 +171,26 @@ void parse_and_cache_response(ScrapeJob *job) { } } -void cleanup_job_handle(ScrapeJob *job, CURL *handle) { +static void cleanup_job_results(ScrapeJob *job) { + if (!job || !job->out_results) + return; + + xml_result_free(*job->out_results, job->results_count); + *job->out_results = NULL; + job->results_count = 0; +} + +static void cleanup_job_handle(ScrapeJob *job, CURL *handle) { struct curl_slist *headers = NULL; - curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers); + if (handle) + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &headers); if (headers) curl_slist_free_all(headers); free(job->response.memory); job->response.memory = NULL; + job->response.size = 0; + job->response.capacity = 0; } void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) { @@ -180,19 +207,25 @@ void process_response(ScrapeJob *job, CURL *handle, CURLMsg *msg) { } int setup_job(ScrapeJob *job, CURLM *multi_handle) { - if (job->handle) + if (job->handle) { + cleanup_job_handle(job, job->handle); curl_easy_cleanup(job->handle); - if (job->response.memory) + job->handle = NULL; + } else if (job->response.memory) { free(job->response.memory); + job->response.memory = NULL; + job->response.size = 0; + job->response.capacity = 0; + } - job->results_count = 0; + cleanup_job_results(job); job->http_status = 0; job->status = SCRAPE_STATUS_PENDING; - if (check_cache_for_job(job)) { - job->results_count = job->results_count > 0 ? job->results_count : 0; + if (check_cache_for_job(job)) return 0; - } + + cleanup_job_results(job); char *encoded_query = curl_easy_escape(NULL, job->query, 0); if (!encoded_query) { @@ -200,12 +233,17 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) { return -1; } - for (char *p = encoded_query + strlen(encoded_query) - 3; p >= encoded_query; p--) { - if (p[0] == '%' && p[1] == '2' && p[2] == '0') { - *p = '+'; - memmove(p + 1, p + 3, strlen(p + 3) + 1); + char *read = encoded_query; + char *write = encoded_query; + while (*read) { + if (read[0] == '%' && read[1] == '2' && read[2] == '0') { + *write++ = '+'; + read += 3; + } else { + *write++ = *read++; } } + *write = '\0'; char *full_url = build_search_url(job->engine->base_url, job->engine->page_param, @@ -228,6 +266,14 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) { job->response.memory = (char *)malloc(INITIAL_BUFFER_SIZE); job->response.size = 0; job->response.capacity = INITIAL_BUFFER_SIZE; + if (!job->response.memory) { + curl_easy_cleanup(job->handle); + job->handle = NULL; + job->response.capacity = 0; + free(full_url); + job->status = SCRAPE_STATUS_FETCH_ERROR; + return -1; + } struct curl_slist *headers = build_request_headers(job->engine->host_header, job->engine->referer); @@ -236,10 +282,33 @@ int setup_job(ScrapeJob *job, CURLM *multi_handle) { curl_easy_setopt(job->handle, CURLOPT_PRIVATE, headers); free(full_url); - curl_multi_add_handle(multi_handle, job->handle); + CURLMcode add_result = curl_multi_add_handle(multi_handle, job->handle); + if (add_result != CURLM_OK) { + cleanup_job_handle(job, job->handle); + curl_easy_cleanup(job->handle); + job->handle = NULL; + job->status = SCRAPE_STATUS_FETCH_ERROR; + return -1; + } return 0; } +static void cleanup_unfinished_jobs(CURLM *multi_handle, ScrapeJob *jobs, + int num_jobs) { + for (int i = 0; i < num_jobs; i++) { + if (!jobs[i].handle) + continue; + + curl_multi_remove_handle(multi_handle, jobs[i].handle); + cleanup_job_handle(&jobs[i], jobs[i].handle); + curl_easy_cleanup(jobs[i].handle); + jobs[i].handle = NULL; + + if (jobs[i].status == SCRAPE_STATUS_PENDING) + jobs[i].status = SCRAPE_STATUS_FETCH_ERROR; + } +} + int handle_responses(CURLM *multi_handle, ScrapeJob *jobs, int num_jobs) { CURLMsg *msg; int msgs_left; @@ -279,34 +348,30 @@ int should_retry(ScrapeJob *jobs, int num_jobs) { int scrape_engines_parallel(ScrapeJob *jobs, int num_jobs) { int retries = 0; -retry: - ; +retry:; CURLM *multi_handle = curl_multi_init(); if (!multi_handle) return -1; for (int i = 0; i < num_jobs; i++) { - if (setup_job(&jobs[i], multi_handle) != 0 && jobs[i].handle) { - curl_multi_remove_handle(multi_handle, jobs[i].handle); - curl_easy_cleanup(jobs[i].handle); - jobs[i].handle = NULL; - } + setup_job(&jobs[i], multi_handle); } http_delay(); int still_running = 0; - curl_multi_perform(multi_handle, &still_running); + CURLMcode mc = curl_multi_perform(multi_handle, &still_running); - do { + while (mc == CURLM_OK && still_running) { int numfds = 0; - CURLMcode mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds); + mc = curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds); if (mc != CURLM_OK) break; - curl_multi_perform(multi_handle, &still_running); - } while (still_running); + mc = curl_multi_perform(multi_handle, &still_running); + } handle_responses(multi_handle, jobs, num_jobs); + cleanup_unfinished_jobs(multi_handle, jobs, num_jobs); curl_multi_cleanup(multi_handle); if (retries < max_proxy_retries && should_retry(jobs, num_jobs)) { diff --git a/src/Scraping/Scraping.h b/src/Scraping/Scraping.h index be65e5a..14e6e55 100644 --- a/src/Scraping/Scraping.h +++ b/src/Scraping/Scraping.h @@ -26,6 +26,7 @@ typedef struct { int page_base; ParserFunc parser; int enabled; + int is_xml; } SearchEngine; typedef struct { @@ -59,6 +60,7 @@ typedef struct { extern SearchEngine ENGINE_REGISTRY[]; extern const int ENGINE_COUNT; void apply_engines_config(const char *engines_str); +void configure_yacy_engine(const char *instance); size_t write_memory_callback(void *contents, size_t size, size_t nmemb, void *userp); diff --git a/src/Scraping/ScrapingHttp.c b/src/Scraping/ScrapingHttp.c index 1a6a292..89f4853 100644 --- a/src/Scraping/ScrapingHttp.c +++ b/src/Scraping/ScrapingHttp.c @@ -89,11 +89,14 @@ struct curl_slist *build_request_headers(const char *host_header, struct curl_slist *headers = NULL; char host_buf[BUFFER_SIZE_MEDIUM], ref_buf[BUFFER_SIZE_MEDIUM]; - snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header); - snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer); - - headers = curl_slist_append(headers, host_buf); - headers = curl_slist_append(headers, ref_buf); + if (host_header && host_header[0] != '\0') { + snprintf(host_buf, sizeof(host_buf), "Host: %s", host_header); + headers = curl_slist_append(headers, host_buf); + } + if (referer && referer[0] != '\0') { + snprintf(ref_buf, sizeof(ref_buf), "Referer: %s", referer); + headers = curl_slist_append(headers, ref_buf); + } headers = curl_slist_append( headers, "Accept: " diff --git a/src/Scraping/ScrapingParsers.c b/src/Scraping/ScrapingParsers.c index 96aaded..2e5689b 100644 --- a/src/Scraping/ScrapingParsers.c +++ b/src/Scraping/ScrapingParsers.c @@ -249,8 +249,8 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc, if (!ctx) return 0; - xmlXPathObjectPtr obj = - xml_xpath_eval(ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]"); + xmlXPathObjectPtr obj = xml_xpath_eval( + ctx, "//ul[@class='results-standard']/li[starts-with(@class, 'r')]"); if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) { free_xpath_objects(ctx, obj); @@ -268,18 +268,17 @@ static int parse_mojeek(const char *engine_name, xmlDocPtr doc, xmlNodePtr result_node = obj->nodesetval->nodeTab[i]; ctx->node = result_node; - xmlXPathObjectPtr link_obj = - xml_xpath_eval(ctx, ".//a[@class='title']"); + xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, ".//a[@class='title']"); char *url = (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) ? (char *)xmlGetProp(link_obj->nodesetval->nodeTab[0], (xmlChar *)"href") : NULL; - char *title = (link_obj && link_obj->nodesetval && - link_obj->nodesetval->nodeNr > 0) - ? xml_node_content(link_obj->nodesetval->nodeTab[0]) - : NULL; + char *title = + (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) + ? xml_node_content(link_obj->nodesetval->nodeTab[0]) + : NULL; xmlXPathObjectPtr snippet_obj = xml_xpath_eval(ctx, ".//p[@class='s']"); char *snippet_text = @@ -310,6 +309,90 @@ static int parse_yahoo(const char *engine_name, xmlDocPtr doc, static int parse_mojeek(const char *engine_name, xmlDocPtr doc, SearchResult **out_results, int max_results); +static int parse_yacy(const char *engine_name, xmlDocPtr doc, + SearchResult **out_results, int max_results) { + (void)engine_name; + int found_count = 0; + + xmlXPathContextPtr ctx = create_xpath_context(doc); + if (!ctx) + return 0; + + xmlXPathObjectPtr obj = xml_xpath_eval(ctx, "//item"); + + if (!obj || !obj->nodesetval || obj->nodesetval->nodeNr == 0) { + free_xpath_objects(ctx, obj); + return 0; + } + + int num_items = obj->nodesetval->nodeNr; + *out_results = alloc_results_array(num_items, max_results); + if (!*out_results) { + free_xpath_objects(ctx, obj); + return 0; + } + + for (int i = 0; i < num_items && found_count < max_results; i++) { + xmlNodePtr item_node = obj->nodesetval->nodeTab[i]; + ctx->node = item_node; + + xmlXPathObjectPtr title_obj = xml_xpath_eval(ctx, "./title"); + char *title = (title_obj && title_obj->nodesetval && + title_obj->nodesetval->nodeNr > 0) + ? xml_node_content(title_obj->nodesetval->nodeTab[0]) + : NULL; + + xmlXPathObjectPtr link_obj = xml_xpath_eval(ctx, "./link"); + char *url = + (link_obj && link_obj->nodesetval && link_obj->nodesetval->nodeNr > 0) + ? xml_node_content(link_obj->nodesetval->nodeTab[0]) + : NULL; + + xmlXPathObjectPtr desc_obj = xml_xpath_eval(ctx, "./description"); + char *snippet_text = + (desc_obj && desc_obj->nodesetval && desc_obj->nodesetval->nodeNr > 0) + ? xml_node_content(desc_obj->nodesetval->nodeTab[0]) + : NULL; + + if (url && title) { + assign_result(&(*out_results)[found_count], url, title, snippet_text, 0); + found_count++; + } + + free_xml_node_list(title, url, snippet_text); + if (title_obj) + xmlXPathFreeObject(title_obj); + if (link_obj) + xmlXPathFreeObject(link_obj); + if (desc_obj) + xmlXPathFreeObject(desc_obj); + } + + ctx->node = NULL; + free_xpath_objects(ctx, obj); + return found_count; +} + +static char yacy_base_url[BUFFER_SIZE_LARGE]; + +void configure_yacy_engine(const char *instance) { + if (!instance || instance[0] == '\0') + instance = DEFAULT_YACY_INSTANCE; + + snprintf(yacy_base_url, sizeof(yacy_base_url), + "%s/" + "yacysearch.rss?resource=global&urlmaskfilter=.*&prefermaskfilter=&" + "nav=all&maximumRecords=%d&query=", + instance, MAX_RESULTS_PER_ENGINE); + + for (int i = 0; i < ENGINE_COUNT; i++) { + if (strcmp(ENGINE_REGISTRY[i].id, "yacy") == 0) { + ENGINE_REGISTRY[i].base_url = yacy_base_url; + break; + } + } +} + SearchEngine ENGINE_REGISTRY[] = { {.id = "ddg", .name = "DuckDuckGo Lite", @@ -350,7 +433,18 @@ SearchEngine ENGINE_REGISTRY[] = { .page_multiplier = 10, .page_base = 1, .parser = parse_mojeek, - .enabled = 1}}; + .enabled = 1}, + {.id = "yacy", + .name = "YaCy", + .base_url = "", + .host_header = NULL, + .referer = NULL, + .page_param = "startRecord", + .page_multiplier = 10, + .page_base = 0, + .parser = parse_yacy, + .enabled = 0, + .is_xml = 1}}; const int ENGINE_COUNT = sizeof(ENGINE_REGISTRY) / sizeof(SearchEngine); @@ -373,6 +467,8 @@ static int engine_id_compare(const char *engine_id, const char *config_id) { void apply_engines_config(const char *engines_str) { if (!engines_str || engines_str[0] == '\0') { for (int i = 0; i < ENGINE_COUNT; i++) { + if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy")) + continue; ENGINE_REGISTRY[i].enabled = 1; } return; @@ -395,6 +491,8 @@ void apply_engines_config(const char *engines_str) { if (strcmp(token, "*") == 0) { for (int i = 0; i < ENGINE_COUNT; i++) { + if (engine_id_compare(ENGINE_REGISTRY[i].id, "yacy")) + continue; ENGINE_REGISTRY[i].enabled = 1; } } else if (token[0] == '-' && token[1] != '\0') { diff --git a/src/Utility/Utility.c b/src/Utility/Utility.c index 1428722..35fc491 100644 --- a/src/Utility/Utility.c +++ b/src/Utility/Utility.c @@ -14,7 +14,6 @@ static int themes_initialized = 0; void init_themes(const char *static_path) { if (themes_initialized) return; - themes_initialized = 1; char themes_dir[512]; snprintf(themes_dir, sizeof(themes_dir), "%s/themes", static_path); @@ -26,20 +25,32 @@ void init_themes(const char *static_path) { struct dirent *entry; int capacity = 4; themes_list = malloc(sizeof(char *) * capacity); + if (!themes_list) { + closedir(dir); + return; + } themes_count = 0; while ((entry = readdir(dir)) != NULL) { size_t len = strlen(entry->d_name); if (len > 4 && strcmp(entry->d_name + len - 4, ".css") == 0) { if (themes_count >= capacity) { - capacity *= 2; - themes_list = realloc(themes_list, sizeof(char *) * capacity); + int new_capacity = capacity * 2; + char **new_list = realloc(themes_list, sizeof(char *) * new_capacity); + if (!new_list) + break; + themes_list = new_list; + capacity = new_capacity; } - themes_list[themes_count] = strndup(entry->d_name, len - 4); + char *theme = strndup(entry->d_name, len - 4); + if (!theme) + break; + themes_list[themes_count] = theme; themes_count++; } } closedir(dir); + themes_initialized = 1; for (int i = 0; i < themes_count; i++) { for (int j = i + 1; j < themes_count; j++) { @@ -206,98 +217,130 @@ int user_engines_contains(const char *engine_id, char **ids, int count) { return 0; } -int add_link_to_collection(const char *href, const char *label, - const char *class_name, char ****collection, - int **inner_counts, int current_count) { - char ***old_collection = *collection; - int *old_inner_counts = *inner_counts; - char ***new_collection = - (char ***)malloc(sizeof(char **) * (current_count + 1)); - int *new_inner_counts = (int *)malloc(sizeof(int) * (current_count + 1)); +char *get_default_search_engine(void) { + char *cookie = get_cookie("default_engine"); + if (cookie && (strcmp(cookie, "all") == 0 || is_engine_id_enabled(cookie))) + return cookie; - if (!new_collection || !new_inner_counts) { - free(new_collection); - free(new_inner_counts); - return current_count; - } - - if (*collection && current_count > 0) { - memcpy(new_collection, *collection, sizeof(char **) * current_count); - } - if (*inner_counts && current_count > 0) { - memcpy(new_inner_counts, *inner_counts, sizeof(int) * current_count); - } - - *collection = new_collection; - *inner_counts = new_inner_counts; - - (*collection)[current_count] = - (char **)malloc(sizeof(char *) * LINK_FIELD_COUNT); - if (!(*collection)[current_count]) { - *collection = old_collection; - *inner_counts = old_inner_counts; - free(new_collection); - free(new_inner_counts); - return current_count; - } - - (*collection)[current_count][0] = strdup(href ? href : ""); - (*collection)[current_count][1] = strdup(label ? label : ""); - (*collection)[current_count][2] = strdup(class_name ? class_name : ""); - - if (!(*collection)[current_count][0] || !(*collection)[current_count][1] || - !(*collection)[current_count][2]) { - free((*collection)[current_count][0]); - free((*collection)[current_count][1]); - free((*collection)[current_count][2]); - free((*collection)[current_count]); - *collection = old_collection; - *inner_counts = old_inner_counts; - free(new_collection); - free(new_inner_counts); - return current_count; - } - - (*inner_counts)[current_count] = LINK_FIELD_COUNT; - - free(old_collection); - free(old_inner_counts); - return current_count + 1; + free(cookie); + return strdup("all"); } -int build_pagination(int page, char *(*href_builder)(int page, void *data), - void *data, char ****out_matrix, int **out_inner_counts) { +void free_string_matrix(char ***matrix, int *inner_counts, int row_count) { + if (matrix) { + for (int i = 0; i < row_count; i++) { + int field_count = inner_counts ? inner_counts[i] : 0; + for (int j = 0; j < field_count; j++) + free(matrix[i][j]); + free(matrix[i]); + } + } + free(matrix); + free(inner_counts); +} + +void string_matrix_init(StringMatrix *matrix) { + if (matrix) + *matrix = (StringMatrix){0}; +} + +int string_matrix_append(StringMatrix *matrix, const char *const *values, + int field_count) { + if (!matrix || !values || field_count <= 0) + return -1; + + char **row = calloc((size_t)field_count, sizeof(*row)); + if (!row) + return -1; + + for (int i = 0; i < field_count; i++) { + row[i] = strdup(values[i] ? values[i] : ""); + if (!row[i]) { + for (int j = 0; j < i; j++) + free(row[j]); + free(row); + return -1; + } + } + + if (matrix->count == matrix->capacity) { + int new_capacity = matrix->capacity ? matrix->capacity * 2 : 8; + char ***new_rows = malloc(sizeof(*new_rows) * (size_t)new_capacity); + int *new_counts = malloc(sizeof(*new_counts) * (size_t)new_capacity); + if (!new_rows || !new_counts) { + free(new_rows); + free(new_counts); + for (int i = 0; i < field_count; i++) + free(row[i]); + free(row); + return -1; + } + if (matrix->count > 0) { + memcpy(new_rows, matrix->rows, sizeof(*new_rows) * matrix->count); + memcpy(new_counts, matrix->field_counts, + sizeof(*new_counts) * matrix->count); + } + free(matrix->rows); + free(matrix->field_counts); + matrix->rows = new_rows; + matrix->field_counts = new_counts; + matrix->capacity = new_capacity; + } + + matrix->rows[matrix->count] = row; + matrix->field_counts[matrix->count] = field_count; + matrix->count++; + return 0; +} + +void string_matrix_free(StringMatrix *matrix) { + if (!matrix) + return; + free_string_matrix(matrix->rows, matrix->field_counts, matrix->count); + *matrix = (StringMatrix){0}; +} + +int string_matrix_build_pagination(int page, + char *(*href_builder)(int page, void *data), + void *data, StringMatrix *out_matrix) { enum { PAGER_WINDOW_SIZE = 5 }; + if (!out_matrix || !href_builder) + return -1; - *out_matrix = NULL; - *out_inner_counts = NULL; - int count = 0; - + string_matrix_init(out_matrix); int pager_start = page <= 3 ? 1 : page - 2; int pager_end = pager_start + PAGER_WINDOW_SIZE - 1; - if (page > 1) { - char *href = href_builder(page - 1, data); - count = add_link_to_collection(href, "←", "pagination-btn prev", out_matrix, - out_inner_counts, count); + for (int i = page > 1 ? pager_start - 1 : pager_start; i <= pager_end + 1; + i++) { + const char *label = NULL; + const char *class_name = "pagination-btn"; + char page_label[16]; + int target_page = i; + + if (page > 1 && i == pager_start - 1) { + label = "←"; + class_name = "pagination-btn prev"; + target_page = page - 1; + } else if (i == pager_end + 1) { + label = "→"; + class_name = "pagination-btn next"; + target_page = page + 1; + } else { + snprintf(page_label, sizeof(page_label), "%d", i); + label = page_label; + if (i == page) + class_name = "pagination-btn pagination-current"; + } + + char *href = href_builder(target_page, data); + const char *values[LINK_FIELD_COUNT] = {href, label, class_name}; + int result = string_matrix_append(out_matrix, values, LINK_FIELD_COUNT); free(href); + if (result != 0) { + string_matrix_free(out_matrix); + return -1; + } } - - for (int i = pager_start; i <= pager_end; i++) { - char label[16]; - snprintf(label, sizeof(label), "%d", i); - char *href = href_builder(i, data); - count = add_link_to_collection( - href, label, - i == page ? "pagination-btn pagination-current" : "pagination-btn", - out_matrix, out_inner_counts, count); - free(href); - } - - char *href = href_builder(page + 1, data); - count = add_link_to_collection(href, "→", "pagination-btn next", out_matrix, - out_inner_counts, count); - free(href); - - return count; + return 0; } diff --git a/src/Utility/Utility.h b/src/Utility/Utility.h index 1e1de09..0f66316 100644 --- a/src/Utility/Utility.h +++ b/src/Utility/Utility.h @@ -8,11 +8,18 @@ #endif #ifndef GIT_REMOTE -#define GIT_REMOTE "https://git.bwaaa.monster/omnisearch" +#define GIT_REMOTE "https://git.bwaaa.space/omnisearch" #endif #define LINK_FIELD_COUNT 3 +typedef struct { + char ***rows; + int *field_counts; + int count; + int capacity; +} StringMatrix; + int hex_to_int(char c); char *get_theme(const char *default_theme); void init_themes(const char *static_path); @@ -23,12 +30,17 @@ char *get_locale(const char *default_locale); int is_engine_id_enabled(const char *engine_id); int get_user_engines(char ***out_ids, int *out_count); int user_engines_contains(const char *engine_id, char **ids, int count); +char *get_default_search_engine(void); -int add_link_to_collection(const char *href, const char *label, - const char *class_name, char ****collection, - int **inner_counts, int current_count); +void free_string_matrix(char ***matrix, int *inner_counts, int row_count); -int build_pagination(int page, char *(*href_builder)(int page, void *data), - void *data, char ****out_matrix, int **out_inner_counts); +void string_matrix_init(StringMatrix *matrix); +int string_matrix_append(StringMatrix *matrix, const char *const *values, + int field_count); +void string_matrix_free(StringMatrix *matrix); + +int string_matrix_build_pagination(int page, + char *(*href_builder)(int page, void *data), + void *data, StringMatrix *out_matrix); #endif diff --git a/static/main.css b/static/main.css index 4039888..dae21e6 100644 --- a/static/main.css +++ b/static/main.css @@ -822,6 +822,38 @@ header .logo-link:hover { height:18px; accent-color:var(--accent); cursor:pointer; + flex-shrink:0; + margin:0; +} + +.settings-engine-list { + margin:0; +} + +.settings-engine-options { + display:grid; + gap:8px; + margin-inline:-3px; +} + +.settings-engine-option { + display:flex; + align-items:center; + justify-content:space-between; + gap:16px; + padding:12px 14px; + border:1px solid var(--border); + border-radius:8px; + background:var(--bg-main); + color:var(--text-primary); + font-size:0.95rem; + cursor:pointer; +} + +.settings-default-engine-field { + margin-top:12px; + padding-top:12px; + border-top:1px solid var(--border); } .settings-actions { @@ -888,6 +920,12 @@ header .logo-link:hover { align-items:stretch; gap:8px; } + .settings-engine-options { + margin-inline:0; + } + .settings-engine-option { + padding-inline:7px; + } .settings-actions { flex-direction:column; } @@ -936,5 +974,3 @@ header .logo-link:hover { flex-direction: column; } } - - diff --git a/templates/opensearch.xml b/templates/opensearch.xml index 14d3760..3c137f7 100644 --- a/templates/opensearch.xml +++ b/templates/opensearch.xml @@ -7,5 +7,5 @@ UTF-8 UTF-8 - {{domain}}/ + {{scheme}}://{{domain}}/ diff --git a/templates/settings.html b/templates/settings.html index 2f4ecb6..96ffdb4 100644 --- a/templates/settings.html +++ b/templates/settings.html @@ -76,16 +76,33 @@ {{if has_enabled_engines}} -
+

{{l("engines_label")}}

{{l("engines_desc")}}

- {{for eng in enabled_engines}} -
- - + +
+
+ {{for eng in enabled_engines}} + + {{endfor}} +
+
+ +
+ +
- {{endfor}}
{{endif}}
@@ -96,4 +113,4 @@
- \ No newline at end of file +