diff --git a/README.md b/README.md
@@ -31,7 +31,7 @@ You can access the full list of LibreX and LibreY instances on one of the follow
### About LibreY
LibreY gives you text results from DuckDuckGo or Google, images from Qwant, and torrents from i.e. Ahmia and popular torrent sites without spying on you.
-<br>LibreY doesn't save any type of data about the user, there are no logs (except NGINX logs if the host sets them), no caches.
+<br>LibreY doesn't save **any** type of data about the user, there are no logs (except NGINX logs if the host sets them).
### LibreY compared to other metasearch engines
diff --git a/config.php.example b/config.php.example
@@ -17,8 +17,11 @@
"disable_hidden_service_search" => false,
// Fallback to another librex instance if google search fails
- // This may greatly increase the time it takes to get a result and in some cases results in 504 errors
- "instance_fallback" => false,
+ // This may greatly increase the time it takes to get a result, if a direct search is not possible
+ "instance_fallback" => true,
+
+ // how long in minutes to put google/other instances on cooldown if they aren't responding
+ "request_cooldown" => 25,
/*
Preset privacy friendly frontends for users, these can be overwritten by users in the settings
@@ -148,7 +151,8 @@
CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP,
CURLOPT_MAXREDIRS => 5,
CURLOPT_TIMEOUT => 3,
- CURLOPT_VERBOSE => false
+ CURLOPT_VERBOSE => false,
+ CURLOPT_FOLLOWLOCATION => true
)
);
?>
diff --git a/docker/attributes.sh b/docker/attributes.sh
@@ -23,7 +23,7 @@ export CONFIG_GOOGLE_DOMAIN="${CONFIG_GOOGLE_DOMAIN:-"com"}"
export CONFIG_GOOGLE_LANGUAGE_SITE="${CONFIG_GOOGLE_LANGUAGE_SITE:-"en"}"
export CONFIG_GOOGLE_LANGUAGE_RESULTS="${CONFIG_GOOGLE_LANGUAGE_RESULTS:-"en"}"
export CONFIG_GOOGLE_NUMBER_OF_RESULTS="${CONFIG_GOOGLE_NUMBER_OF_RESULTS:-"10"}"
-export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}
+export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}"
export CONFIG_INVIDIOUS_INSTANCE="${CONFIG_INVIDIOUS_INSTANCE:-"invidious.snopyta.org"}"
export CONFIG_HIDDEN_SERVICE_SEARCH=${CONFIG_HIDDEN_SERVICE_SEARCH:-false}
export CONFIG_DISABLE_BITTORRENT_SEARCH=${CONFIG_DISABLE_BITTORRENT_SEARCH:-false}
diff --git a/docker/php/php.dockerfile b/docker/php/php.dockerfile
@@ -56,7 +56,7 @@ ENV CURLOPT_VERBOSE=true
# Install PHP-FPM using Alpine's package manager, apk
# Configure PHP-FPM to listen on a Unix socket instead of a TCP port, which is more secure and efficient
-RUN apk add php8 php8-fpm php8-dom php8-curl php8-json --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\
+RUN apk add php8 php8-fpm php8-dom php8-curl php8-json php8-apcu --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\
sed -i 's/^\s*listen = 127.0.0.1:9000/listen = \/run\/php8\/php-fpm8.sock/' ${WWW_CONFIG} &&\
sed -i 's/^\s*;\s*listen.owner = nobody/listen.owner = nginx/' ${WWW_CONFIG} &&\
sed -i 's/^\s*;\s*listen.group = nobody/listen.group = nginx/' ${WWW_CONFIG} &&\
diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php
@@ -15,25 +15,29 @@
$response = json_decode($response, true);
if (!$response)
return array();
-
+
return array_values($response);
}
}
-
- function get_librex_results($opts) {
- if (!$opts->do_fallback)
- return array();
-
+ function load_instances($cooldowns) {
$instances_json = json_decode(file_get_contents("instances.json"), true);
if (empty($instances_json["instances"]))
return array();
- // TODO pick instances which aren't on cooldown
-
$instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet'])));
+ $instances = array_filter($instances, fn($n) => !has_cooldown($n, $cooldowns));
shuffle($instances);
+ return $instances;
+ }
+
+ function get_librex_results($opts) {
+ if (!$opts->do_fallback)
+ return array();
+
+ $cooldowns = $opts->cooldowns;
+ $instances = load_instances($cooldowns);
$results = array();
$tries = 0;
@@ -52,12 +56,13 @@
if (count($results) > 1)
return $results;
- } while ( !empty($instances));
+ // on fail then do this
+ $timeout = ($opts->request_cooldown ?? "1") * 60;
+ $cooldowns = set_cooldown($instance, $timeout, $cooldowns);
- if (empty($instances))
- return array();
+ } while (!empty($instances));
- return array_values($results);
+ return array();
}
?>
diff --git a/engines/text/google.php b/engines/text/google.php
@@ -1,7 +1,6 @@
<?php
class GoogleRequest extends EngineRequest {
- function get_request_url()
- {
+ public function get_request_url() {
$query_encoded = str_replace("%22", "\"", urlencode($this->query));
$results = array();
diff --git a/engines/text/text.php b/engines/text/text.php
@@ -5,19 +5,23 @@
$this->page = $opts->page;
$this->opts = $opts;
- $engine = $opts->preferred_engines["text"] ?? "google";
+ $this->engine = $opts->preferred_engines["text"] ?? "google";
$query_parts = explode(" ", $this->query);
$last_word_query = end($query_parts);
if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!")
check_ddg_bang($this->query, $opts);
- if ($engine == "google") {
+ if (has_cooldown($this->engine, $this->opts->cooldowns))
+ return;
+
+ if ($this->engine == "google") {
+
require "engines/text/google.php";
$this->engine_request = new GoogleRequest($opts, $mh);
}
- if ($engine == "duckduckgo") {
+ if ($this->engine == "duckduckgo") {
require "engines/text/duckduckgo.php";
$this->engine_request = new DuckDuckGoRequest($opts, $mh);
}
@@ -27,6 +31,9 @@
}
public function get_results() {
+ if (!$this->engine_request)
+ return array();
+
$results = $this->engine_request->get_results();
if ($this->special_request) {
@@ -36,6 +43,9 @@
$results = array_merge(array($special_result), $results);
}
+ if (count($results) <= 1)
+ set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns);
+
return $results;
}
diff --git a/misc/cooldowns.php b/misc/cooldowns.php
@@ -0,0 +1,22 @@
+<?php
+ function load_cooldowns() {
+ if (function_exists("apcu_fetch"))
+ return apcu_exists("cooldowns") ? apcu_fetch("cooldowns") : array();
+ return array();
+ }
+
+ function save_cooldowns($cooldowns) {
+ if (function_exists("apcu_store"))
+ apcu_store("cooldowns", $cooldowns);
+ }
+
+ function set_cooldown($instance, $timeout, $cooldowns) {
+ $cooldowns[$instance] = time() + $timeout;
+ save_cooldowns($cooldowns);
+ return $cooldowns;
+ }
+
+ function has_cooldown($instance, $cooldowns) {
+ return ($cooldowns[$instance] ?? 0) > time();
+ }
+?>
diff --git a/misc/search_engine.php b/misc/search_engine.php
@@ -33,6 +33,8 @@
function load_opts() {
$opts = require "config.php";
+ $opts->request_cooldown ??= 25;
+
$opts->query = trim($_REQUEST["q"] ?? "");
$opts->type = (int) ($_REQUEST["t"] ?? 0);
$opts->page = (int) ($_REQUEST["p"] ?? 0);
@@ -45,7 +47,7 @@
$opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]);
- $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? ""));
+ $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language));
$opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0;
if (!$opts->instance_fallback) {
@@ -57,6 +59,7 @@
foreach (array_keys($opts->frontends ?? array()) as $frontend) {
$opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"];
}
+
return $opts;
}
@@ -110,6 +113,9 @@
}
function fetch_search_results($opts, $do_print) {
+ require "misc/cooldowns.php";
+ $opts->cooldowns = load_cooldowns();
+
$start_time = microtime(true);
$mh = curl_multi_init();
$search_category = init_search($opts, $mh);
| | | | | | | | |
