LibreY

- privacy respecting meta search engine
git clone git://git.acid.vegas/LibreY.git
Log | Files | Refs | Archive | README | LICENSE

commit 0fd041a35372a0b5aed12be2be2097bddb66982b
parent 2c36d28a2f957830e42cb782e96ae7e8cb171e78
Author: davidovski <david@davidovski.xyz>
Date: Fri, 25 Aug 2023 16:23:53 +0100

Add timeouts for failing instances

Diffstat:
Mconfig.php.example | 3+++
Mengines/librex/fallback.php | 37++++++++++++++++++++++++++-----------
Amisc/cooldowns.php | 20++++++++++++++++++++
Mmisc/search_engine.php | 1+

4 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/config.php.example b/config.php.example
@@ -20,6 +20,9 @@
         // This may greatly increase the time it takes to get a result and in some cases results in 504 errors
         "instance_fallback" => false,
 
+        // how long in minutes to put google/other instances on cooldown if they aren't responding
+        "request_cooldown" => 5,
+
         /*
             Preset privacy friendly frontends for users, these can be overwritten by users in the settings
             e.g.: Preset the invidious instance URL: "instance_url" => "https://yewtu.be",
diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php
@@ -15,25 +15,38 @@
             $response = json_decode($response, true);
             if (!$response)
                 return array();
-            
+
             return array_values($response);
         }
     }
 
+    function load_instances($cooldowns) {
+        $instances_json = json_decode(file_get_contents("instances.json"), true);
+
+        if (empty($instances_json["instances"]))
+            return array();
+
+        $instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet'])));
+        $instances = array_filter($instances, fn($n) => !has_cooldown($n, $cooldowns));
+        shuffle($instances);
+        return $instances;
+    }
 
     function get_librex_results($opts) {
         if (!$opts->do_fallback)
             return array();
 
-        $instances_json = json_decode(file_get_contents("instances.json"), true);
+        require "misc/cooldowns.php";
 
-        if (empty($instances_json["instances"]))
-            return array();
+        $cooldowns = load_cooldowns();
+        error_log("loaded" . count($cooldowns));
+        error_log(print_r($cooldowns,true));
 
-        // TODO pick instances which aren't on cooldown
+        echo "<pre>";
+        print_r($cooldowns);
+        echo "</pre>";
 
-        $instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet'])));
-        shuffle($instances);
+        $instances = load_instances($cooldowns);
 
         $results = array();
         $tries = 0;
@@ -43,6 +56,7 @@
 
             $instance = array_pop($instances);
 
+            error_log($instance .  "--- " . parse_url($instance)["host"]);
             if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"])
                 continue;
 
@@ -52,12 +66,13 @@
             if (count($results) > 1)
                 return $results;
 
-        } while ( !empty($instances));
+            // on fail then do this
+            $timeout = ($opts->request_cooldown ?? "1") * 60;
+            $cooldowns = set_cooldown($instance, $timeout, $cooldowns);
 
-        if (empty($instances))
-            return array();
+        } while (!empty($instances));
 
-        return array_values($results);
+        return array();
     }
 
 ?>
diff --git a/misc/cooldowns.php b/misc/cooldowns.php
@@ -0,0 +1,20 @@
+<?php
+    function load_cooldowns() {
+        return apcu_exists("cooldowns") ? apcu_fetch("cooldowns") : array();
+    }
+
+    function save_cooldowns($cooldowns) {
+        apcu_store("cooldowns", $cooldowns);
+    }
+
+    function set_cooldown($instance, $timeout, $cooldowns) {
+        $cooldowns[$instance] = time() + $timeout;
+        error_log("cooldown on instnace $instance for $timeout seconds");
+        save_cooldowns($cooldowns);
+        return $cooldowns;
+    }
+
+    function has_cooldown($instance, $cooldowns) {
+        return ($cooldowns[$instance] ?? 0) > time();
+    }
+?>
diff --git a/misc/search_engine.php b/misc/search_engine.php
@@ -57,6 +57,7 @@
         foreach (array_keys($opts->frontends ?? array()) as $frontend) {
             $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"];
         }
+
         return $opts;
     }