From b41d61502137624d47b5894be77e06b2084f8208 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 12 Feb 2017 00:11:05 +0100 Subject: machines/katara: searx anti-soundcloud patch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Soundcloud sometimes fails startup of searx, since it’s not actively used here it’s patched out until upstream resolves the issue. --- .../profpatsch/patches/searx-rm-soundcloud.patch | 15 ++ machines/profpatsch/patches/searx-secret-key.patch | 271 +++++++++++++++++++++ machines/profpatsch/patches/searx_secret_key.patch | 271 --------------------- machines/profpatsch/pkgs.nix | 5 +- 4 files changed, 290 insertions(+), 272 deletions(-) create mode 100644 machines/profpatsch/patches/searx-rm-soundcloud.patch create mode 100644 machines/profpatsch/patches/searx-secret-key.patch delete mode 100644 machines/profpatsch/patches/searx_secret_key.patch (limited to 'machines/profpatsch') diff --git a/machines/profpatsch/patches/searx-rm-soundcloud.patch b/machines/profpatsch/patches/searx-rm-soundcloud.patch new file mode 100644 index 00000000..35b427b9 --- /dev/null +++ b/machines/profpatsch/patches/searx-rm-soundcloud.patch @@ -0,0 +1,15 @@ +diff --git a/searx/settings.yml b/searx/settings.yml +index 053cb44..48a542b 100644 +--- a/searx/settings.yml ++++ b/searx/settings.yml +@@ -437,10 +437,6 @@ engines: + engine : scanr_structures + disabled : True + +- - name : soundcloud +- engine : soundcloud +- shortcut : sc +- + - name : stackoverflow + engine : stackoverflow + shortcut : st diff --git a/machines/profpatsch/patches/searx-secret-key.patch b/machines/profpatsch/patches/searx-secret-key.patch new file mode 100644 index 00000000..f7ff9fe5 --- /dev/null +++ b/machines/profpatsch/patches/searx-secret-key.patch @@ -0,0 +1,271 @@ +diff --git a/README.rst b/README.rst +index a0bb12f..9e32b53 100644 +--- a/README.rst ++++ b/README.rst +@@ -18,8 +18,7 @@ Installation + ``git clone https://github.com/asciimoo/searx.git && cd searx`` + - install dependencies: ``./manage.sh update_packages`` + - edit your +- `settings.yml `__ +- (set your ``secret_key``!) ++ `settings.yml ` + - run ``python searx/webapp.py`` to start the application + + For all the details, follow this `step by step +diff --git a/requirements.txt b/requirements.txt +index ac6a2c9..5cc663a 100644 +--- a/requirements.txt ++++ b/requirements.txt +@@ -1,11 +1,12 @@ +-certifi==2016.9.26 ++certifi + flask==0.12 + flask-babel==0.11.1 +-lxml==3.7.1 ++lxml==3.7.* + ndg-httpsclient==0.4.2 + pyasn1==0.1.9 + pygments==2.1.3 + pyopenssl==16.2.0 + python-dateutil==2.5.3 +-pyyaml==3.11 ++pyyaml==3.* + requests[socks]==2.12.4 ++pyxdg==0.25 +diff --git a/searx/settings.yml b/searx/settings.yml +index 8515326..e65e5e8 100644 +--- a/searx/settings.yml ++++ b/searx/settings.yml +@@ -10,7 +10,6 @@ search: + server: + port : 8888 + bind_address : "127.0.0.1" # address to listen on +- secret_key : "ultrasecretkey" # change this! + base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/" + image_proxy : False # Proxying image results through searx + http_protocol_version : "1.0" # 1.0 and 1.1 are supported +diff --git a/searx/settings_robot.yml b/searx/settings_robot.yml +index dbaf2fd..2c8f7cf 100644 +--- a/searx/settings_robot.yml ++++ b/searx/settings_robot.yml +@@ -10,7 +10,6 @@ search: + server: + port : 11111 + bind_address : 127.0.0.1 +- secret_key : "ultrasecretkey" # change this! + base_url : False + image_proxy : False + http_protocol_version : "1.0" +diff --git a/searx/utils.py b/searx/utils.py +index 35cb6f8..284087d 100644 +--- a/searx/utils.py ++++ b/searx/utils.py +@@ -2,6 +2,8 @@ import cStringIO + import csv + import os + import re ++import stat ++import xdg.BaseDirectory + + from babel.dates import format_date + from codecs import getincrementalencoder +@@ -300,3 +302,61 @@ def load_module(filename, module_dir): + module = load_source(modname, filepath) + module.name = modname + return module ++ ++ ++class SecretAppKeyError(IOError): ++ def __init__(self, reason, caught=None): ++ self.reason = reason ++ self.caught = caught ++ ++ def __str__(self): ++ err = "" ++ if self.caught != None: ++ err = '\n' + str(self.caught) ++ return repr(self.reason) + err ++ ++ ++_secret_app_key_length = 512 ++ ++ ++_secret_app_key_file_name = "secret_key" ++ ++ ++# tries to read the secret key from the xdg cache directory, ++# if none exists it creates one ++# If directory is given it has to be an existing, readable directory. ++def get_secret_app_key(directory=None): ++ ++ if directory is None: ++ try: ++ directory = xdg.BaseDirectory.save_cache_path("searx") ++ except OSError as e: ++ raise(SecretAppKeyError("could not get XDG_CACHE_DIR")) ++ ++ ++ # we save it as plaintext, assuming only the owner has access ++ f = os.path.join(directory, _secret_app_key_file_name) ++ ++ def saError(msg, e=None): ++ raise SecretAppKeyError("{} {}".format(f, msg), e) ++ ++ # if it exists, read it ++ if os.path.isfile(f): ++ try: ++ with open(f, 'r') as fh: ++ return fh.read() ++ except IOError as e: ++ saError("could not be read", e) ++ # if it doesn't, create it ++ else: ++ key = os.urandom(_secret_app_key_length) ++ try: ++ with open(f, 'w') as fh: ++ fh.write(key) ++ # the file should be readable/writable only by the owner ++ os.chmod(f, stat.S_IRUSR | stat.S_IWUSR) ++ return key ++ except IOError as e: ++ saError("could not be created", e) ++ except OSError as e: ++ saError("could not be chmodded to 600", e) +diff --git a/searx/webapp.py b/searx/webapp.py +index 929d9e2..31395af 100644 +--- a/searx/webapp.py ++++ b/searx/webapp.py +@@ -28,6 +28,7 @@ import hmac + import json + import os + import requests ++import xdg + + from searx import logger + logger = logger.getChild('webapp') +@@ -59,7 +60,7 @@ from searx.engines import ( + from searx.utils import ( + UnicodeWriter, highlight_content, html_to_text, get_themes, + get_static_files, get_result_templates, gen_useragent, dict_subset, +- prettify_url ++ prettify_url, get_secret_app_key + ) + from searx.version import VERSION_STRING + from searx.languages import language_codes +@@ -103,7 +104,11 @@ app = Flask( + + app.jinja_env.trim_blocks = True + app.jinja_env.lstrip_blocks = True +-app.secret_key = settings['server']['secret_key'] ++ ++# notify the user that the secret_key is no longer used ++if 'secret_key' in settings['server']: ++ logger.warning(' The "secret_key" config key is no longer used.') ++app.secret_key = get_secret_app_key() + + if not searx_debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true": + initialize_engines(settings['engines']) +@@ -265,7 +270,7 @@ def proxify(url): + url.encode('utf-8'), + hashlib.sha256).hexdigest() + +- return '{0}?{1}'.format(settings['result_proxy']['url'], ++ return '{0}?{1}'.format(settings['re sult_proxy']['url'], + urlencode(url_params)) + + +@@ -280,7 +285,7 @@ def image_proxify(url): + if settings.get('result_proxy'): + return proxify(url) + +- h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest() ++ h = hmac.new(app.secret_key, url.encode('utf-8'), hashlib.sha256).hexdigest() + + return '{0}?{1}'.format(url_for('image_proxy'), + urlencode(dict(url=url.encode('utf-8'), h=h))) +@@ -684,7 +689,7 @@ def image_proxy(): + if not url: + return '', 400 + +- h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest() ++ h = hmac.new(app.secret_key, url, hashlib.sha256).hexdigest() + + if h != request.args.get('h'): + return '', 400 +diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py +index 0448079..7c88445 100644 +--- a/tests/unit/test_utils.py ++++ b/tests/unit/test_utils.py +@@ -1,4 +1,8 @@ + # -*- coding: utf-8 -*- ++import os ++import tempfile ++import stat ++ + import mock + from searx.testing import SearxTestCase + from searx import utils +@@ -99,3 +103,63 @@ class TestUnicodeWriter(SearxTestCase): + rows = [1, 2, 3] + self.unicode_writer.writerows(rows) + self.assertEqual(self.unicode_writer.writerow.call_count, len(rows)) ++ ++ ++class TestSecretAppKey(SearxTestCase): ++ ++ def setUp(self): ++ self.getkey = utils.get_secret_app_key ++ self.fn = utils._secret_app_key_file_name ++ ++ def keyfile(self, dir_): ++ return os.path.join(dir_, self.fn) ++ ++ @staticmethod ++ def freshdir(): ++ return tempfile.mkdtemp() ++ ++ # generation of a key ++ def test_empty_dir(self): ++ dir_ = self.freshdir() ++ key = self.getkey(dir_) ++ self.assertNotEqual(key, "") ++ file_ = self.keyfile(dir_) ++ self.assertTrue(os.path.isfile(file_)) ++ mode = os.stat(file_).st_mode ++ # equal to read and write for user ++ self.assertEquals(mode & (stat.S_IRWXG | stat.S_IRWXU | stat.S_IRWXO), ++ (stat.S_IRUSR | stat.S_IWUSR)) ++ ++ # generation & successive read of the generated key ++ def test_existing_key(self): ++ dir_ = self.freshdir() ++ key = self.getkey(dir_) ++ key2 = self.getkey(dir_) ++ self.assertEquals(key, key2) ++ ++ def test_not_nice(self): ++ def touch(f, mode): ++ open(f, 'w').close() ++ os.chmod(f, mode) ++ ++ def raisesappkeyerror(dir_): ++ with self.assertRaises(utils.SecretAppKeyError): ++ self.getkey(dir_) ++ ++ # input dir doesn't exist ++ raisesappkeyerror("") ++ ++ # read-only ++ d1 = self.freshdir() ++ touch(self.keyfile(d1), 0) ++ raisesappkeyerror(d1) ++ ++ # dir ++ d2 = self.freshdir() ++ os.mkdir(self.keyfile(d2)) ++ raisesappkeyerror(d2) ++ ++ # non-writable dir ++ d3 = self.freshdir() ++ os.chmod(d3, stat.S_IRUSR) ++ raisesappkeyerror(d3) diff --git a/machines/profpatsch/patches/searx_secret_key.patch b/machines/profpatsch/patches/searx_secret_key.patch deleted file mode 100644 index f7ff9fe5..00000000 --- a/machines/profpatsch/patches/searx_secret_key.patch +++ /dev/null @@ -1,271 +0,0 @@ -diff --git a/README.rst b/README.rst -index a0bb12f..9e32b53 100644 ---- a/README.rst -+++ b/README.rst -@@ -18,8 +18,7 @@ Installation - ``git clone https://github.com/asciimoo/searx.git && cd searx`` - - install dependencies: ``./manage.sh update_packages`` - - edit your -- `settings.yml `__ -- (set your ``secret_key``!) -+ `settings.yml ` - - run ``python searx/webapp.py`` to start the application - - For all the details, follow this `step by step -diff --git a/requirements.txt b/requirements.txt -index ac6a2c9..5cc663a 100644 ---- a/requirements.txt -+++ b/requirements.txt -@@ -1,11 +1,12 @@ --certifi==2016.9.26 -+certifi - flask==0.12 - flask-babel==0.11.1 --lxml==3.7.1 -+lxml==3.7.* - ndg-httpsclient==0.4.2 - pyasn1==0.1.9 - pygments==2.1.3 - pyopenssl==16.2.0 - python-dateutil==2.5.3 --pyyaml==3.11 -+pyyaml==3.* - requests[socks]==2.12.4 -+pyxdg==0.25 -diff --git a/searx/settings.yml b/searx/settings.yml -index 8515326..e65e5e8 100644 ---- a/searx/settings.yml -+++ b/searx/settings.yml -@@ -10,7 +10,6 @@ search: - server: - port : 8888 - bind_address : "127.0.0.1" # address to listen on -- secret_key : "ultrasecretkey" # change this! - base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/" - image_proxy : False # Proxying image results through searx - http_protocol_version : "1.0" # 1.0 and 1.1 are supported -diff --git a/searx/settings_robot.yml b/searx/settings_robot.yml -index dbaf2fd..2c8f7cf 100644 ---- a/searx/settings_robot.yml -+++ b/searx/settings_robot.yml -@@ -10,7 +10,6 @@ search: - server: - port : 11111 - bind_address : 127.0.0.1 -- secret_key : "ultrasecretkey" # change this! - base_url : False - image_proxy : False - http_protocol_version : "1.0" -diff --git a/searx/utils.py b/searx/utils.py -index 35cb6f8..284087d 100644 ---- a/searx/utils.py -+++ b/searx/utils.py -@@ -2,6 +2,8 @@ import cStringIO - import csv - import os - import re -+import stat -+import xdg.BaseDirectory - - from babel.dates import format_date - from codecs import getincrementalencoder -@@ -300,3 +302,61 @@ def load_module(filename, module_dir): - module = load_source(modname, filepath) - module.name = modname - return module -+ -+ -+class SecretAppKeyError(IOError): -+ def __init__(self, reason, caught=None): -+ self.reason = reason -+ self.caught = caught -+ -+ def __str__(self): -+ err = "" -+ if self.caught != None: -+ err = '\n' + str(self.caught) -+ return repr(self.reason) + err -+ -+ -+_secret_app_key_length = 512 -+ -+ -+_secret_app_key_file_name = "secret_key" -+ -+ -+# tries to read the secret key from the xdg cache directory, -+# if none exists it creates one -+# If directory is given it has to be an existing, readable directory. -+def get_secret_app_key(directory=None): -+ -+ if directory is None: -+ try: -+ directory = xdg.BaseDirectory.save_cache_path("searx") -+ except OSError as e: -+ raise(SecretAppKeyError("could not get XDG_CACHE_DIR")) -+ -+ -+ # we save it as plaintext, assuming only the owner has access -+ f = os.path.join(directory, _secret_app_key_file_name) -+ -+ def saError(msg, e=None): -+ raise SecretAppKeyError("{} {}".format(f, msg), e) -+ -+ # if it exists, read it -+ if os.path.isfile(f): -+ try: -+ with open(f, 'r') as fh: -+ return fh.read() -+ except IOError as e: -+ saError("could not be read", e) -+ # if it doesn't, create it -+ else: -+ key = os.urandom(_secret_app_key_length) -+ try: -+ with open(f, 'w') as fh: -+ fh.write(key) -+ # the file should be readable/writable only by the owner -+ os.chmod(f, stat.S_IRUSR | stat.S_IWUSR) -+ return key -+ except IOError as e: -+ saError("could not be created", e) -+ except OSError as e: -+ saError("could not be chmodded to 600", e) -diff --git a/searx/webapp.py b/searx/webapp.py -index 929d9e2..31395af 100644 ---- a/searx/webapp.py -+++ b/searx/webapp.py -@@ -28,6 +28,7 @@ import hmac - import json - import os - import requests -+import xdg - - from searx import logger - logger = logger.getChild('webapp') -@@ -59,7 +60,7 @@ from searx.engines import ( - from searx.utils import ( - UnicodeWriter, highlight_content, html_to_text, get_themes, - get_static_files, get_result_templates, gen_useragent, dict_subset, -- prettify_url -+ prettify_url, get_secret_app_key - ) - from searx.version import VERSION_STRING - from searx.languages import language_codes -@@ -103,7 +104,11 @@ app = Flask( - - app.jinja_env.trim_blocks = True - app.jinja_env.lstrip_blocks = True --app.secret_key = settings['server']['secret_key'] -+ -+# notify the user that the secret_key is no longer used -+if 'secret_key' in settings['server']: -+ logger.warning(' The "secret_key" config key is no longer used.') -+app.secret_key = get_secret_app_key() - - if not searx_debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true": - initialize_engines(settings['engines']) -@@ -265,7 +270,7 @@ def proxify(url): - url.encode('utf-8'), - hashlib.sha256).hexdigest() - -- return '{0}?{1}'.format(settings['result_proxy']['url'], -+ return '{0}?{1}'.format(settings['re sult_proxy']['url'], - urlencode(url_params)) - - -@@ -280,7 +285,7 @@ def image_proxify(url): - if settings.get('result_proxy'): - return proxify(url) - -- h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest() -+ h = hmac.new(app.secret_key, url.encode('utf-8'), hashlib.sha256).hexdigest() - - return '{0}?{1}'.format(url_for('image_proxy'), - urlencode(dict(url=url.encode('utf-8'), h=h))) -@@ -684,7 +689,7 @@ def image_proxy(): - if not url: - return '', 400 - -- h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest() -+ h = hmac.new(app.secret_key, url, hashlib.sha256).hexdigest() - - if h != request.args.get('h'): - return '', 400 -diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py -index 0448079..7c88445 100644 ---- a/tests/unit/test_utils.py -+++ b/tests/unit/test_utils.py -@@ -1,4 +1,8 @@ - # -*- coding: utf-8 -*- -+import os -+import tempfile -+import stat -+ - import mock - from searx.testing import SearxTestCase - from searx import utils -@@ -99,3 +103,63 @@ class TestUnicodeWriter(SearxTestCase): - rows = [1, 2, 3] - self.unicode_writer.writerows(rows) - self.assertEqual(self.unicode_writer.writerow.call_count, len(rows)) -+ -+ -+class TestSecretAppKey(SearxTestCase): -+ -+ def setUp(self): -+ self.getkey = utils.get_secret_app_key -+ self.fn = utils._secret_app_key_file_name -+ -+ def keyfile(self, dir_): -+ return os.path.join(dir_, self.fn) -+ -+ @staticmethod -+ def freshdir(): -+ return tempfile.mkdtemp() -+ -+ # generation of a key -+ def test_empty_dir(self): -+ dir_ = self.freshdir() -+ key = self.getkey(dir_) -+ self.assertNotEqual(key, "") -+ file_ = self.keyfile(dir_) -+ self.assertTrue(os.path.isfile(file_)) -+ mode = os.stat(file_).st_mode -+ # equal to read and write for user -+ self.assertEquals(mode & (stat.S_IRWXG | stat.S_IRWXU | stat.S_IRWXO), -+ (stat.S_IRUSR | stat.S_IWUSR)) -+ -+ # generation & successive read of the generated key -+ def test_existing_key(self): -+ dir_ = self.freshdir() -+ key = self.getkey(dir_) -+ key2 = self.getkey(dir_) -+ self.assertEquals(key, key2) -+ -+ def test_not_nice(self): -+ def touch(f, mode): -+ open(f, 'w').close() -+ os.chmod(f, mode) -+ -+ def raisesappkeyerror(dir_): -+ with self.assertRaises(utils.SecretAppKeyError): -+ self.getkey(dir_) -+ -+ # input dir doesn't exist -+ raisesappkeyerror("") -+ -+ # read-only -+ d1 = self.freshdir() -+ touch(self.keyfile(d1), 0) -+ raisesappkeyerror(d1) -+ -+ # dir -+ d2 = self.freshdir() -+ os.mkdir(self.keyfile(d2)) -+ raisesappkeyerror(d2) -+ -+ # non-writable dir -+ d3 = self.freshdir() -+ os.chmod(d3, stat.S_IRUSR) -+ raisesappkeyerror(d3) diff --git a/machines/profpatsch/pkgs.nix b/machines/profpatsch/pkgs.nix index 45ce34f4..a220aed4 100644 --- a/machines/profpatsch/pkgs.nix +++ b/machines/profpatsch/pkgs.nix @@ -65,7 +65,10 @@ let xmpp-client = pkgs.callPackage (import ./xmpp-client.nix myLib.philip.home "irc/xmppOla.wtf") { inherit (pkgs) xmpp-client; }; searx = pkgs.pythonPackages.searx.overrideDerivation (old: { - patches = old.patches or [] ++ [ ./patches/searx_secret_key.patch ]; + patches = old.patches or [] ++ [ + ./patches/searx-secret-key.patch + ./patches/searx-rm-soundcloud.patch + ]; }); # A ghci with some sane default packages in scope, & hoogle -- cgit 1.4.1