diff --git a/CLAUDE.md b/CLAUDE.md index cfd2f97b..0a1351b3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,6 +93,7 @@ Known bugs are pinned as `xfail(strict=True)`, so a test fails once its bug is f - `util/`: - `callback.py`: the `Callback` progress interface, `CancelEvent` - `net.py`: fetch and download + - `proxy.py`: `setup_proxy()`, called by both `main()`s after sudo, adds EmulationStation's HTTP proxy (es_settings.cfg, ArkOS4Clone's Wi-Fi settings) to the environment for requests, unless a proxy is set already - `text.py`, `files.py`, `data.py` Every name is re-exported from `harbourmaster.util`. diff --git a/PortMaster/pylibs/harbourmaster/cli.py b/PortMaster/pylibs/harbourmaster/cli.py index 921116cf..2bd23e50 100644 --- a/PortMaster/pylibs/harbourmaster/cli.py +++ b/PortMaster/pylibs/harbourmaster/cli.py @@ -22,6 +22,7 @@ add_list_unique, runtime_nicename, ) +from harbourmaster.util import setup_proxy LOG_FILE = harbourmaster.HM_TOOLS_DIR / "PortMaster" / "harbourmaster.txt" @@ -623,6 +624,8 @@ def main(argv): elif config['force-colour']: console.do_color(True) + setup_proxy() + ccb = ConsoleCallback(config) hm = HarbourMaster(config, temp_dir=temp_dir, callback=ccb) diff --git a/PortMaster/pylibs/harbourmaster/util/__init__.py b/PortMaster/pylibs/harbourmaster/util/__init__.py index df900aa9..87dd6dd0 100644 --- a/PortMaster/pylibs/harbourmaster/util/__init__.py +++ b/PortMaster/pylibs/harbourmaster/util/__init__.py @@ -36,6 +36,9 @@ get_path_fs, make_temp_directory, ) +from .proxy import ( + setup_proxy, + ) from .data import ( json_safe_loads, json_safe_load, @@ -85,6 +88,7 @@ 'remove_dict_list', 'remove_pm_signature', 'runtime_nicename', + 'setup_proxy', 'timeit', 'version_parse', 'PORT_SORT_FUNCS', diff --git a/PortMaster/pylibs/harbourmaster/util/proxy.py b/PortMaster/pylibs/harbourmaster/util/proxy.py new file mode 100644 index 00000000..f92d0026 --- /dev/null +++ b/PortMaster/pylibs/harbourmaster/util/proxy.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: MIT +# +# HTTP proxy settings from EmulationStation (ArkOS4Clone's Wi-Fi settings). +# +# EmulationStation keeps the proxy in es_settings.cfg and uses it for its own requests, +# without exporting http_proxy & co, and sudo drops those anyway. PortMaster reads it +# itself after sudo, so the GUI, autoinstall and the CLI all use it (requests honours +# the variables). + +import os +import re +import xml.etree.ElementTree as ET +from pathlib import Path +from urllib.parse import urlsplit + +from loguru import logger + + +PROXY_KEYS = ('http_proxy', 'https_proxy', 'all_proxy', 'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY') +NO_PROXY_KEYS = ('no_proxy', 'NO_PROXY') + +DEFAULT_NO_PROXY = "localhost,127.0.0.1,::1" + + +def es_settings_files(environ): + """ + Where es_settings.cfg may be. PortMaster runs as root through sudo, so the + frontend user's home comes first: SUDO_USER's, then HOME, then ArkOS' ark. + """ + homes = [] + + sudo_user = environ.get('SUDO_USER') + if sudo_user: + try: + import pwd + homes.append(pwd.getpwnam(sudo_user).pw_dir) + except (ImportError, KeyError): + pass + + if environ.get('HOME'): + homes.append(environ['HOME']) + + homes.append('/home/ark') + + files = [] + for home in homes: + settings_file = Path(home) / '.emulationstation' / 'es_settings.cfg' + if settings_file not in files: + files.append(settings_file) + + return files + + +def _check_value(value): + ## These end up in the environment of everything PortMaster runs. + if any(ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError('control character in proxy setting') + + return value + + +def es_proxy_settings(settings_file): + """ + {key: value} to add to the environment for es_settings.cfg's HTTP proxy, or {} when + it has none enabled. Raises ValueError (or ET.ParseError, OSError) for bad settings. + """ + ## ES settings can have an XML declaration and several top-level nodes. + text = re.sub(r'<\?xml[^>]*\?>', '', Path(settings_file).read_text()) + root = ET.fromstring('' + text + '') + settings = {node.get('name'): node.get('value', '') for node in root.iter()} + + if settings.get('ProxyEnabled', '').strip().lower() != 'true': + return {} + + ## SOCKS would need PySocks, which isn't bundled. + if settings.get('ProxyType', 'http').strip().lower() != 'http': + raise ValueError('only HTTP proxies are supported') + + raw = _check_value(settings.get('ProxyHost', '').strip()) + parsed = urlsplit(raw if '://' in raw else 'http://' + raw) + host = parsed.hostname + if (parsed.scheme != 'http' or not host or + any(char.isspace() for char in host) or + parsed.path not in ('', '/') or parsed.query or parsed.fragment): + raise ValueError('invalid HTTP proxy host') + + ## (.port raises ValueError for a bad one, eg: "proxy:abc", even when ProxyPort is set.) + host_port = parsed.port + port = int(settings.get('ProxyPort', '').strip() or host_port or 80) + if not 1 <= port <= 65535: + raise ValueError('invalid HTTP proxy port') + + if ':' in host: + host = '[' + host + ']' + + credentials = parsed.netloc.rsplit('@', 1)[0] + '@' if '@' in parsed.netloc else '' + proxy = 'http://' + credentials + host + ':' + str(port) + no_proxy = _check_value(settings.get('ProxyNoProxy', DEFAULT_NO_PROXY).strip()) + + result = {key: proxy for key in ('http_proxy', 'https_proxy', 'HTTP_PROXY', 'HTTPS_PROXY')} + result.update({key: no_proxy for key in NO_PROXY_KEYS}) + return result + + +def setup_proxy(environ=None, settings_files=None): + """ + Adds EmulationStation's HTTP proxy to `environ` (os.environ), unless a proxy is set + there already (even an empty one). A no_proxy already set is kept. Returns True when + a proxy was added. Never raises, and never logs the proxy (it can hold a password). + """ + if environ is None: + environ = os.environ + + if any(key in environ for key in PROXY_KEYS): + return False + + if settings_files is None: + settings_files = es_settings_files(environ) + + for settings_file in settings_files: + try: + if not settings_file.is_file(): + continue + + settings = es_proxy_settings(settings_file) + + except (OSError, ValueError, ET.ParseError) as err: + logger.warning(f"Unable to use the proxy settings in {settings_file} ({type(err).__name__}).") + return False + + if not settings: + ## The frontend's settings say no proxy. + return False + + explicit_no_proxy = next((environ[key] for key in NO_PROXY_KEYS if key in environ), None) + for key, value in settings.items(): + if key in NO_PROXY_KEYS and explicit_no_proxy is not None: + value = explicit_no_proxy + + environ[key] = value + + logger.info(f"Using the HTTP proxy from {settings_file}.") + return True + + return False diff --git a/PortMaster/pylibs/pugwash/main.py b/PortMaster/pylibs/pugwash/main.py index 80b4eceb..74bcbc18 100644 --- a/PortMaster/pylibs/pugwash/main.py +++ b/PortMaster/pylibs/pugwash/main.py @@ -8,6 +8,7 @@ from loguru import logger from pugwash.scenes import BlankScene from harbourmaster import HarbourMaster, make_temp_directory +from harbourmaster.util import setup_proxy from gettext import gettext as _ from pugwash import PORTMASTER_DEBUG from pugwash.app import PortMasterGUI @@ -93,6 +94,9 @@ def main(argv): elif config['force-colour']: console.do_color(True) + ## Before anything goes online: the update check, sources, autoinstall's runtimes. + setup_proxy() + if not get_ip_address(): config['offline'] = True logger.warning("No internet connection found, running in offline mode.") diff --git a/tests/conftest.py b/tests/conftest.py index 09ac4cbe..3bb690e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -431,3 +431,10 @@ def reset_hardware_cache(): harbourmaster.hardware._CACHED_DICT = None yield harbourmaster.hardware._CACHED_DICT = None + + +@pytest.fixture(autouse=True) +def no_real_proxy_settings(monkeypatch): + """main() reads EmulationStation's proxy settings: never the real ones (devices run the suite too).""" + from harbourmaster.util import proxy + monkeypatch.setattr(proxy, "es_settings_files", lambda environ: []) diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index baa55b9e..6624969d 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -486,3 +486,13 @@ def test_main_no_log_without_a_log_file(run_main, monkeypatch, capsys): assert run_main("--offline", "--no-log", "ports")[0] == 0 assert "No ports found." in capsys.readouterr().out assert None not in removed + + +def test_main_sets_up_the_proxy(run_main, monkeypatch): + calls = [] + monkeypatch.setattr(cli, "setup_proxy", lambda: calls.append(True)) + + result, config = run_main("--offline", "nothing") + + assert result == 0 + assert calls == [True] diff --git a/tests/test_main.py b/tests/test_main.py index 8dc58b2a..a616fd4b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -203,3 +203,13 @@ def test_frontend_refresh(on_device, es_name, changed, expected): calls = [call if isinstance(call, str) else call[0] for call in FakeGUI.made[-1].calls] assert calls == ['run'] + expected + ['quit'] + + +def test_proxy_is_set_up_first(main, monkeypatch): + ## Before the update check, HarbourMaster's sources and autoinstall go online. + calls = [] + monkeypatch.setattr(main, "setup_proxy", lambda: calls.append(len(main.hms))) + + assert run(main) == 0 + assert run(main, "fifo_control", "pipe", "done") == 0 + assert calls == [0, 1] diff --git a/tests/test_proxy.py b/tests/test_proxy.py new file mode 100644 index 00000000..a2946a58 --- /dev/null +++ b/tests/test_proxy.py @@ -0,0 +1,248 @@ +# SPDX-License-Identifier: MIT +# +# HTTP proxy settings from EmulationStation's es_settings.cfg (ArkOS4Clone's Wi-Fi +# settings), set up by pugwash and harbourmaster after sudo. + +from pathlib import Path +from types import SimpleNamespace +from xml.sax.saxutils import quoteattr + +import pytest + +from harbourmaster.util import proxy + + +PROXY = 'http://192.0.2.10:7890' + +## conftest's no_real_proxy_settings swaps it out during tests. +es_settings_files = proxy.es_settings_files + + +def capture_log(level): + from loguru import logger + + messages = [] + handler_id = logger.add(lambda message: messages.append(message.record["message"]), level=level) + return messages, handler_id + + +@pytest.fixture +def log_warnings(): + from loguru import logger + + messages, handler_id = capture_log("WARNING") + yield messages + logger.remove(handler_id) + + +@pytest.fixture +def log_messages(): + from loguru import logger + + messages, handler_id = capture_log("DEBUG") + yield messages + logger.remove(handler_id) + + +@pytest.fixture +def settings(tmp_path): + settings_file = tmp_path / "home" / ".emulationstation" / "es_settings.cfg" + settings_file.parent.mkdir(parents=True) + + def write(**overrides): + values = dict(ProxyEnabled='true', ProxyType='http', ProxyHost='192.0.2.10', ProxyPort='7890') + values.update(overrides) + settings_file.write_text('\n' + ''.join( + f'\n' + for key, value in values.items() if value is not None)) + return settings_file + + write.path = settings_file + return write + + +def setup_with(environ, *files): + return proxy.setup_proxy(environ, settings_files=list(files)) + + +################################################################################ +## es_settings.cfg +def test_http_proxy(settings): + env = proxy.es_proxy_settings(settings()) + + for key in ('http_proxy', 'https_proxy', 'HTTP_PROXY', 'HTTPS_PROXY'): + assert env[key] == PROXY + assert env['no_proxy'] == env['NO_PROXY'] == 'localhost,127.0.0.1,::1' + + +def test_settings_in_a_config_root(settings): + ## Some ES builds write one root, others several top-level nodes. + settings.path.write_text( + '\n\n' + '\t\n' + '\t\n' + '\t\n' + '\n') + + assert proxy.es_proxy_settings(settings.path)['http_proxy'] == 'http://proxy.lan:3128' + + +@pytest.mark.parametrize('enabled', ['false', '', None]) +def test_proxy_disabled(settings, enabled): + assert proxy.es_proxy_settings(settings(ProxyEnabled=enabled)) == {} + + +@pytest.mark.parametrize('host, expected', [ + ('http://proxy.example:8000', 'http://proxy.example:7890'), + ('[::1]', 'http://[::1]:7890'), + ('http://user:pass%40word@proxy.example', 'http://user:pass%40word@proxy.example:7890'), + ]) +def test_proxy_hosts(settings, host, expected): + assert proxy.es_proxy_settings(settings(ProxyHost=host))['http_proxy'] == expected + + +def test_port_from_the_host(settings): + assert proxy.es_proxy_settings(settings(ProxyHost='http://proxy.example:8080', ProxyPort=''))['http_proxy'] == 'http://proxy.example:8080' + + +def test_default_port(settings): + assert proxy.es_proxy_settings(settings(ProxyPort=''))['http_proxy'] == 'http://192.0.2.10:80' + + +def test_no_proxy_from_the_settings(settings): + assert proxy.es_proxy_settings(settings(ProxyNoProxy='localhost,.lan'))['no_proxy'] == 'localhost,.lan' + + +@pytest.mark.parametrize('overrides', [ + dict(ProxyPort='70000'), dict(ProxyPort='0'), dict(ProxyPort='not-a-port'), + dict(ProxyHost=''), dict(ProxyHost='http://proxy.example/path'), dict(ProxyHost='proxy.example:abc'), + dict(ProxyHost='https://proxy.example'), dict(ProxyType='socks5'), + dict(ProxyHost='proxy\nLD_PRELOAD=bad.so'), dict(ProxyNoProxy='localhost\nLD_PRELOAD=bad.so'), + ], ids=lambda overrides: ','.join(overrides)) +def test_bad_settings(settings, overrides): + with pytest.raises(ValueError): + proxy.es_proxy_settings(settings(**overrides)) + + +################################################################################ +## setup_proxy +def test_setup_adds_the_proxy(settings): + environ = {} + + assert setup_with(environ, settings()) is True + assert environ['https_proxy'] == PROXY + assert environ['no_proxy'] == 'localhost,127.0.0.1,::1' + + +@pytest.mark.parametrize('key', proxy.PROXY_KEYS) +@pytest.mark.parametrize('value', ['http://other:8080', '']) +def test_a_proxy_already_set_wins(settings, key, value): + ## Even an empty one: that's someone saying no proxy. The settings aren't even read. + settings.path.write_text('