Honor ArkOS4Clone HTTP proxy settings across sudo

CLAUDE.md

@@ -93,6 +93,7 @@ Known bugs are pinned as `xfail(strict=True)`, so a test fails once its bug is f
 - `util/`:
   - `callback.py`: the `Callback` progress interface, `CancelEvent`
   - `net.py`: fetch and download
+  - `proxy.py`: `setup_proxy()`, called by both `main()`s after sudo, adds EmulationStation's HTTP proxy (es_settings.cfg, ArkOS4Clone's Wi-Fi settings) to the environment for requests, unless a proxy is set already
   - `text.py`, `files.py`, `data.py`
 
   Every name is re-exported from `harbourmaster.util`.

PortMaster/pylibs/harbourmaster/cli.py

@@ -22,6 +22,7 @@
     add_list_unique,
     runtime_nicename,
     )
+from harbourmaster.util import setup_proxy
 
 
 LOG_FILE = harbourmaster.HM_TOOLS_DIR / "PortMaster" / "harbourmaster.txt"
@@ -623,6 +624,8 @@ def main(argv):
         elif config['force-colour']:
             console.do_color(True)
 
+        setup_proxy()
+
         ccb = ConsoleCallback(config)
         hm = HarbourMaster(config, temp_dir=temp_dir, callback=ccb)
 

PortMaster/pylibs/harbourmaster/util/__init__.py

@@ -36,6 +36,9 @@
     get_path_fs,
     make_temp_directory,
     )
+from .proxy import (
+    setup_proxy,
+    )
 from .data import (
     json_safe_loads,
     json_safe_load,
@@ -85,6 +88,7 @@
     'remove_dict_list',
     'remove_pm_signature',
     'runtime_nicename',
+    'setup_proxy',
     'timeit',
     'version_parse',
     'PORT_SORT_FUNCS',

PortMaster/pylibs/harbourmaster/util/proxy.py

@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: MIT
+#
+# HTTP proxy settings from EmulationStation (ArkOS4Clone's Wi-Fi settings).
+#
+# EmulationStation keeps the proxy in es_settings.cfg and uses it for its own requests,
+# without exporting http_proxy & co, and sudo drops those anyway. PortMaster reads it
+# itself after sudo, so the GUI, autoinstall and the CLI all use it (requests honours
+# the variables).
+
+import os
+import re
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from loguru import logger
+
+
+PROXY_KEYS = ('http_proxy', 'https_proxy', 'all_proxy', 'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY')
+NO_PROXY_KEYS = ('no_proxy', 'NO_PROXY')
+
+DEFAULT_NO_PROXY = "localhost,127.0.0.1,::1"
+
+
+def es_settings_files(environ):
+    """
+    Where es_settings.cfg may be. PortMaster runs as root through sudo, so the
+    frontend user's home comes first: SUDO_USER's, then HOME, then ArkOS' ark.
+    """
+    homes = []
+
+    sudo_user = environ.get('SUDO_USER')
+    if sudo_user:
+        try:
+            import pwd
+            homes.append(pwd.getpwnam(sudo_user).pw_dir)
+        except (ImportError, KeyError):
+            pass
+
+    if environ.get('HOME'):
+        homes.append(environ['HOME'])
+
+    homes.append('/home/ark')
+
+    files = []
+    for home in homes:
+        settings_file = Path(home) / '.emulationstation' / 'es_settings.cfg'
+        if settings_file not in files:
+            files.append(settings_file)
+
+    return files
+
+
+def _check_value(value):
+    ## These end up in the environment of everything PortMaster runs.
+    if any(ord(char) < 32 or ord(char) == 127 for char in value):
+        raise ValueError('control character in proxy setting')
+
+    return value
+
+
+def es_proxy_settings(settings_file):
+    """
+    {key: value} to add to the environment for es_settings.cfg's HTTP proxy, or {} when
+    it has none enabled. Raises ValueError (or ET.ParseError, OSError) for bad settings.
+    """
+    ## ES settings can have an XML declaration and several top-level nodes.
+    text = re.sub(r'<\?xml[^>]*\?>', '', Path(settings_file).read_text())
+    root = ET.fromstring('<settings>' + text + '</settings>')
+    settings = {node.get('name'): node.get('value', '') for node in root.iter()}
+
+    if settings.get('ProxyEnabled', '').strip().lower() != 'true':
+        return {}
+
+    ## SOCKS would need PySocks, which isn't bundled.
+    if settings.get('ProxyType', 'http').strip().lower() != 'http':
+        raise ValueError('only HTTP proxies are supported')
+
+    raw = _check_value(settings.get('ProxyHost', '').strip())
+    parsed = urlsplit(raw if '://' in raw else 'http://' + raw)
+    host = parsed.hostname
+    if (parsed.scheme != 'http' or not host or
+            any(char.isspace() for char in host) or
+            parsed.path not in ('', '/') or parsed.query or parsed.fragment):
+        raise ValueError('invalid HTTP proxy host')
+
+    ## (.port raises ValueError for a bad one, eg: "proxy:abc", even when ProxyPort is set.)
+    host_port = parsed.port
+    port = int(settings.get('ProxyPort', '').strip() or host_port or 80)
+    if not 1 <= port <= 65535:
+        raise ValueError('invalid HTTP proxy port')
+
+    if ':' in host:
+        host = '[' + host + ']'
+
+    credentials = parsed.netloc.rsplit('@', 1)[0] + '@' if '@' in parsed.netloc else ''
+    proxy = 'http://' + credentials + host + ':' + str(port)
+    no_proxy = _check_value(settings.get('ProxyNoProxy', DEFAULT_NO_PROXY).strip())
+
+    result = {key: proxy for key in ('http_proxy', 'https_proxy', 'HTTP_PROXY', 'HTTPS_PROXY')}
+    result.update({key: no_proxy for key in NO_PROXY_KEYS})
+    return result
+
+
+def setup_proxy(environ=None, settings_files=None):
+    """
+    Adds EmulationStation's HTTP proxy to `environ` (os.environ), unless a proxy is set
+    there already (even an empty one). A no_proxy already set is kept. Returns True when
+    a proxy was added. Never raises, and never logs the proxy (it can hold a password).
+    """
+    if environ is None:
+        environ = os.environ
+
+    if any(key in environ for key in PROXY_KEYS):
+        return False
+
+    if settings_files is None:
+        settings_files = es_settings_files(environ)
+
+    for settings_file in settings_files:
+        try:
+            if not settings_file.is_file():
+                continue
+
+            settings = es_proxy_settings(settings_file)
+
+        except (OSError, ValueError, ET.ParseError) as err:
+            logger.warning(f"Unable to use the proxy settings in {settings_file} ({type(err).__name__}).")
+            return False
+
+        if not settings:
+            ## The frontend's settings say no proxy.
+            return False
+
+        explicit_no_proxy = next((environ[key] for key in NO_PROXY_KEYS if key in environ), None)
+        for key, value in settings.items():
+            if key in NO_PROXY_KEYS and explicit_no_proxy is not None:
+                value = explicit_no_proxy
+
+            environ[key] = value
+
+        logger.info(f"Using the HTTP proxy from {settings_file}.")
+        return True
+
+    return False

PortMaster/pylibs/pugwash/main.py

@@ -8,6 +8,7 @@
 from loguru import logger
 from pugwash.scenes import BlankScene
 from harbourmaster import HarbourMaster, make_temp_directory
+from harbourmaster.util import setup_proxy
 from gettext import gettext as _
 from pugwash import PORTMASTER_DEBUG
 from pugwash.app import PortMasterGUI
@@ -93,6 +94,9 @@ def main(argv):
         elif config['force-colour']:
             console.do_color(True)
 
+        ## Before anything goes online: the update check, sources, autoinstall's runtimes.
+        setup_proxy()
+
         if not get_ip_address():
             config['offline'] = True
             logger.warning("No internet connection found, running in offline mode.")

tests/conftest.py

@@ -431,3 +431,10 @@ def reset_hardware_cache():
     harbourmaster.hardware._CACHED_DICT = None
     yield
     harbourmaster.hardware._CACHED_DICT = None
+
+
+@pytest.fixture(autouse=True)
+def no_real_proxy_settings(monkeypatch):
+    """main() reads EmulationStation's proxy settings: never the real ones (devices run the suite too)."""
+    from harbourmaster.util import proxy
+    monkeypatch.setattr(proxy, "es_settings_files", lambda environ: [])

tests/test_cli_commands.py

@@ -486,3 +486,13 @@ def test_main_no_log_without_a_log_file(run_main, monkeypatch, capsys):
     assert run_main("--offline", "--no-log", "ports")[0] == 0
     assert "No ports found." in capsys.readouterr().out
     assert None not in removed
+
+
+def test_main_sets_up_the_proxy(run_main, monkeypatch):
+    calls = []
+    monkeypatch.setattr(cli, "setup_proxy", lambda: calls.append(True))
+
+    result, config = run_main("--offline", "nothing")
+
+    assert result == 0
+    assert calls == [True]

tests/test_main.py

@@ -203,3 +203,13 @@ def test_frontend_refresh(on_device, es_name, changed, expected):
 
     calls = [call if isinstance(call, str) else call[0] for call in FakeGUI.made[-1].calls]
     assert calls == ['run'] + expected + ['quit']
+
+
+def test_proxy_is_set_up_first(main, monkeypatch):
+    ## Before the update check, HarbourMaster's sources and autoinstall go online.
+    calls = []
+    monkeypatch.setattr(main, "setup_proxy", lambda: calls.append(len(main.hms)))
+
+    assert run(main) == 0
+    assert run(main, "fifo_control", "pipe", "done") == 0
+    assert calls == [0, 1]