diff --git a/.codacy.yml b/.codacy.yml deleted file mode 100644 index a283ad6d..00000000 --- a/.codacy.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -exclude_paths: - - 'tests/**' - - 'scripts/**' diff --git a/.dockerignore b/.dockerignore index 99903e8d..5c12c135 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,5 @@ dist build .github .gitmodules +.venv +*.egg-info diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md deleted file mode 100644 index 5df55bfe..00000000 --- a/.github/copilot-instructions.md +++ /dev/null @@ -1,20 +0,0 @@ -- always keep the changes minimal and purposeful -- focus on fixing the exact problem or implementing the exact feature -- keep the code simple, do not write defensive code -- do not describe your changes in details after you made changes, focus on writing code -- do not generate any documentation, the code should be self-explanatory -- do not generate any in-line comments -- for the new files, always add a license header, same format as in the existing files -- no commented out code -- no console logs in production code -- no unused imports -- no redundant code - move repeated logic into helper functions -- use type hints to specify the expected types of function arguments and return values - -- check `pyproject.toml` for formatting rules -- always lint changes using `uv run ruff check` -- tests should be placed in `tests/` directory, follow the existing structure and code style -- always use `uv` to run all commands in the repo (e.g., `uv run ruff`, `uv run pytest`, etc.) -- for running tests, export environment variables in the terminal before running the tests: `. ./scripts/export_env.sh` - -- additional external context is located in context directory \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index b07e9efe..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 -updates: -- package-ecosystem: pip - directory: "/" - schedule: - interval: weekly - day: saturday - open-pull-requests-limit: 10 diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 00000000..887a0a4c --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,19 @@ +name: Audit +on: + push: + schedule: + - cron: "0 6 * * 1" + +permissions: + contents: read + +jobs: + audit: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.3 + with: + version: "0.13.6" + - run: uv export --no-dev --no-emit-workspace --no-hashes -o requirements.txt + - run: uvx pip-audit==2.10.1 -r requirements.txt --no-deps --disable-pip -s osv diff --git a/.github/workflows/issue_check.yml b/.github/workflows/issue_check.yml deleted file mode 100644 index ee5ec6b2..00000000 --- a/.github/workflows/issue_check.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Get linked issues -on: - pull_request: - types: [ edited, synchronize, opened, reopened ] - -jobs: - check-linked-issues: - name: Check if pull request has linked issues - runs-on: ubuntu-latest - steps: - - name: Get issues - id: get-issues - uses: mondeja/pr-linked-issues-action@v2 - env: - GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - name: PR has not linked issues - if: join(steps.get-issues.outputs.issues) == '' - run: - exit 1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6239d0bd..49548574 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,64 +16,39 @@ jobs: runs-on: ubuntu-24.04 permissions: contents: write + env: + BRANCH: ${{ github.base_ref }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: submodules: true - - name: Set up Python 3.14 - uses: actions/setup-python@v5 - with: - python-version: '3.14' - - name: Calculate version - id: versioning run: | - bash ./scripts/set_versions_ga.sh - - - name: Determine prerelease status - id: release_info - run: | - if [[ "${{ env.BRANCH }}" == "stable" ]]; then - echo "prerelease=false" >> $GITHUB_OUTPUT - else - echo "prerelease=true" >> $GITHUB_OUTPUT - fi + export VERSION=$(grep '^version = ' pyproject.toml | cut -d'"' -f2) + echo "VERSION=$(bash ./helper-scripts/calculate_version.sh)" >> "$GITHUB_ENV" - name: Build binaries - id: build run: | - mkdir -p ${{ github.workspace }}/dist + mkdir -p ./dist docker build . -t node-cli-builder - - docker run --rm -v ${{ github.workspace }}/dist:/app/dist node-cli-builder \ - bash scripts/build.sh ${{ env.VERSION }} ${{ env.BRANCH }} skale - - docker run --rm -v ${{ github.workspace }}/dist:/app/dist node-cli-builder \ - bash scripts/build.sh ${{ env.VERSION }} ${{ env.BRANCH }} fair - - echo "dist_path=${{ github.workspace }}/dist" >> $GITHUB_OUTPUT - + docker run --rm -v "$PWD/dist":/app/dist node-cli-builder bash scripts/build.sh "$VERSION" "$BRANCH" skale + docker run --rm -v "$PWD/dist":/app/dist node-cli-builder bash scripts/build.sh "$VERSION" "$BRANCH" fair + - name: Generate checksums + working-directory: dist run: | - cd ${{ steps.build.outputs.dist_path }} for file in skale-*; do sha512sum "$file" > "$file.sha512" done - echo "Checksums generated:" - ls -l *.sha512 - name: Create GitHub release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: tag_name: ${{ env.VERSION }} target_commitish: ${{ github.sha }} name: Release ${{ env.VERSION }} - draft: false - prerelease: ${{ steps.release_info.outputs.prerelease }} + prerelease: ${{ github.base_ref != 'stable' }} generate_release_notes: true - files: | - ${{ steps.build.outputs.dist_path }}/skale-* - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + files: dist/skale-* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c9935f4..525406af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,48 +1,27 @@ name: Test on: [push] +permissions: + contents: read + jobs: test: runs-on: ubuntu-24.04 - strategy: - matrix: - python-version: ['3.14'] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: true - - name: Checkout submodules - run: git submodule update --init - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install uv - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - - name: Cache uv - uses: actions/cache@v4 + uses: astral-sh/setup-uv@v10.3.3 with: - path: ~/.cache/uv - key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} - - - name: Install ubuntu dependencies - run: | - sudo apt-get update - sudo apt-get install iptables nftables python3-nftables + version: "0.13.6" - name: Install python dependencies - run: | - uv venv - uv pip install -e ".[dev]" - site_packages=$(uv run python -c 'import site; print(site.getsitepackages()[0])') - echo /usr/lib/python3/dist-packages > "$site_packages/ubuntu-system-packages.pth" + run: uv sync --all-extras - name: Generate info - run: bash ./scripts/generate_info.sh 1.0.0 my-branch skale + run: bash ./scripts/generate_info.sh test test skale - name: Check with ruff run: | @@ -54,33 +33,22 @@ jobs: - name: Build binary - skale run: | mkdir -p ./dist - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test skale - docker rm -f $(docker ps -aq) + docker run --rm -v "$PWD/dist":/app/dist node-cli-builder bash scripts/build.sh test test skale - name: Check build - skale - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + run: sudo ./dist/skale-test-Linux-x86_64 - name: Build binary - fair run: | - mkdir -p ./dist - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test fair - docker rm -f $(docker ps -aq) + docker run --rm -v "$PWD/dist":/app/dist node-cli-builder bash scripts/build.sh test test fair - name: Check build - fair - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-fair - - - name: Run prepare test build - run: | - uv run bash scripts/build.sh test test skale + run: sudo ./dist/skale-test-Linux-x86_64-fair - - name: Run redis + - name: Run nftables tests run: | - bash ./helper-scripts/redis/run.sh + bash ./scripts/run_nftables_test.sh - name: Run tests run: | uv run bash ./scripts/run_tests.sh - - - name: Run nftables tests - run: | - uv run scripts/run_nftables_test.sh diff --git a/.gitignore b/.gitignore index 65232943..fb1f5325 100644 --- a/.gitignore +++ b/.gitignore @@ -1,128 +1,23 @@ -# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python build/ -develop-eggs/ dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ *.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -#*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments .env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ - +*.log .idea/ .vscode/ - +.zed .DS_Store +uv.lock -# info.py file generated automatically during package build node_cli/cli/info.py meta.json - -block_device.txt -sgx_server_url.txt resource_allocation.json conf.json test-env - nginx.conf tests/.skale/node_data/docker.json tests/.skale/node_data/node_options.json tests/.skale/config/nginx.conf.j2 - -.zed -uv.lock \ No newline at end of file diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..6324d401 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/Dockerfile b/Dockerfile index 835de85b..f594735d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,14 @@ FROM ubuntu:24.04 AS builder -ARG PYTHON_VERSION=3.13 - -ENV DEBIAN_FRONTEND=noninteractive ENV UV_PYTHON_INSTALL_DIR=/opt/python -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv - -RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates && \ - rm -rf /var/lib/apt/lists/* +COPY --from=ghcr.io/astral-sh/uv:0.13.6 /uv /usr/local/bin/uv WORKDIR /app COPY pyproject.toml ./ -RUN uv python install "$PYTHON_VERSION" && \ - uv venv --python "$PYTHON_VERSION" /opt/venv && \ +RUN uv venv --python 3.14.10 /opt/venv && \ uv pip install --python /opt/venv --no-cache ".[dev]" FROM ubuntu:24.04 @@ -26,14 +18,9 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y --no-install-recommends \ binutils \ - ca-certificates \ git \ - iptables \ - kmod \ - nftables \ patchelf \ - python3-nftables \ - wget && \ + python3-nftables && \ rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -41,13 +28,10 @@ WORKDIR /app COPY --from=builder /opt/python /opt/python COPY --from=builder /opt/venv /opt/venv -RUN printf '%s\n' '/usr/lib/python3/dist-packages' \ - > /opt/venv/lib/python3.13/site-packages/ubuntu-system-packages.pth +RUN ln -s /usr/lib/python3/dist-packages/nftables /opt/venv/lib/python3.14/site-packages/ COPY . . RUN patchelf --remove-needed libSegFault.so datafiles/skaled-ssl-test ENV PATH="/opt/venv/bin:$PATH" -ENV PYTHONPATH="/app" -ENV COLUMNS=80 diff --git a/README.md b/README.md index 9b629489..a8c20bc5 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,15 @@ ![Test](https://github.com/skalenetwork/node-cli/workflows/Test/badge.svg) [![Discord](https://img.shields.io/discord/534485763354787851.svg)](https://discord.gg/vvUtWJB) -SKALE Node CLI, part of the SKALE suite of validator tools, is the command line interface to setup, register and maintain your SKALE node. It comes in three distinct build types: Standard (for validator nodes), Passive (for dedicated sChain synchronization), and Fair. +SKALE Node CLI, part of the SKALE suite of validator tools, is the command line interface to setup, register and maintain your SKALE node. It comes in two binaries: `skale` (validator and passive SKALE nodes) and `fair` (Fair network nodes). ## Table of Contents 1. [Installation](#installation) - 1. [Standard Node Binary](#standard-node-binary) - 2. [Passive Node Binary](#passive-node-binary) - 3. [Fair Node Binary](#fair-node-binary) - 4. [Permissions and Testing](#permissions-and-testing) -2. [Standard Node Usage (`skale` - Normal Build)](#standard-node-usage-skale---normal-build) + 1. [SKALE Node Binary](#skale-node-binary) + 2. [Fair Node Binary](#fair-node-binary) + 3. [Permissions and Testing](#permissions-and-testing) +2. [Standard Node Usage (`skale`)](#standard-node-usage-skale) 1. [Top level commands (Standard)](#top-level-commands-standard) 2. [Node commands (Standard)](#node-commands-standard) 3. [Wallet commands (Standard)](#wallet-commands-standard) @@ -22,9 +21,8 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 6. [SSL commands (Standard)](#ssl-commands-standard) 7. [SGX commands (Standard)](#sgx-commands-standard) 8. [Logs commands (Standard)](#logs-commands-standard) -3. [Passive Node Usage (`skale` - Passive Build)](#passive-node-usage-skale---passive-build) - 1. [Top level commands (Passive)](#top-level-commands-passive) - 2. [Passive node commands](#passive-node-commands) +3. [Passive Node Usage (`skale passive-node`)](#passive-node-usage-skale-passive-node) + 1. [Passive node commands](#passive-node-commands) 4. [Fair Node Usage (`fair`)](#fair-node-usage-fair) 1. [Top level commands (Fair)](#top-level-commands-fair) 2. [Fair Boot commands](#fair-boot-commands) @@ -45,7 +43,7 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line ### Prerequisites -Ensure that the following packages are installed: **docker**, **docker-compose** (1.27.4+) +Ensure that **docker** with the **compose** plugin and **git** are installed. ### SKALE Node Binary @@ -72,7 +70,7 @@ sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/downl Apply executable permissions to the downloaded binary (adjust name accordingly): ```shell -# For Standard or Passive binary +# For SKALE binary sudo chmod +x /usr/local/bin/skale # For Fair binary @@ -82,7 +80,7 @@ sudo chmod +x /usr/local/bin/fair Test the installation: ```shell -# Standard or Passive build +# SKALE binary skale --help # Fair build @@ -91,7 +89,7 @@ fair --help *** -## Standard Node Usage (`skale` - Normal Build) +## Standard Node Usage (`skale`) Commands available in the **standard `skale` binary** for managing nodes. @@ -579,32 +577,9 @@ Options: *** -## Passive Node Usage (`skale` - Passive Build) +## Passive Node Usage (`skale passive-node`) -Commands available in the **passive `skale` binary** for managing dedicated Passive nodes. -Note that this binary contains a **different set of commands** compared to the standard build. - -### Top level commands (Passive) - -#### Info (Passive) - -Print build info for the `skale` (passive) binary. - -```shell -skale info -``` - -#### Version (Passive) - -Print version number for the `skale` (passive) binary. - -```shell -skale version -``` - -Options: - -* `--short` - prints version only, without additional text. +Commands of the `skale` binary for managing dedicated Passive nodes. ### Passive node commands @@ -1278,7 +1253,7 @@ Arguments: ### Passive Fair Node commands -> Prefix: `fair passive-node` (passive Fair build) +> Prefix: `fair passive-node` Commands for operating a passive Fair node (sync/indexer/archive). @@ -1385,49 +1360,27 @@ Exit codes conventions for SKALE CLI tools #### Dependencies -* Python 3.11 +* [uv](https://docs.astral.sh/uv/) 0.13.6+ (installs Python 3.14 from `.python-version`) * Git #### Clone the repository -Clone with HTTPS: - -```shell -git clone https://github.com/skalenetwork/node-cli.git -``` - -Or with SSH: - ```shell -git clone git@github.com:skalenetwork/node-cli.git -``` - -#### Create and source virtual environment - -```shell -python3.11 -m venv venv -source venv/bin/activate +git clone --recurse-submodules https://github.com/skalenetwork/node-cli.git ``` #### Install development dependencies ```shell -pip install -e ".[dev]" +uv sync --all-extras ``` #### Generate info.py locally -Specify the build type (`normal`, `passive`, or `fair`): +Specify the build type (`skale` or `fair`): ```shell -# Example for Standard build -./scripts/generate_info.sh 1.0.0 my-branch normal - -# Example for Passive build -./scripts/generate_info.sh 1.0.0 my-branch passive - -# Example for Fair build -./scripts/generate_info.sh 1.0.0 my-branch fair +bash scripts/generate_info.sh 1.0.0 my-branch skale ``` #### Add linting git hook @@ -1436,11 +1389,9 @@ In file `.git/hooks/pre-commit` add: ```shell #!/bin/sh -./venv/bin/ruff check . +uv run ruff check ``` -> **Note:** This hook assumes your virtual environment is named 'venv' and is located at the root of the repository. - Make the hook executable: ```shell diff --git a/lvmpy b/lvmpy index 694e2533..049dd460 160000 --- a/lvmpy +++ b/lvmpy @@ -1 +1 @@ -Subproject commit 694e25337b51fe97aaadd3ea45e88759e838184e +Subproject commit 049dd460df464ea6fde3a5ef9fffd96d842217aa diff --git a/main.spec b/main.spec index 8b7da407..3e103a21 100644 --- a/main.spec +++ b/main.spec @@ -1,43 +1,12 @@ -# -*- mode: python -*- - -import os - -block_cipher = None - a = Analysis( ['node_cli/main.py'], pathex=[SPECPATH], datas=[ - (os.path.join(SPECPATH, "text.yml"), "data"), - (os.path.join(SPECPATH, "datafiles/skaled-ssl-test"), "data/datafiles") + ('text.yml', 'data'), + ('datafiles/skaled-ssl-test', 'data/datafiles'), ], - hiddenimports=[], - hookspath=[], - runtime_hooks=[], - excludes=[], - win_no_prefer_redirects=False, - win_private_assemblies=False, - cipher=block_cipher, - noarchive=False ) -pyz = PYZ( - a.pure, a.zipped_data, - cipher=block_cipher -) +pyz = PYZ(a.pure) -exe = EXE( - pyz, - a.scripts, - a.binaries, - a.zipfiles, - a.datas, - [], - name='main', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - runtime_tmpdir=None, - console=True -) +exe = EXE(pyz, a.scripts, a.binaries, a.datas, name='main') diff --git a/node_cli/cli/exit.py b/node_cli/cli/exit.py index c7f57db3..fcdb2b6a 100644 --- a/node_cli/cli/exit.py +++ b/node_cli/cli/exit.py @@ -71,8 +71,3 @@ def status(format): print_exit_status(exit_status) else: error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) - - -@node_exit.command('finalize', help='Finalize exit process') -def finalize(): - pass diff --git a/node_cli/cli/fair_boot.py b/node_cli/cli/fair_boot.py index d5ff6039..c7e6591e 100644 --- a/node_cli/cli/fair_boot.py +++ b/node_cli/cli/fair_boot.py @@ -23,7 +23,7 @@ from node_cli.core.node import get_node_info, get_node_signature from node_cli.core.node import register_node as register from node_cli.fair.boot import init, update -from node_cli.utils.helper import IP_TYPE, abort_if_false, error_exit, streamed_cmd +from node_cli.utils.helper import IP_TYPE, abort_if_false, streamed_cmd @click.group() @@ -69,10 +69,7 @@ def register_boot(name, ip, port, domain): @boot.command('signature', help='Get fair node signature for a validator ID (during Boot Phase).') @click.argument('validator_id') def signature_boot(validator_id): - res = get_node_signature(validator_id) - if isinstance(res, dict) and 'error' in res: - error_exit(f'Error getting signature: {res.get("message", res)}') - print(f'Signature: {res}') + print(f'Signature: {get_node_signature(validator_id)}') @boot.command('update', help='Update Fair node from .env file') diff --git a/node_cli/cli/sgx.py b/node_cli/cli/sgx.py index e1635617..441d8082 100644 --- a/node_cli/cli/sgx.py +++ b/node_cli/cli/sgx.py @@ -20,7 +20,6 @@ import json import click -from terminaltables import SingleTable from node_cli.configs.sgx import SGX_SIGN_TIMEOUT from node_cli.core.sgx import ( @@ -33,6 +32,7 @@ from node_cli.utils.decorators import check_inited, check_user from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import abort_if_false, error_exit +from node_cli.utils.print_formatters import Formatter from node_cli.utils.settings import get_sgx_url from node_cli.utils.texts import safe_load_texts @@ -62,7 +62,7 @@ def options(json_format: bool) -> None: if json_format: print(json.dumps(payload)) else: - rows = [['SGX option', 'Value']] + rows = [] for group, values in payload.items(): entries = ( [(f'{group}.{key}', value) for key, value in values.items()] @@ -73,7 +73,7 @@ def options(json_format: bool) -> None: [key, value if isinstance(value, str) else json.dumps(value)] for key, value in entries ) - print(SingleTable(rows).table) + print(Formatter().table(['SGX option', 'Value'], rows)) @sgx.command('cert-status', help=TEXTS['status']['help']) @@ -143,7 +143,6 @@ def _configured_sgx_url() -> str: def print_certificate_status(info: dict) -> None: present = info['present'] rows = [ - ['SGX client certificate', ''], ['Directory', info['directory']], ['Private key', _presence(present['key'])], ['Signing request', _presence(present['csr'])], @@ -163,7 +162,7 @@ def print_certificate_status(info: dict) -> None: ) if info.get('server_version'): rows.append(['SGX server', f'accepted the certificate, version {info["server_version"]}']) - print(SingleTable(rows).table) + print(Formatter().table(['SGX client certificate', ''], rows)) for notice in _notices(info): print(notice) diff --git a/node_cli/cli/ssl.py b/node_cli/cli/ssl.py index c22ac226..97f54584 100644 --- a/node_cli/cli/ssl.py +++ b/node_cli/cli/ssl.py @@ -18,17 +18,16 @@ # along with this program. If not, see . import click -from terminaltables import SingleTable from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import error_exit +from node_cli.utils.print_formatters import Formatter from node_cli.utils.texts import safe_load_texts from node_cli.configs.ssl import DEFAULT_SSL_CHECK_PORT, SSL_CERT_FILEPATH, SSL_KEY_FILEPATH from node_cli.core.ssl import check_cert, upload_cert, cert_status TEXTS = safe_load_texts() -BLUEPRINT_NAME = 'ssl' @click.group() @@ -48,13 +47,13 @@ def status(): if payload.get('is_empty'): print(TEXTS['ssl']['no_cert']) else: - table_data = [ - ['Issued to', payload['issued_to']], - ['Expiration date', payload['expiration_date']], - ] - table = SingleTable(table_data) print('SSL certificates status:') - print(table.table) + print( + Formatter().table( + ['Issued to', 'Expiration date'], + [[payload['issued_to'], payload['expiration_date']]], + ) + ) else: error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index d65e4613..c20acb7a 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -56,7 +56,6 @@ CONTAINER_CONFIG_TMP_PATH = os.path.join(SKALE_TMP_DIR, 'config') CONTRACTS_PATH = os.path.join(SKALE_DIR, 'contracts_info') REPORTS_PATH = os.path.join(SKALE_DIR, 'reports') -BACKUP_CONTRACTS_PATH = os.path.join(SKALE_DIR, '.old_contracts_info') SKALE_RUN_DIR = '/var/run/skale' COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') @@ -81,7 +80,6 @@ SRC_FILEBEAT_CONFIG_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'filebeat.yml') FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml') -DOCKER_LVMPY_PATH = os.path.join(SKALE_DIR, 'docker-lvmpy') DOCKER_LVMPY_BIN_PATH = '/usr/local/bin/skale' LVMPY_CMD = f'{DOCKER_LVMPY_BIN_PATH} lvmpy' @@ -91,19 +89,13 @@ LVMPY_CRON_LOG_PATH = '/var/log/docker-lvmpy/cron.log' LVMPY_CRON_SCHEDULE_MINUTES = 3 -LVMPY_LOG_DIR = '/var/log/docker-lvmpy' - -IPTABLES_DIR = '/etc/iptables/' -IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4') -DEFAULT_SSH_PORT = 22 - DOCKER_CONFIG_FILEPATH = '/etc/docker/daemon.json' HIDE_STREAM_LOG = os.getenv('HIDE_STREAM_LOG') def _get_env(): try: - sys._MEIPASS # type: ignore + sys._MEIPASS except AttributeError: return 'dev' return 'prod' @@ -116,7 +108,7 @@ def _get_env(): PARDIR = os.path.join(CURRENT_FILE_LOCATION, os.pardir) PROJECT_DIR = os.path.join(PARDIR, os.pardir) else: - PARDIR = os.path.join(sys._MEIPASS, 'data') # type: ignore + PARDIR = os.path.join(sys._MEIPASS, 'data') PROJECT_DIR = PARDIR TEXT_FILE = os.path.join(PROJECT_DIR, 'text.yml') @@ -130,7 +122,6 @@ def _get_env(): ADMIN_PORT = 3007 ADMIN_HOST = 'localhost' -DEFAULT_URL_SCHEME = 'http://' DEFAULT_NODE_BASE_PORT = 10000 @@ -143,7 +134,6 @@ def _get_env(): META_FILEPATH = os.path.join(NODE_DATA_PATH, 'meta.json') SKALE_NODE_REPO_URL = 'https://github.com/skalenetwork/skale-node.git' -DOCKER_LVMPY_REPO_URL = 'https://github.com/skalenetwork/docker-lvmpy.git' DOCKER_DAEMON_CONFIG_PATH = '/etc/docker/daemon.json' DOCKER_DAEMON_HOSTS = ('fd://', 'unix:///var/run/skale/docker.sock') @@ -156,8 +146,6 @@ def _get_env(): AUTOLOAD_KERNEL_MODULES_PATH = '/etc/modules' BTRFS_KERNEL_MODULE = 'btrfs' -TELEGRAF_TEMPLATE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'telegraf.conf.j2') -TELEGRAF_CONFIG_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'telegraf.conf') NODE_DOCKER_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'docker.json') NFTABLES_CHAIN_FOLDER_PATH = '/etc/nft.conf.d/skale/chains' diff --git a/node_cli/configs/cli_logger.py b/node_cli/configs/cli_logger.py index f0e4dff5..4dc21c91 100644 --- a/node_cli/configs/cli_logger.py +++ b/node_cli/configs/cli_logger.py @@ -20,9 +20,8 @@ import os from node_cli.configs import SKALE_DIR -LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' STREAM_LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s' -FILE_LOG_FORMAT = '[%(asctime)s %(levelname)s] %(name)s:%(lineno)d - %(threadName)s - %(message)s' # noqa +FILE_LOG_FORMAT = '[%(asctime)s %(levelname)s] %(name)s:%(lineno)d - %(threadName)s - %(message)s' LOG_FILE_SIZE_MB = 300 LOG_FILE_SIZE_BYTES = LOG_FILE_SIZE_MB * 1000000 diff --git a/node_cli/configs/resource_allocation.py b/node_cli/configs/resource_allocation.py index 8a51729d..c69dee5b 100644 --- a/node_cli/configs/resource_allocation.py +++ b/node_cli/configs/resource_allocation.py @@ -29,14 +29,8 @@ TIMES = 1 TIMEOUT = 1 MEMORY_FACTOR = 0.8 -DISK_FACTOR = 0.95 MAX_CPU_SHARES = 1024 -VOLUME_CHUNK = 512 * SMALL_DIVIDER - RESOURCE_ALLOCATION_FILENAME = 'resource_allocation.json' RESOURCE_ALLOCATION_FILEPATH = os.path.join(NODE_DATA_PATH, RESOURCE_ALLOCATION_FILENAME) - -SGX_SERVER_URL_FILENAME = 'sgx_server_url.txt' -SGX_SERVER_URL_FILEPATH = os.path.join(NODE_DATA_PATH, SGX_SERVER_URL_FILENAME) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 3cda6bee..1423b367 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -30,16 +30,14 @@ 'maintenance-on', 'maintenance-off', 'signature', - 'send-tg-notification', 'exit/start', 'exit/status', 'set-domain-name', 'update-safe', ], - 'health': ['containers', 'schains'], - 'info': ['sgx', 'sgx-options'], - 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'repair', 'get'], - 'ssl': ['status', 'upload'], + 'health': ['schains'], + 'info': ['containers', 'sgx', 'sgx-options'], + 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'get'], 'wallet': ['info', 'send-eth'], 'fair-node': ['info', 'register', 'set-domain-name', 'change-ip', 'exit'], 'fair-chain': ['record', 'checks'], @@ -75,12 +73,3 @@ def get_route(blueprint, method, api_version=CURRENT_API_VERSION, check=True): if check and not route_exists(blueprint, method, api_version): raise RouteNotFoundException(route) return route - - -def get_all_available_routes(api_version=CURRENT_API_VERSION): - routes = ROUTES[api_version] - return [ - get_route(blueprint, method, api_version) - for blueprint in routes - for method in routes[blueprint] - ] diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index ecc613fb..cf623c63 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -43,8 +43,8 @@ Union, ) -import docker # type: ignore -import psutil # type: ignore +import docker +import psutil from debian import debian_support from packaging.version import parse as version_parse @@ -60,9 +60,8 @@ from node_cli.core.host import is_ufw_ipv6_chain_exists, is_ufw_ipv6_option_enabled from node_cli.core.resources import get_disk_size from node_cli.core.static_config import get_static_params -from node_cli.utils.docker_utils import NodeType from node_cli.utils.helper import run_cmd, safe_mkdir -from node_cli.utils.node_type import NodeMode +from node_cli.utils.node_type import NodeMode, NodeType logger = logging.getLogger(__name__) @@ -83,7 +82,7 @@ def check_quietly(check: Func, *args, **kwargs) -> CheckResult: try: return check(*args, **kwargs) except Exception as err: - logger.exception('%s check errored') + logger.exception('%s check errored', check.__name__) return CheckResult(name=check.__name__, status='error', info=repr(err)) @@ -174,14 +173,6 @@ def get_checks(self, check_type: CheckType = CheckType.ALL) -> FuncList: ) return [functools.partial(m[1], self) for m in methods] - def preinstall_check(self) -> ResultList: - check_methods = self.get_checks(check_type=CheckType.PREINSTALL) - return [cm() for cm in check_methods] - - def postinstall_check(self) -> ResultList: - check_methods = self.get_checks(check_type=CheckType.POSTINSTALL) - return [cm() for cm in check_methods] - def check(self) -> ResultList: check_methods = self.get_checks() return [cm() for cm in check_methods] @@ -263,12 +254,11 @@ def disk(self) -> CheckResult: def network(self) -> CheckResult: name = 'network' try: - socket.setdefaulttimeout(self.network_timeout) - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect( - (CLOUDFLARE_DNS_HOST, CLOUDFLARE_DNS_HOST_PORT) - ) - return self._ok(name=name) - except socket.error as err: + with socket.create_connection( + (CLOUDFLARE_DNS_HOST, CLOUDFLARE_DNS_HOST_PORT), timeout=self.network_timeout + ): + return self._ok(name=name) + except OSError as err: info = f'Network checking returned error: {err}' return self._failed(name=name, info=info) @@ -277,8 +267,7 @@ class PackageChecker(BaseChecker): def __init__(self, requirements: Dict) -> None: super().__init__(requirements=requirements) - def _check_apt_package(self, package_name: str, version: str | None = None) -> CheckResult: - # TODO: check versions + def _check_apt_package(self, package_name: str) -> CheckResult: dpkg_cmd_result = run_cmd(['dpkg', '-s', package_name], check_code=False) output = dpkg_cmd_result.stdout.decode('utf-8').strip() if dpkg_cmd_result.returncode != 0: @@ -350,7 +339,7 @@ def docker_engine(self) -> CheckResult: if not version_info: return self._failed(name=name, info='Docker api request failed. Is docker installed?') logger.debug('Docker version info %s', version_info) - actual_version = self.docker_client.version()['Version'] + actual_version = version_info['Version'] expected_version = self.requirements['docker-engine'] info = {'expected_version': expected_version, 'actual_version': actual_version} if version_parse(actual_version) < version_parse(expected_version): @@ -394,7 +383,7 @@ def docker_compose(self) -> CheckResult: actual_version = output.split(',')[0].split()[-1].strip() expected_version = self.requirements['docker-compose'] - info = f'Expected docker compose version {expected_version}, actual {actual_version}' # noqa + info = f'Expected docker compose version {expected_version}, actual {actual_version}' if version_parse(actual_version) < version_parse(expected_version): return self._failed(name=name, info=info) else: @@ -426,7 +415,7 @@ def _check_docker_hosts_option(self, config: Dict) -> Tuple: sactual, sexpected = set(actual_value), set(DOCKER_DAEMON_HOSTS) if sactual & sexpected != sexpected: missing = sorted(sexpected - sactual) - info = f'Docker daemon hosts is misconfigured. Missing hosts: {missing}' # noqa + info = f'Docker daemon hosts is misconfigured. Missing hosts: {missing}' return False, info else: info = 'Hosts is properly configured' diff --git a/node_cli/core/docker_config.py b/node_cli/core/docker_config.py index 67cf3d86..97b027bd 100644 --- a/node_cli/core/docker_config.py +++ b/node_cli/core/docker_config.py @@ -7,7 +7,7 @@ import shutil import time import typing -from typing import Optional, Tuple +from typing import Optional from node_cli.configs import ( DOCKER_DAEMON_CONFIG_PATH, @@ -18,7 +18,6 @@ NODE_DOCKER_CONFIG_PATH, SKALE_RUN_DIR, ) -from node_cli.utils.docker_utils import docker_client, get_containers from node_cli.utils.helper import run_cmd logger = logging.getLogger(__name__) @@ -43,9 +42,6 @@ def get_content(filename: Path) -> Optional[str]: return f.read() -gdclient = docker_client() - - class DockerConfigError(Exception): pass @@ -54,14 +50,6 @@ class OverridenConfigExsitsError(DockerConfigError): pass -class NoSocketFileError(DockerConfigError): - pass - - -class ContainersExistError(DockerConfigError): - pass - - class SocketInitTimeoutError(DockerConfigError): pass @@ -156,13 +144,6 @@ def ensure_run_dir(run_dir: Path = SKALE_RUN_DIR) -> DockerConfigResult: return DockerConfigResult.UNCHANGED -def assert_no_containers(ignore: Tuple[str] = ()): - containers = [c.name for c in get_containers() if c.name not in ignore] - if len(containers) > 0: - logger.fatal('%s containers exist', ' '.join(containers)) - raise ContainersExistError(f'Existed containers amount {len(containers)}') - - def configure_docker() -> None: logger.info('Configuring docker') pre_restart_tasks = ( diff --git a/node_cli/core/health.py b/node_cli/core/health.py index b4e7b0ee..3a805497 100644 --- a/node_cli/core/health.py +++ b/node_cli/core/health.py @@ -18,9 +18,8 @@ # along with this program. If not, see . import json -from terminaltables import SingleTable -from node_cli.utils.print_formatters import print_containers, print_schains_healthchecks +from node_cli.utils.print_formatters import Formatter, print_containers, print_schains_healthchecks from node_cli.utils.helper import error_exit, get_request from node_cli.utils.exit_codes import CLIExitCodes @@ -31,7 +30,7 @@ def get_containers(_all): status, payload = get_request( - blueprint=BLUEPRINT_NAME, method='containers', params={'all': _all} + blueprint=INFO_BLUEPRINT_NAME, method='containers', params={'all': _all} ) if status == 'ok': print_containers(payload) @@ -57,15 +56,13 @@ def get_sgx_info(): status, payload = get_request(blueprint=INFO_BLUEPRINT_NAME, method='sgx') if status == 'ok': data = payload - table_data = [ - ['SGX info', ''], + rows = [ ['Server URL', data['sgx_server_url']], ['SGXWallet Version', data['sgx_wallet_version']], ['Node SGX keyname', data['sgx_keyname']], ['Status HTTPS', data['status_https']], ['Status ZMQ', data['status_zmq']], ] - table = SingleTable(table_data) - print(table.table) + print(Formatter().table(['SGX info', ''], rows)) else: error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/node_cli/core/host.py b/node_cli/core/host.py index a040b8c4..331ebef5 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -20,7 +20,6 @@ import logging import os from shutil import chown -from urllib.parse import urlparse from skale_core.types import EnvType @@ -28,10 +27,8 @@ from node_cli.utils.helper import error_exit from node_cli.configs import ( - ADMIN_PORT, AUTOLOAD_KERNEL_MODULES_PATH, BTRFS_KERNEL_MODULE, - DEFAULT_URL_SCHEME, NODE_DATA_PATH, SKALE_DIR, CONTAINER_CONFIG_PATH, @@ -56,25 +53,9 @@ from node_cli.core.nftables import NFTablesManager from node_cli.utils.helper import safe_mkdir -from node_cli.utils.texts import safe_load_texts - -TEXTS = safe_load_texts() - logger = logging.getLogger(__name__) -def fix_url(url): - try: - result = urlparse(url) - if not result.scheme: - url = f'{DEFAULT_URL_SCHEME}{url}' - if not url.endswith(str(ADMIN_PORT)): - return f'{url}:{ADMIN_PORT}' - return url - except ValueError: - return False - - def prepare_host(env_type: EnvType, allocation: bool = False) -> None: try: logger.info('Preparing host started') @@ -118,10 +99,6 @@ def init_logs_dir(): safe_mkdir(REMOVED_CONTAINERS_FOLDER_PATH) -def init_data_dir(): - safe_mkdir(NODE_DATA_PATH) - - def is_btrfs_module_autoloaded(modules_filepath=AUTOLOAD_KERNEL_MODULES_PATH): if not os.path.isfile(modules_filepath): return False diff --git a/node_cli/core/nftables.py b/node_cli/core/nftables.py index 0d8cf0ba..a8baaad8 100644 --- a/node_cli/core/nftables.py +++ b/node_cli/core/nftables.py @@ -21,14 +21,12 @@ import logging import os import shutil -import sys from dataclasses import dataclass from pathlib import Path from typing import Optional from node_cli.configs import ( DEFAULT_NODE_BASE_PORT, - ENV, NFTABLES_CHAIN_CONFIG_WILDCARD, NFTABLES_CHAIN_FOLDER_PATH, NFTABLES_MAIN_CONFIG_PATH, @@ -40,16 +38,6 @@ logger = logging.getLogger(__name__) -try: - import nftables -except (FileNotFoundError, AttributeError, ModuleNotFoundError) as err: - if 'pytest' in sys.modules or ENV == 'dev': - from collections import namedtuple # hotfix for tests - - iptc = namedtuple('nftables', ['Chain', 'Rule']) - else: - logger.error(f'Unable to import nftables due to an error {err}') - @dataclass class ServicePort: @@ -188,6 +176,8 @@ def to_expr(self) -> list[dict]: class NFTablesManager: def __init__(self, family: str = 'inet', table: str = 'firewall', chain: str = 'skale') -> None: + import nftables + self.nft = nftables.Nftables() self.nft.set_json_output(True) self.nft.set_stateless_output(True) @@ -389,13 +379,6 @@ def add_drop_rule(self, rule: Rule) -> None: label=f'{rule.protocol} drop rule', ) - def remove_drop_rule(self, protocol: str) -> None: - expr = [ip_protocol_match(protocol), {'counter': None}, {'drop': None}] - if self._remove_rule_by_expr(self.chain, expr): - logger.info('Removed drop rule for %s', protocol) - else: - logger.info('Drop rule does not exist for %s', protocol) - def add_rule(self, rule: Rule) -> None: self._ensure_rule( rule.chain, @@ -403,12 +386,6 @@ def add_rule(self, rule: Rule) -> None: label=f'{rule.protocol} {rule.icmp_type or rule.first_port} {rule.action} rule', ) - def remove_rule(self, rule: Rule) -> None: - if self._remove_rule_by_expr(rule.chain, rule.to_expr()): - logger.info('Removed %s rule for %s', rule.protocol, rule.first_port) - else: - logger.info('No %s rule for %s to remove', rule.protocol, rule.first_port) - def _table_listing(self) -> dict: """Parsed json listing of the managed table; empty when absent.""" rc, output, error = self.nft.cmd(f'list table {self.family} {self.table}') @@ -702,41 +679,6 @@ def setup_firewall(self, keep_accept_policy: bool = False) -> None: POLICY_DROP if default_drop else POLICY_ACCEPT, ) - def cleanup_legacy_rules(self, ssh: bool = False, dns: bool = False) -> None: - """Cleans up all node-cli generated rules.""" - self.remove_drop_rule('tcp') - self.remove_drop_rule('udp') - tcp_ports = [ - ServicePort.HTTPS, - ServicePort.WATCHDOG_HTTP, - ServicePort.WATCHDOG_HTTPS, - ServicePort.EXPORTER, - ServicePort.CADVISOR, - ServicePort.DNS, # tcp is redundant, making sure it's removed - ] - if ssh: - tcp_ports.extend(get_ssh_ports()) - for port in tcp_ports: - self.remove_rule(Rule(chain=self.chain, protocol='tcp', first_port=port)) - if dns: - self.remove_rule(Rule(chain=self.chain, protocol='udp', first_port=ServicePort.DNS)) - - def flush_chain(self, chain: str) -> None: - """Remove all rules from a specific chain.""" - json_cmd = { - 'nftables': [ - {'flush': {'chain': {'family': self.family, 'table': self.table, 'name': chain}}} - ] - } - - try: - rc, output, error = self.nft.json_cmd(json_cmd) - if rc != 0: - raise NFTablesError(f'Failed to flush chain: {error}') - except Exception as e: - logger.error(f'Failed to flush chain: {str(e)}') - raise NFTablesError('Flushing chain errored') - def delete_chain(self, chain: str) -> None: chain_spec = {'family': self.family, 'table': self.table, 'name': chain} self.execute_cmd( diff --git a/node_cli/core/node.py b/node_cli/core/node.py index ea37e640..c298c4c2 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -29,7 +29,6 @@ import docker from filelock import FileLock -from node_cli.cli import __version__ from node_cli.configs import ( BACKUP_ARCHIVE_NAME, CONTAINER_CONFIG_PATH, @@ -55,7 +54,6 @@ passive_fair, ) from node_cli.core.nftables import get_registered_base_port -from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 from node_cli.operations import ( cleanup_skale_op, configure_nftables, @@ -85,7 +83,6 @@ read_json, save_json, ) -from node_cli.utils.meta import CliMetaManager from node_cli.utils.node_type import NodeType, NodeMode from node_cli.utils.print_formatters import ( print_failed_requirements_checks, @@ -133,11 +130,6 @@ def is_update_safe(node_mode: NodeMode) -> bool: @check_inited @check_user def register_node(name, p2p_ip, public_ip, port, domain_name): - if not is_node_inited(): - print(TEXTS['node']['not_inited']) - return - - # todo: add name, ips and port checks json_data = { 'name': name, 'ip': p2p_ip, @@ -190,7 +182,8 @@ def init(config_file: str, node_type: NodeType) -> None: settings = validate_and_save_node_settings(config_file, node_type, node_mode) compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) - init_op(settings=settings, compose_env=compose_env, node_mode=node_mode) + if not init_op(settings=settings, compose_env=compose_env, node_mode=node_mode): + error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): @@ -280,9 +273,6 @@ def enable_firewall_default_drop_when_port_available(timeout: int = 300, interva @check_user def update_passive(config_file: str) -> None: logger.info('Node update started') - prev_version = CliMetaManager().get_meta_info().version - if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': - migrate_2_6() save_internal_settings(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) settings = validate_and_save_node_settings(config_file, NodeType.SKALE, NodeMode.PASSIVE) compose_env = compose_node_env(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) @@ -337,11 +327,10 @@ def update( error_msg = 'Cannot update safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) - prev_version = CliMetaManager().get_meta_info().version - if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': - migrate_2_6() logger.info('Node update started') - save_internal_settings(node_type=node_type, node_mode=node_mode) + save_internal_settings( + node_type=node_type, node_mode=node_mode, pull_config_for_schain=pull_config_for_schain + ) settings = validate_and_save_node_settings(config_file, node_type, node_mode) compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) update_ok = update_op(settings=settings, compose_env=compose_env, node_mode=node_mode) @@ -361,8 +350,7 @@ def get_node_signature(validator_id): status, payload = get_request(blueprint=BLUEPRINT_NAME, method='signature', params=params) if status == 'ok': return payload['signature'] - else: - return payload + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) def backup(path): diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index 764e3b5f..f27e105e 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -83,12 +83,6 @@ def mark_active_node() -> None: logger.info('Node marked as active.') -def mark_passive_node() -> None: - node_options = NodeOptions() - node_options.node_mode = NodeMode.PASSIVE - logger.info('Node marked as passive.') - - def set_passive_node_options( archive: bool, indexer: bool, diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 34f4cac5..8e6149ba 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -17,7 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import os import logging from time import sleep from typing import Dict @@ -26,11 +25,8 @@ from skale_core.types import EnvType -from node_cli.utils.settings import validate_and_save_node_settings from node_cli.utils.docker_utils import ensure_volume -from node_cli.utils.schain_types import SchainTypes -from node_cli.utils.helper import write_json, read_json, run_cmd, safe_load_yml -from node_cli.utils.node_type import NodeType, NodeMode +from node_cli.utils.helper import write_json, run_cmd, safe_load_yml from node_cli.configs import ALLOCATION_FILEPATH, STATIC_PARAMS_FILEPATH, SNAPSHOTS_SHARED_VOLUME from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, @@ -47,10 +43,6 @@ logger = logging.getLogger(__name__) -class NotEnoughDiskSpaceError(Exception): - pass - - class ResourceAlloc: def __init__(self, value, fractional=False): self.values = { @@ -68,13 +60,6 @@ def dict(self): return self.values -def get_resource_allocation_info(): - try: - return read_json(RESOURCE_ALLOCATION_FILEPATH) - except FileNotFoundError: - return None - - def compose_resource_allocation_config(env_type: EnvType, params_by_env_type: Dict = None) -> Dict: params_by_env_type = params_by_env_type or safe_load_yml(STATIC_PARAMS_FILEPATH) common_config = params_by_env_type['common'] @@ -87,49 +72,18 @@ def compose_resource_allocation_config(env_type: EnvType, params_by_env_type: Di 'cpu_shares': schain_cpu_alloc.dict(), 'mem': schain_mem_alloc.dict(), 'disk': schain_allocation_data[env_type]['disk'], - 'volume_limits': schain_allocation_data[env_type]['volume_limits'], # noqa - 'leveldb_limits': schain_allocation_data[env_type]['leveldb_limits'], # noqa + 'volume_limits': schain_allocation_data[env_type]['volume_limits'], + 'leveldb_limits': schain_allocation_data[env_type]['leveldb_limits'], }, 'ima': {'cpu_shares': ima_cpu_alloc.dict(), 'mem': ima_mem_alloc.dict()}, } -def generate_resource_allocation_config( - env_file: str, - node_type: NodeType, - node_mode: NodeMode, - force: bool = False, -) -> None: - if not force and os.path.isfile(RESOURCE_ALLOCATION_FILEPATH): - msg = 'Resource allocation file already exists' - logger.debug(msg) - print(msg) - return - settings = validate_and_save_node_settings(env_file, node_type, node_mode) - logger.info('Generating resource allocation file ...') - try: - update_resource_allocation(settings.env_type) - except Exception as e: - logger.exception(e) - print("Can't generate resource allocation file, check out CLI logs") - else: - print(f'Resource allocation file generated: {RESOURCE_ALLOCATION_FILEPATH}') - - def update_resource_allocation(env_type: EnvType) -> None: resource_allocation_config = compose_resource_allocation_config(env_type) write_json(RESOURCE_ALLOCATION_FILEPATH, resource_allocation_config) -def get_available_memory(): - memory = [] - for _ in range(0, TIMES): - mem_info = psutil.virtual_memory() - memory.append(mem_info.available) - sleep(TIMEOUT) - return sum(memory) / TIMES * MEMORY_FACTOR - - def get_total_memory(): memory = [] for _ in range(0, TIMES): @@ -154,19 +108,6 @@ def get_cpu_alloc(common_config: Dict) -> ResourceAlloc: return ResourceAlloc(schain_max_cpu_shares), ResourceAlloc(ima_max_cpu_shares) -def verify_disk_size(disk_device: str, env_configs: dict): - disk_size = get_disk_size(disk_device) - env_disk_size = env_configs['server']['disk'] - check_disk_size(disk_size, env_disk_size) - - -def check_disk_size(disk_size: int, env_disk_size: int): - if env_disk_size > disk_size: - raise NotEnoughDiskSpaceError( - f'Disk size: {disk_size}, required disk size: {env_disk_size}' - ) - - def get_disk_size(disk_device: str) -> int: disk_size_cmd = construct_disk_size_cmd(disk_device) output = run_cmd(disk_size_cmd).stdout.decode('utf-8') @@ -177,19 +118,6 @@ def construct_disk_size_cmd(disk_path: str) -> list: return ['blockdev', '--getsize64', disk_path] -def check_is_partition(disk_path): - res = run_cmd(['blkid', disk_path]) - output = str(res.stdout) - if 'PARTUUID' in output: - return True - return False - - -def get_allocation_option_name(schain): - part_of_node = int(schain['partOfNode']) - return SchainTypes(part_of_node).name - - def init_shared_space_volume(env_type): logger.info('Configuring shared space volume') schain_allocation_data = safe_load_yml(ALLOCATION_FILEPATH) diff --git a/node_cli/core/sgx.py b/node_cli/core/sgx.py index 31cfdf99..0318d261 100644 --- a/node_cli/core/sgx.py +++ b/node_cli/core/sgx.py @@ -113,12 +113,8 @@ def get_certificate_status(directory: str | Path | None = None) -> dict: return status cert = _load_certificate(paths['crt']) now = datetime.datetime.now(datetime.timezone.utc) - try: - not_before = cert.not_valid_before_utc - not_after = cert.not_valid_after_utc - except AttributeError: - not_before = cert.not_valid_before.replace(tzinfo=datetime.timezone.utc) - not_after = cert.not_valid_after.replace(tzinfo=datetime.timezone.utc) + not_before = cert.not_valid_before_utc + not_after = cert.not_valid_after_utc days_left = (not_after - now).days status.update( { diff --git a/node_cli/core/ssl/status.py b/node_cli/core/ssl/status.py index 9b035b1d..20b1057b 100644 --- a/node_cli/core/ssl/status.py +++ b/node_cli/core/ssl/status.py @@ -18,9 +18,9 @@ # along with this program. If not, see . import logging -from dateutil import parser -from OpenSSL import crypto +from cryptography import x509 +from cryptography.x509.oid import NameOID from node_cli.core.ssl.utils import is_ssl_folder_empty, cert_from_file from node_cli.configs.ssl import SSL_CERT_FILEPATH, CERTS_INVALID_FORMAT @@ -45,14 +45,11 @@ def cert_status(): def get_cert_info(cert): try: - crypto_cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert) - subject = crypto_cert.get_subject() - issued_to = subject.CN - expiration_date_raw = crypto_cert.get_notAfter() - expiration_date = parser.parse(expiration_date_raw).strftime('%Y-%m-%dT%H:%M:%S') + crypto_cert = x509.load_pem_x509_certificate(cert) + common_names = crypto_cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) + issued_to = common_names[0].value if common_names else None + expiration_date = crypto_cert.not_valid_after_utc.strftime('%Y-%m-%dT%H:%M:%S') except Exception as err: logger.exception('Error during parsing certs') return err_result(str(err)) - return ok_result( - {'subject': subject, 'issued_to': issued_to, 'expiration_date': expiration_date} - ) + return ok_result({'issued_to': issued_to, 'expiration_date': expiration_date}) diff --git a/node_cli/core/ssl/utils.py b/node_cli/core/ssl/utils.py index c80b3a8a..0fd9924f 100644 --- a/node_cli/core/ssl/utils.py +++ b/node_cli/core/ssl/utils.py @@ -37,7 +37,7 @@ def cert_from_file(cert_filepath): if not os.path.isfile(cert_filepath): logger.warning(f'Trying to read cert that does not exist: {cert_filepath}') return None - with open(cert_filepath) as cert_file: + with open(cert_filepath, 'rb') as cert_file: return cert_file.read() diff --git a/node_cli/core/static_config.py b/node_cli/core/static_config.py index 06e0e7f4..c858d250 100644 --- a/node_cli/core/static_config.py +++ b/node_cli/core/static_config.py @@ -44,8 +44,7 @@ def get_static_params( static_params_filename = os.path.basename(static_params_base_filepath) static_params_filepath = os.path.join(config_path, static_params_filename) with open(static_params_filepath) as requirements_file: - ydata = yaml.load(requirements_file, Loader=yaml.Loader) - return ydata['envs'][env_type] + return yaml.safe_load(requirements_file)['envs'][env_type] def get_fair_chain_name(env_type: EnvType) -> str: diff --git a/node_cli/fair/active.py b/node_cli/fair/active.py index 4c9ee2b3..2d9dcb07 100644 --- a/node_cli/fair/active.py +++ b/node_cli/fair/active.py @@ -23,7 +23,6 @@ from typing import cast from node_cli.configs import DEFAULT_SKALED_BASE_PORT, RESTORE_SLEEP_TIMEOUT -from node_cli.core.host import is_node_inited from node_cli.core.node import compose_node_env, is_base_containers_alive from node_cli.operations import ( FairUpdateType, @@ -88,10 +87,6 @@ def migrate_from_boot( @check_inited @check_user def register(ip: str) -> None: - if not is_node_inited(): - print(TEXTS['fair']['node']['not_inited']) - return - json_data = {'ip': ip, 'port': DEFAULT_SKALED_BASE_PORT} status, payload = post_request(blueprint=BLUEPRINT_NAME, method='register', json=json_data) if status == 'ok': @@ -107,10 +102,6 @@ def register(ip: str) -> None: @check_inited @check_user def change_ip(ip: str) -> None: - if not is_node_inited(): - print(TEXTS['fair']['node']['not_inited']) - return - json_data = {'ip': ip, 'port': DEFAULT_SKALED_BASE_PORT} status, payload = post_request(blueprint=BLUEPRINT_NAME, method='change-ip', json=json_data) if status == 'ok': @@ -126,10 +117,6 @@ def change_ip(ip: str) -> None: @check_inited @check_user def exit() -> None: - if not is_node_inited(): - print(TEXTS['fair']['node']['not_inited']) - return - status, payload = post_request(blueprint=BLUEPRINT_NAME, method='exit', json={}) if status == 'ok': msg = TEXTS['fair']['node']['exited'] @@ -164,10 +151,6 @@ def restore(backup_path: str, config_file: str, config_only: bool = False): @check_inited @check_user def set_domain_name(domain_name): - if not is_node_inited(): - print(TEXTS['fair']['node']['not_inited']) - return - status, payload = post_request( blueprint=BLUEPRINT_NAME, method='set-domain-name', json={'domain_name': domain_name} ) diff --git a/node_cli/fair/boot.py b/node_cli/fair/boot.py index cbbcc5e6..e061f85a 100644 --- a/node_cli/fair/boot.py +++ b/node_cli/fair/boot.py @@ -43,7 +43,8 @@ def init(config_file: str) -> None: settings = validate_and_save_node_settings(config_file, node_type, node_mode) compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) - init_fair_boot_op(settings=settings, compose_env=compose_env, node_mode=node_mode) + if not init_fair_boot_op(settings=settings, compose_env=compose_env, node_mode=node_mode): + error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Waiting for fair containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=True): @@ -56,7 +57,9 @@ def init(config_file: str) -> None: def update(config_file: str, pull_config_for_schain: str) -> None: logger.info('Fair boot node update started') node_mode = upsert_node_mode(node_mode=NodeMode.ACTIVE) - save_internal_settings(node_type=NodeType.FAIR, node_mode=node_mode) + save_internal_settings( + node_type=NodeType.FAIR, node_mode=node_mode, pull_config_for_schain=pull_config_for_schain + ) settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) migrate_ok = update_fair_boot_op( diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 35642010..f0d194a1 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -102,7 +102,9 @@ def update( ) node_mode = upsert_node_mode(node_mode=node_mode) - save_internal_settings(node_type=NodeType.FAIR, node_mode=node_mode) + save_internal_settings( + node_type=NodeType.FAIR, node_mode=node_mode, pull_config_for_schain=pull_config_for_schain + ) settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) update_ok = update_fair_op( diff --git a/node_cli/fair/record/chain_record.py b/node_cli/fair/record/chain_record.py index b6d48886..a02e4a02 100644 --- a/node_cli/fair/record/chain_record.py +++ b/node_cli/fair/record/chain_record.py @@ -67,15 +67,9 @@ def force_skaled_start(self) -> bool: def set_config_version(self, version: str) -> None: self._set_field('config_version', version) - def set_repair_date(self, date: datetime) -> None: - self._set_field('repair_date', date) - def set_snapshot_from(self, value: str | None) -> None: self._set_field('snapshot_from', value) - def set_repair_ts(self, value: int | None) -> None: - self._set_field('repair_ts', value) - def set_force_skaled_start(self, value: bool) -> None: self._set_field('force_skaled_start', value) diff --git a/node_cli/fair/record/redis_record.py b/node_cli/fair/record/redis_record.py index f40aa898..749e8b1b 100644 --- a/node_cli/fair/record/redis_record.py +++ b/node_cli/fair/record/redis_record.py @@ -47,9 +47,6 @@ def __init__(self, name: str): self._set_defaults() self._save() - def to_dict(self) -> dict: - return self.mget(*self._record_fields().keys()) - def mget(self, *args) -> dict[str, Any]: key_names = [self._get_field_key(field_name) for field_name in args] raw_res = rs.mget(*key_names) @@ -135,19 +132,6 @@ def _save(self) -> None: def _record_fields(self) -> dict[str, FieldInfo]: """Return a list of FieldInfo objects representing the fields of the record.""" - @classmethod - def _redis_key_to_field_name(cls, key: bytes) -> str: - return key[:-5].decode('utf-8') - - @classmethod - def find_all(cls) -> list['FlatRedisRecord']: - name_keys = rs.keys('*_name') - records = [] - for key in name_keys: - chain_name = cls._redis_key_to_field_name(key) - records.append(cls(chain_name)) - return records - def __eq__(self, other) -> bool: if not isinstance(other, FlatRedisRecord): return False diff --git a/node_cli/fair/staking.py b/node_cli/fair/staking.py index 95b93adf..c8231317 100644 --- a/node_cli/fair/staking.py +++ b/node_cli/fair/staking.py @@ -131,7 +131,7 @@ def get_exit_requests(raw: bool = False) -> None: f'unlock_date: {unlock_date}' ) print(base + (f' ({unlock_iso})' if unlock_iso else '')) - except Exception: # noqa: BLE001 + except Exception: print(req) return error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/node_cli/main.py b/node_cli/main.py index b04b3869..c24a6d36 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -98,7 +98,7 @@ def info(): def get_command_groups() -> List[click.Group]: if TYPE == NodeType.FAIR: - return [ # type: ignore + return [ logs_cli, fair_boot_cli, fair_node_cli, @@ -110,7 +110,7 @@ def get_command_groups() -> List[click.Group]: sgx_cli, ] else: - return [ # type: ignore + return [ health_cli, schains_cli, logs_cli, diff --git a/node_cli/migrations/__init__.py b/node_cli/migrations/__init__.py deleted file mode 100644 index e99ab45f..00000000 --- a/node_cli/migrations/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from node_cli.migrations.focal_to_jammy import migrate # noqa diff --git a/node_cli/migrations/focal_to_jammy.py b/node_cli/migrations/focal_to_jammy.py deleted file mode 100644 index 7ff5c04a..00000000 --- a/node_cli/migrations/focal_to_jammy.py +++ /dev/null @@ -1,139 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - - -import logging - -from node_cli.core.nftables import ( - LEGACY_CHAIN, - LEGACY_FAMILY, - LEGACY_TABLE, - POLICY_ACCEPT, - NFTablesManager, - remove_legacy_saved_rules, -) -from node_cli.utils.helper import run_cmd - -logger = logging.getLogger(__name__) - - -ALLOWED_INCOMING_TCP_PORTS = [ - '80', # filestorage - '311', # watchdog https - '8080', # http - '443', # https - '53', # dns - '3009', # watchdog http - '9100', # node exporter -] - -ALLOWED_INCOMING_UDP_PORTS = [ - '53' # dns -] - - -class NFTablesCmdFailedError(Exception): - pass - - -def remove_tcp_rules() -> None: - tcp_rule_template = 'iptables -{} {} -p tcp -m tcp --dport {} -j ACCEPT' - for tcp_port in [*ALLOWED_INCOMING_TCP_PORTS]: - check_cmd = tcp_rule_template.format('C', LEGACY_CHAIN, tcp_port).split(' ') - remove_cmd = tcp_rule_template.format('D', LEGACY_CHAIN, tcp_port).split(' ') - result = run_cmd(check_cmd, check_code=False) - if result.returncode == 0: - result = run_cmd(remove_cmd) - - -def remove_udp_rules() -> None: - udp_rule_template = 'iptables -{} {} -p udp -m udp --dport {} -j ACCEPT' - for udp_port in [*ALLOWED_INCOMING_UDP_PORTS]: - check_cmd = udp_rule_template.format('C', LEGACY_CHAIN, udp_port).split(' ') - remove_cmd = udp_rule_template.format('D', LEGACY_CHAIN, udp_port).split(' ') - result = run_cmd(check_cmd, check_code=False) - if result.returncode == 0: - result = run_cmd(remove_cmd) - - -def remove_loopback_rules() -> None: - loopback_rule_template = 'iptables -{} {} -i lo -j ACCEPT' - check_cmd = loopback_rule_template.format('C', LEGACY_CHAIN).split(' ') - remove_cmd = loopback_rule_template.format('D', LEGACY_CHAIN).split(' ') - result = run_cmd(check_cmd, check_code=False) - if result.returncode == 0: - result = run_cmd(remove_cmd) - - -def remove_icmp_rules() -> None: - icmp_rule_template = 'iptables -{} {} -p icmp -m icmp --icmp-type {} -j ACCEPT' - for icmp_type in [3, 4, 11]: - check_cmd = icmp_rule_template.format('C', LEGACY_CHAIN, icmp_type).split(' ') - remove_cmd = icmp_rule_template.format('D', LEGACY_CHAIN, icmp_type).split(' ') - result = run_cmd(check_cmd, check_code=False) - if result.returncode == 0: - result = run_cmd(remove_cmd) - - -def remove_conntrack_rules() -> None: - track_rule_template = 'iptables -{} {} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT' - check_cmd = track_rule_template.format('C', LEGACY_CHAIN).split(' ') - remove_cmd = track_rule_template.format('D', LEGACY_CHAIN).split(' ') - result = run_cmd(check_cmd, check_code=False) - if result.returncode == 0: - result = run_cmd(remove_cmd) - - -def remove_drop_rules() -> None: - drop_rule_template = 'iptables -{} {} -p {} -j DROP' - protocols = ['tcp', 'udp'] - for proto in protocols: - check_cmd = drop_rule_template.format('C', LEGACY_CHAIN, proto).split(' ') - remove_cmd = drop_rule_template.format('D', LEGACY_CHAIN, proto).split(' ') - result = run_cmd(check_cmd, check_code=False) - if result.returncode == 0: - result = run_cmd(remove_cmd) - - -def remove_old_iptables_rules() -> None: - remove_drop_rules() - remove_conntrack_rules() - remove_loopback_rules() - remove_tcp_rules() - remove_icmp_rules() - - -def migrate() -> None: - nft = NFTablesManager(family=LEGACY_FAMILY, table=LEGACY_TABLE, chain=LEGACY_CHAIN) - logger.info('Making sure legacy chain has default policy accept') - nft.update_chain_policy(chain=LEGACY_CHAIN, policy=POLICY_ACCEPT) - - logger.info('Running migration from focal to jammy') - remove_old_iptables_rules() - - logger.info('Flushing nftables rules generated by release upgrade') - nft.cleanup_legacy_rules(ssh=False, dns=False) - - # Logging rules after migration - res = run_cmd(['nft', 'list', 'ruleset']) - plain_rules = res.stdout.decode('utf-8') - logger.debug(plain_rules) - - remove_legacy_saved_rules() - - logger.info('Migration from focal to jammy completed') diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index d2bb5c57..4b534d47 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -138,7 +138,6 @@ def update(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) - generate_nginx_config() prepare_host(settings.env_type, allocation=True) - save_internal_settings(node_type=NodeType.SKALE, node_mode=node_mode) init_shared_space_volume(settings.env_type) meta_manager = CliMetaManager() @@ -170,7 +169,7 @@ def update(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) - @checked_host -def init(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) -> None: +def init(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) -> bool: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() if not settings.skip_docker_config: @@ -205,6 +204,7 @@ def init(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) -> node_mode=node_mode, ) compose_up(env=compose_env, settings=settings, node_type=NodeType.SKALE, node_mode=node_mode) + return True def init_passive( @@ -429,14 +429,6 @@ def restore( return True -def cleanup_passive(compose_env: dict, schain_name: str) -> None: - turn_off(compose_env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) - cleanup_nftables() - cleanup_no_lvm_datadir(chain_name=schain_name) - rm_dir(GLOBAL_SKALE_DIR) - rm_dir(SKALE_DIR) - - def cleanup( node_mode: NodeMode, compose_env: dict, schain_name: Optional[str] = None, prune: bool = False ) -> None: diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index c595a3b8..2ac82703 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -44,4 +44,4 @@ def configure_filebeat(): def unpack_backup_archive(backup_path: str) -> None: logger.info('Unpacking backup archive...') with tarfile.open(backup_path) as tar: - tar.extractall(path=G_CONF_HOME) + tar.extractall(path=G_CONF_HOME, filter='tar') diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index eb307d55..c61a1569 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -18,21 +18,14 @@ # along with this program. If not, see . import logging -import os -import shutil import crontab -from node_cli.utils.git_utils import sync_repo from node_cli.configs import ( - DOCKER_LVMPY_PATH, - DOCKER_LVMPY_REPO_URL, - FILESTORAGE_MAPPING, LVMPY_RUN_CMD, LVMPY_HEAL_CMD, LVMPY_CRON_LOG_PATH, LVMPY_CRON_SCHEDULE_MINUTES, - SCHAINS_MNT_DIR_REGULAR, VOLUME_GROUP, ) from node_cli.operations.volume import ensure_filestorage_mapping @@ -41,24 +34,9 @@ logger = logging.getLogger(__name__) -def update_docker_lvmpy_env(env): - env['PHYSICAL_VOLUME'] = env['BLOCK_DEVICE'] - env['VOLUME_GROUP'] = 'schains' - env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING - env['MNT_DIR'] = SCHAINS_MNT_DIR_REGULAR - env['PATH'] = os.environ.get('PATH', None) - return env - - -def sync_docker_lvmpy_repo(env): - if os.path.isdir(DOCKER_LVMPY_PATH): - shutil.rmtree(DOCKER_LVMPY_PATH) - sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_VERSION']) - - def lvmpy_install(block_device: str) -> None: ensure_filestorage_mapping() - logging.info('Configuring and starting lvmpy') + logger.info('Configuring and starting lvmpy') setup_lvmpy(block_device=block_device, volume_group=VOLUME_GROUP, exec_start=LVMPY_RUN_CMD) init_healing_cron() logger.info('docker-lvmpy is configured and started') diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index f2e8746c..d7680691 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -80,7 +80,6 @@ class FairUpdateType(Enum): REGULAR = 'regular' - INFRA_ONLY = 'infra_only' FROM_BOOT = 'from_boot' @@ -89,7 +88,7 @@ def init_fair_boot( settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode, -) -> None: +) -> bool: sync_skale_node() cleanup_volume_artifacts(settings.block_device) @@ -129,6 +128,7 @@ def init_fair_boot( node_mode=NodeMode.ACTIVE, is_fair_boot=True, ) + return True @checked_host @@ -218,7 +218,6 @@ def update_fair_boot( prepare_block_device(settings.block_device, force=fair_settings.enforce_btrfs) prepare_host(settings.env_type) - save_internal_settings(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream @@ -262,7 +261,7 @@ def update( ) -> bool: compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=compose_env) rm_legacy_containers() - if update_type not in (FairUpdateType.INFRA_ONLY, FairUpdateType.FROM_BOOT): + if update_type != FairUpdateType.FROM_BOOT: remove_dynamic_containers() sync_skale_node() @@ -275,7 +274,6 @@ def update( generate_nginx_config() prepare_host(settings.env_type, allocation=True) - save_internal_settings(node_type=NodeType.FAIR, node_mode=node_mode) meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream if not settings.skip_docker_cleanup and current_stream != settings.node_version: diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index e944547a..a32400a4 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -20,16 +20,11 @@ import glob import logging import os -import shutil import tempfile from node_cli.utils.helper import run_cmd -from node_cli.utils.git_utils import sync_repo from node_cli.configs import ( - DOCKER_LVMPY_PATH, - DOCKER_LVMPY_REPO_URL, FILESTORAGE_MAPPING, - SCHAINS_MNT_DIR_REGULAR, SCHAINS_MNT_DIR_SINGLE_CHAIN, SKALE_STATE_DIR, ) @@ -37,45 +32,8 @@ logger = logging.getLogger(__name__) -class FilesystemExistsError(Exception): - pass - - -def update_docker_lvmpy_env(env): - env['PHYSICAL_VOLUME'] = env['BLOCK_DEVICE'] - env['VOLUME_GROUP'] = 'schains' - env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING - env['SCHAINS_MNT_DIR'] = SCHAINS_MNT_DIR_REGULAR - env['PATH'] = os.environ.get('PATH', None) - return env - - -def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): - if not os.path.isdir(FILESTORAGE_MAPPING): - os.makedirs(FILESTORAGE_MAPPING) - - -def sync_docker_lvmpy_repo(env): - if os.path.isdir(DOCKER_LVMPY_PATH): - shutil.rmtree(DOCKER_LVMPY_PATH) - sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_VERSION']) - - -def docker_lvmpy_update(env): - sync_docker_lvmpy_repo(env) - ensure_filestorage_mapping() - logger.info('Running docker-lvmpy update script') - update_docker_lvmpy_env(env) - run_cmd(cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/update.sh'.split(), env=env) - logger.info('docker-lvmpy update done') - - -def docker_lvmpy_install(env): - sync_docker_lvmpy_repo(env) - ensure_filestorage_mapping() - update_docker_lvmpy_env(env) - run_cmd(cmd=f'sudo -H -E {DOCKER_LVMPY_PATH}/scripts/install.sh'.split(), env=env) - logger.info('docker-lvmpy installed') +def ensure_filestorage_mapping(): + os.makedirs(FILESTORAGE_MAPPING, exist_ok=True) def block_device_mounted(block_device): diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index b9b8e547..40ae07af 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -95,9 +95,7 @@ **REDIS_SERVICE_DICT, } -TELEGRAF_SERVICES = ('telegraf',) NOTIFICATION_COMPOSE_SERVICES = ('celery',) -COMPOSE_TIMEOUT = 10 DOCKER_DEFAULT_STOP_TIMEOUT = 20 @@ -111,10 +109,6 @@ def docker_client() -> DockerClient: return docker.from_env() -def get_sanitized_container_name(container_info: dict) -> str: - return container_info['Names'][0].replace('/', '', 1) - - def get_containers(container_name_filter=None, _all=True) -> list: filters = {} if container_name_filter: @@ -195,18 +189,6 @@ def stop_container_by_name( container.stop(timeout=timeout) -def remove_container_by_name( - container_name: str, - timeout: int = DOCKER_DEFAULT_STOP_TIMEOUT, - dclient: Optional[DockerClient] = None, -) -> None: - dc = dclient or docker_client() - container_names = [container.name for container in get_containers()] - if container_name in container_names: - container = dc.containers.get(container_name) - safe_rm(container, timeout=timeout) - - def start_container_by_name(container_name: str, dclient: Optional[DockerClient] = None) -> None: dc = dclient or docker_client() container = dc.containers.get(container_name) @@ -214,13 +196,6 @@ def start_container_by_name(container_name: str, dclient: Optional[DockerClient] container.start() -def remove_schain_container_by_name( - schain_name: str, dclient: Optional[DockerClient] = None -) -> None: - container_name = f'sk_skaled_{schain_name}' - remove_container_by_name(container_name, timeout=SCHAIN_REMOVE_TIMEOUT, dclient=dclient) - - def backup_container_logs( container: Container, tail: int | str = DOCKER_DEFAULT_TAIL_LINES, diff --git a/node_cli/utils/git_utils.py b/node_cli/utils/git_utils.py index 38e5a11c..857632a5 100644 --- a/node_cli/utils/git_utils.py +++ b/node_cli/utils/git_utils.py @@ -35,18 +35,6 @@ def clone_repo(repo_url: str, repo_path: str, ref_name: str) -> None: fetch_pull_repo(repo_path, ref_name) -def sync_repo(repo_url: str, repo_path: str, ref_name: str) -> None: - """ - Sync Git repository by cloning if it doesn't exist locally. If it exists, fetch latest changes. - """ - - logger.info(f'Sync repo {repo_url} → {repo_path}') - if not os.path.isdir(os.path.join(repo_path, '.git')): - clone_repo(repo_url, repo_path, ref_name) - else: - fetch_pull_repo(repo_path, ref_name) - - def fetch_pull_repo(repo_path: str, ref_name: str) -> None: """Fetch latest changes and checkout/pull specific git reference.""" diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 2ae9ab50..f3bf1d3f 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -22,13 +22,10 @@ import logging import logging.handlers as py_handlers import os -import re import shutil -import socket import subprocess import sys import urllib.parse -import urllib.request import uuid from pathlib import Path from functools import wraps @@ -44,7 +41,6 @@ from node_cli.configs import ( ADMIN_HOST, ADMIN_PORT, - DEFAULT_SSH_PORT, GLOBAL_SKALE_CONF_FILEPATH, GLOBAL_SKALE_DIR, HIDE_STREAM_LOG, @@ -78,10 +74,6 @@ } -class InvalidEnvFileError(Exception): - pass - - def read_json(path: str) -> dict: with open(path, encoding='utf-8') as data_file: return json.loads(data_file.read()) @@ -125,14 +117,6 @@ def run_cmd(cmd, env={}, shell=False, secure=False, check_code=True, separate_st return res -def format_output(res: subprocess.CompletedProcess) -> str: - return res.stdout.decode('UTF-8').rstrip() - - -def download_file(url, filepath): - return urllib.request.urlretrieve(url, filepath) - - def process_template(source, destination, data): """ :param source: j2 template source path @@ -146,10 +130,6 @@ def process_template(source, destination, data): f.write(processed_template) -def get_username(): - return os.environ.get('USERNAME') or os.environ.get('USER') - - def error_exit(error_payload: Any, exit_code: CLIExitCodes = CLIExitCodes.FAILURE) -> NoReturn: """Print error message and exit the program with specified exit code. @@ -172,14 +152,6 @@ def error_exit(error_payload: Any, exit_code: CLIExitCodes = CLIExitCodes.FAILUR sys.exit(exit_code.value) -def safe_get_config(config, key): - try: - return config[key] - except KeyError as e: - logger.error(e) - return None - - def safe_load_yml(filepath): with open(filepath, 'r') as stream: try: @@ -197,12 +169,12 @@ def abort_if_false(ctx, param, value): ctx.abort() -def post_request(blueprint, method, json=None, files=None): +def post_request(blueprint, method, json=None): route = get_route(blueprint, method) url = construct_url(route) try: response = api_session.post( - url, json=json, files=files, headers=get_api_headers(), allow_redirects=False + url, json=json, headers=get_api_headers(), allow_redirects=False ) data = response.json() except APIAuthError as err: @@ -236,28 +208,6 @@ def get_request( return status, payload -def download_dump(path, container_name=None): - route = get_route('logs', 'dump') - url = construct_url(route) - params = {} - if container_name: - params['container_name'] = container_name - with requests.get(url, params=params, stream=True) as r: - if r is None: - return None - if r.status_code != requests.codes.ok: # pylint: disable=no-member - print('Request failed, status code:', r.status_code) - error_exit(r.json()) - return None - d = r.headers['Content-Disposition'] - fname_q = re.findall('filename=(.+)', d)[0] - fname = fname_q.replace('"', '') - filepath = os.path.join(path, fname) - with open(filepath, 'wb') as f: - shutil.copyfileobj(r.raw, f) - return fname - - def init_default_logger(): f_handler = get_file_handler(LOG_FILEPATH, logging.INFO) debug_f_handler = get_file_handler(DEBUG_LOG_FILEPATH, logging.DEBUG) @@ -288,11 +238,6 @@ def read_file(path): return file.read() -def to_camel_case(snake_str): - components = snake_str.split('_') - return components[0] + ''.join(x.title() for x in components[1:]) - - def streamed_cmd(func): """Decorator that allows function to print logs into stderr.""" @@ -321,10 +266,6 @@ def get_g_conf_user(): return get_g_conf()['user'] -def get_g_conf_home(): - return get_g_conf()['home_dir'] - - def rm_dir(folder: str) -> None: if os.path.exists(folder): logger.info(f'{folder} exists, removing...') @@ -473,17 +414,6 @@ def get_ssh_ports() -> list[int]: return _validated_ssh_ports(values) -def get_ssh_port(ssh_service_name='ssh') -> int: - """Return the first SSH port; firewall callers must use get_ssh_ports().""" - if ssh_service_name == 'ssh': - return get_ssh_ports()[0] - try: - return socket.getservbyname(ssh_service_name) - except OSError: - logger.exception('Cannot get ssh service port') - return DEFAULT_SSH_PORT - - def is_btrfs_subvolume(path: str) -> bool: """Check if the given path is a Btrfs subvolume.""" try: diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index 1da07d51..c7baa83d 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -24,7 +24,6 @@ from typing import Any import texttable -from dateutil import parser from node_cli.configs import LONG_LINE from node_cli.configs.cli_logger import DEBUG_LOG_FILEPATH @@ -85,11 +84,11 @@ def print_containers(containers): headers = ['Name', 'Status', 'Started At', 'Image'] rows = [] for container in containers: - date = parser.parse(container['state']['StartedAt']) + date = datetime.datetime.fromisoformat(container['state']['StartedAt']) status = container['state']['Status'].capitalize() if not container['state']['Running']: - finished_date = parser.parse(container['state']['FinishedAt']) + finished_date = datetime.datetime.fromisoformat(container['state']['FinishedAt']) status = f'{status} ({format_date(finished_date)})' rows.append([container['name'], status, format_date(date), container['image']]) @@ -170,34 +169,6 @@ def print_schains_healthchecks(schains): print(Formatter().table(headers, rows)) -def print_logs(logs): - print('Base logs\n') - print_log_list(logs['base']) - print(f'\n{LONG_LINE}') - print('\nsChains logs\n') - print_schains_logs(logs['schains']) - - -def print_schains_logs(schains_logs): - for name in schains_logs: - print(f'\n{name} \n') - print_log_list(schains_logs[name]['logs']) - - -def print_log_list(logs): - headers = ['Name', 'Size', 'Created At'] - rows = [] - for log in logs: - date = datetime.datetime.fromtimestamp(log['created_at']) - rows.append([log['name'], log['size'], format_date(date)]) - print(Formatter().table(headers, rows)) - - -def print_dict(title, rows, headers=['Key', 'Value']): - print(title) - print(Formatter().table(headers, rows)) - - def print_exit_status(exit_status_info): headers = ['Schain name', 'Status'] logs = exit_status_info['data'] @@ -208,7 +179,7 @@ def print_exit_status(exit_status_info): print(f'\n{status_info}\n') if node_exit_status == 'wait_for_rotations': exit_time = exit_status_info['exit_time'] - exit_time_utc = datetime.datetime.utcfromtimestamp(exit_time) + exit_time_utc = datetime.datetime.fromtimestamp(exit_time, datetime.UTC) print(f'Rotation finish time: {exit_time_utc}') diff --git a/node_cli/utils/schain_types.py b/node_cli/utils/schain_types.py deleted file mode 100644 index 22602a5e..00000000 --- a/node_cli/utils/schain_types.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from enum import Enum - - -class SchainTypes(Enum): - test = 0 - test4 = 4 - large = 1 - medium = 8 - small = 128 diff --git a/node_cli/utils/settings.py b/node_cli/utils/settings.py index 378ef4df..77a39427 100644 --- a/node_cli/utils/settings.py +++ b/node_cli/utils/settings.py @@ -18,7 +18,6 @@ # along with this program. If not, see . import tomllib -from pathlib import Path from dotenv.main import DotEnv @@ -52,11 +51,6 @@ def load_config_file(filepath: str) -> dict: return {k.lower(): v for k, v in DotEnv(filepath).dict().items()} -def _remove_if_exists(path: Path) -> None: - if path.exists(): - path.unlink() - - def validate_and_save_node_settings( config_filepath: str, node_type: NodeType, @@ -64,8 +58,6 @@ def validate_and_save_node_settings( ) -> BaseNodeSettings: data = load_config_file(config_filepath) settings_type = SETTINGS_MAP[(node_type.value, node_mode.value)] - settings_type.model_validate(data) - _remove_if_exists(NODE_SETTINGS_PATH) write_node_settings_file(path=NODE_SETTINGS_PATH, settings_type=settings_type, data=data) return settings_type() @@ -83,8 +75,6 @@ def save_internal_settings( 'backup_run': backup_run, 'pull_config_for_schain': pull_config_for_schain, } - InternalSettings.model_validate(data) - _remove_if_exists(INTERNAL_SETTINGS_PATH) write_internal_settings_file(path=INTERNAL_SETTINGS_PATH, data=data) diff --git a/pyproject.toml b/pyproject.toml index 652dedcb..9212c08b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,86 +1,54 @@ [build-system] -requires = ["setuptools>=75", "wheel"] +requires = ["setuptools>=75"] build-backend = "setuptools.build_meta" [project] name = "node-cli" -version = "3.3.2" -description = "Node CLI tools" -readme = "README.md" -requires-python = ">=3.13" -license = { file = "LICENSE" } -keywords = ["skale", "cli"] -authors = [{ name = "SKALE Labs", email = "support@skalelabs.com" }] -classifiers = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: GNU Affero General Public License v3", - "Natural Language :: English", - "Programming Language :: Python :: 3.13", -] - +version = "3.4.0" +requires-python = ">=3.14" dependencies = [ - "click==8.3.1", + "click==8.5.0", "distro==1.9.0", - "docker==7.1.0", - "texttable==1.7.0", - "python-dateutil==2.9.0.post0", + "docker==7.2.0", + "texttable==1.7.2", "Jinja2==3.1.6", - "psutil==7.1.3", - "python-dotenv==1.2.1", - "terminaltables==3.1.10", - "requests==2.32.5", - "GitPython==3.1.45", - "packaging==25.0", - "python-debian==1.0.1", + "psutil==7.2.2", + "python-dotenv==1.2.4", + "requests==2.34.2", + "GitPython==3.2.1", + "packaging==26.3", + "python-debian==1.1.1", "PyYAML==6.0.3", - "pyOpenSSL==25.3.0", - "MarkupSafe==3.0.3", - "Flask==3.1.2", - "itsdangerous==2.2.0", - "cryptography==46.0.5", - "filelock==3.20.0", - "sh==2.2.2", - "python-crontab==3.3.0", - "requests-mock==1.12.1", - "redis==7.1.1", - "PyInstaller==6.18.0", - "skale.py-core==7.14.dev2", + "Flask==3.1.3", + "cryptography==50.0.2", + "filelock==4.2.4", + "python-crontab==3.4.0", + "redis==8.1.0", + "PyInstaller==6.22.3", + "skale.py-core==8.0dev0", ] -[project.urls] -Homepage = "https://github.com/skalenetwork/node-cli" - [project.optional-dependencies] dev = [ - "ruff==0.14.6", - "bumpversion==0.6.0", - "pytest==9.0.1", - "pytest-cov==7.0.0", - "twine==6.2.0", - "mock==5.2.0", + "ruff==0.17.3", + "pytest==9.2.0", "freezegun==1.5.5", + "requests-mock==1.12.1", ] -[tool.setuptools] -package-dir = { "" = "." } -include-package-data = true - [tool.setuptools.packages.find] -where = ["."] -exclude = ["tests"] +include = ["node_cli*"] [tool.ruff] line-length = 100 -target-version = "py313" +extend-exclude = ["helper-scripts"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] [tool.ruff.format] quote-style = "single" -[tool.uv] -prerelease = "allow" - - [tool.pytest.ini_options] log_cli = false log_cli_level = "INFO" diff --git a/scripts/build.sh b/scripts/build.sh index 8e26577a..365dcd69 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,53 +1,22 @@ #!/usr/bin/env bash - set -e VERSION=$1 BRANCH=$2 TYPE=$3 -USAGE_MSG='Usage: build.sh [VERSION] [BRANCH] [TYPE]' - -if [ -z "$1" ] -then - (>&2 echo 'You should provide version') - echo $USAGE_MSG - exit 1 -fi - -if [ -z "$2" ] -then - (>&2 echo 'You should provide git branch') - echo $USAGE_MSG - exit 1 -fi - -if [ -z "$3" ] -then - (>&2 echo 'You should provide type: skale, or fair') - echo $USAGE_MSG - exit 1 -fi - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -PARENT_DIR="$(dirname "$DIR")" - -OS=`uname -s`-`uname -m` - -# Use the new generate_info.sh script -bash "${DIR}/generate_info.sh" "$VERSION" "$BRANCH" "$TYPE" +cd "$(dirname "${BASH_SOURCE[0]}")/.." +bash scripts/generate_info.sh "$VERSION" "$BRANCH" "$TYPE" +EXECUTABLE_NAME=skale-$VERSION-$(uname -s)-$(uname -m) if [ "$TYPE" = "fair" ]; then - EXECUTABLE_NAME=skale-$VERSION-$OS-fair -else - EXECUTABLE_NAME=skale-$VERSION-$OS + EXECUTABLE_NAME=$EXECUTABLE_NAME-fair fi pyinstaller main.spec -mv $PARENT_DIR/dist/main $PARENT_DIR/dist/$EXECUTABLE_NAME +mv dist/main "dist/$EXECUTABLE_NAME" -echo "=========================================================================================" echo "Built node-cli v$VERSION, branch: $BRANCH" echo "Executable: $EXECUTABLE_NAME" diff --git a/scripts/generate_info.sh b/scripts/generate_info.sh index 67ee6aef..7135e3c9 100644 --- a/scripts/generate_info.sh +++ b/scripts/generate_info.sh @@ -3,56 +3,22 @@ set -e VERSION=$1 BRANCH=$2 -TYPE_STR=$3 +TYPE=$3 -USAGE_MSG='Usage: generate_info.sh [VERSION] [BRANCH] [TYPE]' - -if [ -z "$VERSION" ]; then - (>&2 echo 'You should provide version') - echo $USAGE_MSG - exit 1 -fi -if [ -z "$BRANCH" ]; then - (>&2 echo 'You should provide git branch') - echo $USAGE_MSG - exit 1 -fi -if [ -z "$TYPE_STR" ]; then - (>&2 echo 'You should provide type: skale or fair') - echo $USAGE_MSG +if [ -z "$VERSION" ] || [ -z "$BRANCH" ] || [[ ! "$TYPE" =~ ^(skale|fair)$ ]]; then + echo 'Usage: generate_info.sh VERSION BRANCH skale|fair' >&2 exit 1 fi -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -PARENT_DIR="$(dirname "$DIR")" -DIST_INFO_FILEPATH=$PARENT_DIR/node_cli/cli/info.py - -LATEST_COMMIT=$(git rev-parse HEAD) -CURRENT_DATETIME="$(date "+%Y-%m-%d %H:%M:%S")" -OS="$(uname -s)-$(uname -m)" - -case "$TYPE_STR" in - skale) - TYPE_ENUM="NodeType.SKALE" - ;; - fair) - TYPE_ENUM="NodeType.FAIR" - ;; - *) - (>&2 echo "Error: Invalid type '$TYPE_STR'. Must be 'skale', or 'fair'") - exit 1 - ;; -esac - -rm -f "$DIST_INFO_FILEPATH" -touch "$DIST_INFO_FILEPATH" +COMMIT=$(git rev-parse HEAD) -echo "from node_cli.utils.node_type import NodeType" >> "$DIST_INFO_FILEPATH" -echo "" >> "$DIST_INFO_FILEPATH" +cat > "$(dirname "${BASH_SOURCE[0]}")/../node_cli/cli/info.py" <> "$DIST_INFO_FILEPATH" -echo "COMMIT = '$LATEST_COMMIT'" >> "$DIST_INFO_FILEPATH" -echo "BRANCH = '$BRANCH'" >> "$DIST_INFO_FILEPATH" -echo "OS = '$OS'" >> "$DIST_INFO_FILEPATH" -echo "VERSION = '$VERSION'" >> "$DIST_INFO_FILEPATH" -echo "TYPE = $TYPE_ENUM" >> "$DIST_INFO_FILEPATH" \ No newline at end of file +BUILD_DATETIME = '$(date '+%Y-%m-%d %H:%M:%S')' +COMMIT = '$COMMIT' +BRANCH = '$BRANCH' +OS = '$(uname -s)-$(uname -m)' +VERSION = '$VERSION' +TYPE = NodeType('$TYPE') +EOF diff --git a/scripts/run_nftables_test.sh b/scripts/run_nftables_test.sh index ac665408..51aa7a0c 100755 --- a/scripts/run_nftables_test.sh +++ b/scripts/run_nftables_test.sh @@ -1,14 +1,6 @@ #!/usr/bin/env bash -set -ea +set -e -docker rm -f ncli-tester || true docker build . -t ncli-tester -docker run \ - -e LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \ - -e HIDE_STREAM_LOG=true \ - -e TEST_HOME_DIR="$PROJECT_DIR/tests/" \ - -e GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \ - -e DOTENV_FILEPATH='tests/test-env' \ - --cap-add=NET_ADMIN --cap-add=NET_RAW \ - --name ncli-tester ncli-tester py.test tests/core/migration_test.py tests/core/nftables_test.py tests/core/monitoring_firewall_test.py $@ - +docker run --rm --cap-add=NET_ADMIN --cap-add=NET_RAW ncli-tester bash -c \ + '. scripts/export_env.sh && pytest tests/core/nftables_test.py tests/core/monitoring_firewall_test.py "$@"' _ "$@" diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 27bf2aca..ce873423 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -PROJECT_DIR=$(dirname $DIR) +. "$(dirname "${BASH_SOURCE[0]}")/export_env.sh" -. "$DIR/export_env.sh" - -py.test --cov=$PROJECT_DIR/ --ignore=tests/core/nftables_test.py --ignore=tests/core/migration_test.py --ignore=tests/core/monitoring_firewall_test.py tests/ $@ +pytest --ignore=tests/core/nftables_test.py --ignore=tests/core/monitoring_firewall_test.py tests/ "$@" diff --git a/scripts/set_versions_ga.sh b/scripts/set_versions_ga.sh deleted file mode 100644 index 7333357f..00000000 --- a/scripts/set_versions_ga.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -echo "PROJECT_DIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV - -echo PROJECT_DIR: $GITHUB_WORKSPACE - -export BRANCH=${GITHUB_REF##*/} -echo "Branch $BRANCH" - -export VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") -export VERSION=$(bash ./helper-scripts/calculate_version.sh) - -echo "VERSION=$VERSION" >> $GITHUB_ENV -echo "Version $VERSION" - -export OS=`uname -s`-`uname -m` -export EXECUTABLE_NAME=skale-$VERSION-$OS - -echo "BRANCH=$BRANCH" >> $GITHUB_ENV -echo "EXECUTABLE_NAME=$EXECUTABLE_NAME" >> $GITHUB_ENV diff --git a/tests/.skale/.skale-cli.yaml b/tests/.skale/.skale-cli.yaml deleted file mode 100644 index 03d160ba..00000000 --- a/tests/.skale/.skale-cli.yaml +++ /dev/null @@ -1 +0,0 @@ -host: https://test.com diff --git a/tests/cli/exit_test.py b/tests/cli/exit_test.py index 83b3f077..4c797bab 100644 --- a/tests/cli/exit_test.py +++ b/tests/cli/exit_test.py @@ -15,4 +15,4 @@ def test_exit_status(): assert ( result.output == "{'status': 'ACTIVE', 'data': [{'name': 'test', 'status': 'ACTIVE'}], 'exit_time': 0}\n" - ) # noqa + ) diff --git a/tests/cli/fair_cli_test.py b/tests/cli/fair_cli_test.py index 14e2aee9..7621a748 100644 --- a/tests/cli/fair_cli_test.py +++ b/tests/cli/fair_cli_test.py @@ -130,7 +130,7 @@ def test_cleanup_node(mocked_g_config, inited_node): mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), mock.patch('node_cli.fair.common.compose_node_env', return_value={'SCHAIN_NAME': 'test'}), mock.patch( - 'node_cli.core.node.CliMetaManager.get_meta_info', + 'node_cli.utils.meta.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): diff --git a/tests/cli/fair_passive_node_test.py b/tests/cli/fair_passive_node_test.py index 00c61ca0..9644ce71 100644 --- a/tests/cli/fair_passive_node_test.py +++ b/tests/cli/fair_passive_node_test.py @@ -1,7 +1,7 @@ import logging import pathlib -import mock +from unittest import mock from node_cli.cli.passive_fair_node import cleanup_node, init_passive_node, update_node from node_cli.configs import NODE_DATA_PATH, SKALE_DIR @@ -96,7 +96,7 @@ def test_cleanup_node(mocked_g_config): mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), mock.patch('node_cli.fair.common.compose_node_env', return_value={'SCHAIN_NAME': 'test'}), mock.patch( - 'node_cli.core.node.CliMetaManager.get_meta_info', + 'node_cli.utils.meta.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index 7b75c02c..36c2543b 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -104,7 +104,12 @@ def test_sgx_status(): result = run_command_mock('node_cli.utils.helper.api_session.get', resp_mock, sgx) assert result.exit_code == 0 - assert ( - result.output - == '\x1b(0lqqqqqqqqqqqqqqqqqqqwqqqqqqqqqqqqqqqqqqqqqqqqk\x1b(B\n\x1b(0x\x1b(B SGX info \x1b(0x\x1b(B \x1b(0x\x1b(B\n\x1b(0tqqqqqqqqqqqqqqqqqqqnqqqqqqqqqqqqqqqqqqqqqqqqu\x1b(B\n\x1b(0x\x1b(B Server URL \x1b(0x\x1b(B https://127.0.0.1:1026 \x1b(0x\x1b(B\n\x1b(0x\x1b(B SGXWallet Version \x1b(0x\x1b(B 1.50.1-stable.0 \x1b(0x\x1b(B\n\x1b(0x\x1b(B Node SGX keyname \x1b(0x\x1b(B test_keyname \x1b(0x\x1b(B\n\x1b(0x\x1b(B Status HTTPS \x1b(0x\x1b(B True \x1b(0x\x1b(B\n\x1b(0x\x1b(B Status ZMQ \x1b(0x\x1b(B True \x1b(0x\x1b(B\n\x1b(0mqqqqqqqqqqqqqqqqqqqvqqqqqqqqqqqqqqqqqqqqqqqqj\x1b(B\n' # noqa + assert result.output == ( + ' SGX info \n' + '------------------------------------------\n' + 'Server URL https://127.0.0.1:1026\n' + 'SGXWallet Version 1.50.1-stable.0 \n' + 'Node SGX keyname test_keyname \n' + 'Status HTTPS True \n' + 'Status ZMQ True \n' ) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index f24ef487..fc8b6911 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -21,7 +21,7 @@ import pathlib from unittest.mock import MagicMock, patch -import mock +from unittest import mock import pytest import requests @@ -66,7 +66,7 @@ def test_configure_firewall_without_monitoring_option(): configure.reset_mock() result = run_command(configure_firewall, ['--yes', '--monitoring']) assert result.exit_code == 2 - assert 'No such option: --monitoring' in result.output + assert "No such option '--monitoring'" in result.output configure.assert_not_called() @@ -88,7 +88,7 @@ def test_register_node(inited_node, resource_alloc, mocked_g_config): assert ( result.output == 'Node registered in SKALE manager.\nFor more info run < skale node info >\n' - ) # noqa + ) def test_register_node_firewall_failure(inited_node, resource_alloc, mocked_g_config): @@ -403,7 +403,7 @@ def test_restore(request, node_type, node_mode, test_user_conf, mocked_g_config, patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), patch('node_cli.utils.decorators.is_node_inited', return_value=False), patch( - 'node_cli.core.node.CliMetaManager.get_meta_info', + 'node_cli.utils.meta.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.4.0', config_stream='3.0.2'), ), patch('node_cli.operations.base.configure_nftables'), @@ -411,12 +411,12 @@ def test_restore(request, node_type, node_mode, test_user_conf, mocked_g_config, user_conf_path = request.getfixturevalue(test_user_conf).as_posix() result = run_command(restore_node, [backup_path, user_conf_path]) assert result.exit_code == 0 - assert 'Node is restored from backup\n' in result.output # noqa + assert 'Node is restored from backup\n' in result.output assert mock_restore_op.call_args.kwargs.get('backup_run') is True result = run_command(restore_node, [backup_path, user_conf_path, '--no-snapshot']) assert result.exit_code == 0 - assert 'Node is restored from backup\n' in result.output # noqa + assert 'Node is restored from backup\n' in result.output assert mock_restore_op.call_args.kwargs.get('backup_run') is False @@ -547,7 +547,7 @@ def test_cleanup_node(mocked_g_config): mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), mock.patch('node_cli.core.node.compose_node_env', return_value={}), mock.patch( - 'node_cli.core.node.CliMetaManager.get_meta_info', + 'node_cli.utils.meta.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): diff --git a/tests/cli/passive_node_test.py b/tests/cli/passive_node_test.py index 3375c9d6..4059718f 100644 --- a/tests/cli/passive_node_test.py +++ b/tests/cli/passive_node_test.py @@ -20,7 +20,7 @@ import logging import pathlib -import mock +from unittest import mock from node_cli.cli.passive_node import cleanup_node, _init_passive, _update_passive from node_cli.configs import NODE_DATA_PATH, SKALE_DIR @@ -120,7 +120,7 @@ def test_update_passive(passive_user_conf, mocked_g_config): mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), mock.patch( - 'node_cli.core.node.CliMetaManager.get_meta_info', + 'node_cli.utils.meta.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): @@ -140,7 +140,7 @@ def test_cleanup_node(mocked_g_config): mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), mock.patch('node_cli.core.node.compose_node_env', return_value={'SCHAIN_NAME': 'test'}), mock.patch( - 'node_cli.core.node.CliMetaManager.get_meta_info', + 'node_cli.utils.meta.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): diff --git a/tests/cli/wallet_test.py b/tests/cli/wallet_test.py index 306cd823..daeb8964 100644 --- a/tests/cli/wallet_test.py +++ b/tests/cli/wallet_test.py @@ -20,7 +20,7 @@ import requests -from mock import MagicMock, Mock +from unittest.mock import MagicMock, Mock from node_cli.configs import G_CONF_HOME from node_cli.cli.wallet import wallet_info, send @@ -63,7 +63,7 @@ def test_wallet_send(): ['0x00000000000000000000000000000000', '10', '--yes'], ) assert result.exit_code == 0 - assert result.output == 'Funds were successfully transferred\n' # noqa + assert result.output == 'Funds were successfully transferred\n' def test_wallet_send_with_error(): diff --git a/tests/conftest.py b/tests/conftest.py index 8b610d96..9b78fbac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,12 +22,11 @@ import pathlib import shutil import tempfile -from contextlib import contextmanager +from contextlib import contextmanager, suppress import docker -import mock +from unittest import mock import pytest -import redis from node_cli.configs import ( CONTAINER_CONFIG_TMP_PATH, @@ -35,9 +34,7 @@ GLOBAL_SKALE_DIR, META_FILEPATH, NGINX_CONFIG_FILEPATH, - NGINX_CONTAINER_NAME, NODE_DATA_PATH, - REDIS_URI, REMOVED_CONTAINERS_FOLDER_PATH, SCHAIN_NODE_DATA_PATH, ) @@ -45,7 +42,6 @@ from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.ssl import SSL_FOLDER_PATH from node_cli.core.node_options import NodeOptions -from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file from node_cli.utils.node_type import NodeMode from tests.fixtures.settings import ( # noqa: F401 @@ -116,16 +112,12 @@ def simple_image(dclient): dclient.images.build(tag=name, rm=True, nocache=True, path='tests/simple_container') yield name finally: - try: - dclient.images.get(name) - except docker.errors.ImageNotFound: - return - dclient.images.remove(name, force=True) + with suppress(docker.errors.ImageNotFound): + dclient.images.remove(name, force=True) @pytest.fixture() def docker_hc(dclient): - dclient = docker.from_env() return dclient.api.create_host_config( log_config=docker.types.LogConfig(type=docker.types.LogConfig.types.JSON) ) @@ -225,30 +217,6 @@ def passive_node_option(): pass -@pytest.fixture -def dutils(): - return docker_client() - - -@pytest.fixture -def nginx_container(dutils, ssl_folder): - c = None - try: - c = dutils.containers.run( - 'nginx:1.20.2', - name=NGINX_CONTAINER_NAME, - detach=True, - volumes={ssl_folder: {'bind': '/ssl', 'mode': 'ro', 'propagation': 'slave'}}, - ) - yield c - finally: - if c is not None: - try: - c.remove(force=True) - except Exception: - pass - - @pytest.fixture def meta_file_v1(): with open(META_FILEPATH, 'w') as f: @@ -347,25 +315,6 @@ def valid_env_file(valid_env_params): _cleanup_settings() -@pytest.fixture -def mock_chain_response(): - return { - 'jsonrpc': '2.0', - 'id': 1, - 'result': '0x1', - } - - -@pytest.fixture -def mock_networks_metadata(): - return { - 'networks': [ - {'chainId': 1, 'name': 'Mainnet', 'path': 'mainnet'}, - {'chainId': 2, 'name': 'Testnet', 'path': 'testnet'}, - ] - } - - @contextmanager def set_env_var(name, value): old_value = os.environ.get(name) @@ -426,28 +375,6 @@ def fair_user_conf(tmp_path): _cleanup_settings() -@pytest.fixture -def fair_boot_user_conf(tmp_path): - test_env_path = pathlib.Path(tmp_path / 'test-env') - _write_settings(INTERNAL_FAIR_ACTIVE, NODE_FAIR_ACTIVE) - try: - test_env = """ - ENDPOINT=http://localhost:8545 - NODE_VERSION='main' - FILEBEAT_HOST=127.0.0.1:3010 - SGX_URL=http://127.0.0.1 - BLOCK_DEVICE=/dev/sss - ENV_TYPE='devnet' - FAIR_CONTRACTS='test-fair' - """ - with open(test_env_path, 'w') as env_file: - env_file.write(test_env) - yield test_env_path - finally: - test_env_path.unlink() - _cleanup_settings() - - @pytest.fixture def fair_passive_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') @@ -492,9 +419,3 @@ def passive_user_conf(tmp_path): finally: test_env_path.unlink() _cleanup_settings() - - -@pytest.fixture -def redis_client(): - cpool = redis.ConnectionPool.from_url(REDIS_URI) - return redis.Redis(connection_pool=cpool) diff --git a/tests/core/core_checks_test.py b/tests/core/core_checks_test.py index aa986cc7..5c8676ca 100644 --- a/tests/core/core_checks_test.py +++ b/tests/core/core_checks_test.py @@ -2,7 +2,7 @@ import shutil import subprocess -import mock +from unittest import mock import pytest from node_cli.configs import STATIC_PARAMS_FILEPATH @@ -253,7 +253,7 @@ def test_checks_docker_hosts(docker_req): r = checker._check_docker_hosts_option(invalid_config) assert r == ( False, - "Docker daemon hosts is misconfigured. Missing hosts: ['fd://', 'unix:///var/run/skale/docker.sock']", # noqa + "Docker daemon hosts is misconfigured. Missing hosts: ['fd://', 'unix:///var/run/skale/docker.sock']", ) invalid_config = {'hosts': ['http://127.0.0.1:8080']} @@ -261,22 +261,14 @@ def test_checks_docker_hosts(docker_req): assert r == ( False, "Docker daemon hosts is misconfigured. Missing hosts: ['fd://', 'unix:///var/run/skale/docker.sock']", - ) # noqa + ) invalid_config = {'hosts': ['fd://']} r = checker._check_docker_hosts_option(invalid_config) assert r == ( False, "Docker daemon hosts is misconfigured. Missing hosts: ['unix:///var/run/skale/docker.sock']", - ) # noqa - - -def test_checks_docker_pre_post_install_checks(docker_req): - checker = DockerChecker(docker_req) - result = checker.preinstall_check() - assert len(result) == 3 - result = checker.postinstall_check() - assert len(result) == 2 + ) @pytest.fixture diff --git a/tests/core/core_logs_test.py b/tests/core/core_logs_test.py index b3414797..d8546a08 100644 --- a/tests/core/core_logs_test.py +++ b/tests/core/core_logs_test.py @@ -71,7 +71,7 @@ def test_create_logs_dump(backup_func, skale_container, removed_containers_folde content = data_file.readlines() assert content == [ 'Hello, SKALE!\n', - '================================================================================\n', # noqa + '================================================================================\n', 'Hello, SKALE!\n', ] diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 4c707c67..53ca0f9d 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -6,7 +6,7 @@ import docker from docker import errors as docker_errors -import mock +from unittest import mock import pytest import requests @@ -34,7 +34,7 @@ dclient = docker.from_env() -ALPINE_IMAGE_NAME = 'alpine:3.12' +ALPINE_IMAGE_NAME = 'alpine' CMD = 'sleep 60' WRONG_CONTAINERS = [ @@ -271,7 +271,6 @@ def test_init_node(regular_user_conf, no_resource_file): # todo: write new init mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.core.host.prepare_host'), - mock.patch('node_cli.core.host.init_data_dir'), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.core.node.init_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), @@ -292,15 +291,14 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.utils.helper.post_request', resp_mock), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.core.host.init_data_dir'), mock.patch( - 'node_cli.core.node.CliMetaManager.get_meta_info', + 'node_cli.utils.meta.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): with mock.patch( 'node_cli.utils.helper.api_session.get', return_value=safe_update_api_response() - ): # noqa + ): result = update( regular_user_conf.as_posix(), pull_config_for_schain=None, diff --git a/tests/core/core_sgx_test.py b/tests/core/core_sgx_test.py index 618e9606..556a74a0 100644 --- a/tests/core/core_sgx_test.py +++ b/tests/core/core_sgx_test.py @@ -91,7 +91,7 @@ def test_status_describes_certificate(certs_dir, rpc): (-1, 1, False, False, True), ], ) -def test_status_with_legacy_certificate_dates( +def test_status_certificate_dates( certs_dir, rpc, monkeypatch, starts_in, ends_in, expired, not_yet_valid, expires_soon ): wallet = FakeSgxWallet(rpc) @@ -100,16 +100,14 @@ def test_status_with_legacy_certificate_dates( start = now + datetime.timedelta(days=starts_in) end = now + datetime.timedelta(days=ends_in) - class LegacyCertificate: - not_valid_before = start.replace(tzinfo=None) - not_valid_after = end.replace(tzinfo=None) + class DatedCertificate: + not_valid_before_utc = start + not_valid_after_utc = end def __getattr__(self, name): - if name in ('not_valid_before_utc', 'not_valid_after_utc'): - raise AttributeError(name) return getattr(wallet.last_issued, name) - monkeypatch.setattr(sgx, '_load_certificate', lambda _: LegacyCertificate()) + monkeypatch.setattr(sgx, '_load_certificate', lambda _: DatedCertificate()) status = sgx.get_certificate_status(certs_dir) assert status['not_valid_before'] == start.isoformat(timespec='seconds') assert status['not_valid_after'] == end.isoformat(timespec='seconds') diff --git a/tests/core/host/docker_config_test.py b/tests/core/host/docker_config_test.py index 9f53828b..1eb0702c 100644 --- a/tests/core/host/docker_config_test.py +++ b/tests/core/host/docker_config_test.py @@ -9,8 +9,6 @@ import pytest from node_cli.core.docker_config import ( - assert_no_containers, - ContainersExistError, DockerConfigResult, ensure_docker_daemon_config, ensure_docker_service_config_dir, @@ -121,24 +119,6 @@ def test_ensure_run_dir(tmp_dir): assert r == DockerConfigResult.UNCHANGED -@pytest.fixture -def container(dclient): - c = dclient.containers.run('hello-world', detach=True) - try: - yield c - finally: - c.remove(force=True) - - -def test_assert_no_containers(): - assert_no_containers(ignore=('redis',)) - - -def test_assert_no_containers_failed(container): - with pytest.raises(ContainersExistError): - assert_no_containers() - - def test_wait_for_socket_initialization(tmp_dir): socket_path = os.path.join(tmp_dir, 'd.socket') with in_time(8): diff --git a/tests/core/iptables_test.py b/tests/core/iptables_test.py index 19b34255..9da05781 100644 --- a/tests/core/iptables_test.py +++ b/tests/core/iptables_test.py @@ -1,10 +1,9 @@ -import socket import subprocess -import mock +from unittest import mock import pytest -from node_cli.utils.helper import get_ssh_port, get_ssh_ports +from node_cli.utils.helper import get_ssh_ports @pytest.fixture(autouse=True) @@ -12,12 +11,6 @@ def clear_ssh_override(monkeypatch): monkeypatch.delenv('SSH_PORT', raising=False) -def test_service_lookup_compatibility(): - assert get_ssh_port('http') == 80 - with mock.patch.object(socket, 'getservbyname', side_effect=OSError): - assert get_ssh_port('missing-service') == 22 - - @pytest.mark.parametrize( 'config,expected', [ @@ -56,7 +49,6 @@ def test_ssh_port_override(monkeypatch): monkeypatch.setenv('SSH_PORT', '2222') with mock.patch('node_cli.utils.helper.subprocess.run') as run: assert get_ssh_ports() == [2222] - assert get_ssh_port() == 2222 run.assert_not_called() @@ -78,7 +70,7 @@ def test_invalid_ssh_port_override(monkeypatch, value): def test_ssh_detection_failure_does_not_guess(error): with mock.patch('node_cli.utils.helper.subprocess.run', side_effect=error): with pytest.raises(RuntimeError, match='SSH_PORT'): - get_ssh_port() + get_ssh_ports() @pytest.mark.parametrize('config', ['', 'port invalid\n', 'port 65536\n']) diff --git a/tests/core/migration_test.py b/tests/core/migration_test.py deleted file mode 100644 index 585e6fb0..00000000 --- a/tests/core/migration_test.py +++ /dev/null @@ -1,84 +0,0 @@ -import pytest - -from node_cli.migrations.focal_to_jammy import migrate, NFTablesManager - -from node_cli.utils.helper import run_cmd - -CUSTOM_CHAIN_NAME = 'TEST' - - -def add_base_rules(): - run_cmd(f'iptables -N {CUSTOM_CHAIN_NAME}'.split(' ')) - run_cmd('iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT'.split(' ')) - run_cmd('iptables -A INPUT -i lo -j ACCEPT'.split(' ')) - run_cmd('iptables -A INPUT -p tcp --dport 22 -j ACCEPT'.split(' ')) - run_cmd('iptables -A INPUT -p tcp --dport 8080 -j ACCEPT'.split(' ')) - run_cmd('iptables -A INPUT -p tcp --dport 443 -j ACCEPT'.split(' ')) - run_cmd('iptables -A INPUT -p tcp --dport 53 -j ACCEPT'.split(' ')) - run_cmd('iptables -A INPUT -p udp --dport 53 -j ACCEPT'.split(' ')) - run_cmd('iptables -A INPUT -p tcp --dport 3009 -j ACCEPT'.split(' ')) - # non skale related rule - run_cmd(f'iptables -A {CUSTOM_CHAIN_NAME} -p tcp --dport 2222 -j ACCEPT'.split(' ')) - run_cmd('iptables -A INPUT -p tcp --dport 9100 -j ACCEPT'.split(' ')) - run_cmd('iptables -A INPUT -p tcp -j DROP'.split(' ')) - run_cmd('iptables -A INPUT -p udp -j DROP'.split(' ')) - run_cmd('iptables -I INPUT -p icmp --icmp-type destination-unreachable -j ACCEPT'.split(' ')) - run_cmd('iptables -I INPUT -p icmp --icmp-type source-quench -j ACCEPT'.split(' ')) - run_cmd('iptables -I INPUT -p icmp --icmp-type time-exceeded -j ACCEPT'.split(' ')) - - -@pytest.fixture -def base_rules(): - try: - add_base_rules() - yield - finally: - run_cmd(['iptables', '-F']) - - -def test_migration(base_rules): - res = run_cmd(['iptables', '-P', 'INPUT', 'DROP']) - migrate() - res = run_cmd(['iptables', '-S']) - output = res.stdout.decode('utf-8') - assert ( - output - == f'-P INPUT ACCEPT\n-P FORWARD ACCEPT\n-P OUTPUT ACCEPT\n-N {CUSTOM_CHAIN_NAME}\n-A INPUT -p tcp -m tcp --dport 22 -j ACCEPT\n-A INPUT -p udp -m udp --dport 53 -j ACCEPT\n-A {CUSTOM_CHAIN_NAME} -p tcp -m tcp --dport 2222 -j ACCEPT\n' # noqa - ) - nft = NFTablesManager(family='ip', table='filter') - assert nft.get_rules(chain='INPUT') == [ - { - 'family': 'ip', - 'table': 'filter', - 'chain': 'INPUT', - 'handle': 5, - 'expr': [ - { - 'match': { - 'op': '==', - 'left': {'payload': {'protocol': 'tcp', 'field': 'dport'}}, - 'right': 22, - } - }, - {'counter': None}, - {'accept': None}, - ], - }, - { - 'family': 'ip', - 'table': 'filter', - 'chain': 'INPUT', - 'handle': 9, - 'expr': [ - { - 'match': { - 'op': '==', - 'left': {'payload': {'protocol': 'udp', 'field': 'dport'}}, - 'right': 53, - } - }, - {'counter': None}, - {'accept': None}, - ], - }, - ] diff --git a/tests/core/nginx_test.py b/tests/core/nginx_test.py index a867b0b0..0645371e 100644 --- a/tests/core/nginx_test.py +++ b/tests/core/nginx_test.py @@ -2,7 +2,7 @@ from pathlib import Path import pytest -import mock +from unittest import mock from node_cli.core.nginx import ( generate_nginx_config, diff --git a/tests/docker_utils_test.py b/tests/docker_utils_test.py index 8bdcda86..0210d200 100644 --- a/tests/docker_utils_test.py +++ b/tests/docker_utils_test.py @@ -3,7 +3,7 @@ from time import sleep from unittest.mock import MagicMock -import mock +from unittest import mock import pytest from node_cli.utils.docker_utils import ( @@ -69,7 +69,7 @@ def test_save_container_logs(simple_container, tmp_dir_path): 'INFO:__main__:Test 2\n', 'INFO:__main__:Test 3\n', 'INFO:__main__:Test 4\n', - '================================================================================\n', # noqa + '================================================================================\n', 'INFO:__main__:Test 6\n', 'INFO:__main__:Test 7\n', 'INFO:__main__:Test 8\n', diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 6554cd3f..8e925a4d 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -261,9 +261,7 @@ def test_cleanup_logs_success_message( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) @mock.patch('node_cli.fair.active.post_request') -@mock.patch('node_cli.fair.active.is_node_inited', return_value=True) def test_exit_success( - mock_is_inited, mock_post_request, mock_is_user_valid, inited_node, @@ -282,9 +280,7 @@ def test_exit_success( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) @mock.patch('node_cli.fair.active.error_exit') @mock.patch('node_cli.fair.active.post_request') -@mock.patch('node_cli.fair.active.is_node_inited', return_value=True) def test_exit_error( - mock_is_inited, mock_post_request, mock_error_exit, mock_is_user_valid, @@ -301,21 +297,3 @@ def test_exit_error( mock_post_request.assert_called_once_with(blueprint='fair-node', method='exit', json={}) mock_error_exit.assert_called_once_with(error_msg, exit_code=mock.ANY) - - -@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.active.is_node_inited', return_value=False) -def test_exit_not_inited( - mock_is_inited, - mock_is_user_valid, - inited_node, - resource_alloc, - meta_file_v3, - capsys, -): - from node_cli.fair.active import exit - - exit() - - captured = capsys.readouterr() - assert 'Node should be initialized to proceed with operation' in captured.out diff --git a/tests/helper.py b/tests/helper.py index 6af0af14..c9c0591c 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -26,7 +26,7 @@ from click.testing import CliRunner CURRENT_TIMESTAMP = 1594903080 -CURRENT_DATETIME = datetime.datetime.utcfromtimestamp(CURRENT_TIMESTAMP) +CURRENT_DATETIME = datetime.datetime.fromtimestamp(CURRENT_TIMESTAMP, datetime.UTC) BLOCK_DEVICE = os.getenv('BLOCK_DEVICE') diff --git a/tests/resources_test.py b/tests/resources_test.py index 1f545080..d0b5b8d7 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -1,7 +1,7 @@ import json import os -import mock +from unittest import mock import pytest from node_cli.configs import ALLOCATION_FILEPATH, STATIC_PARAMS_FILEPATH @@ -11,7 +11,6 @@ update_resource_allocation, get_cpu_alloc, get_memory_alloc, - verify_disk_size, ) from node_cli.utils.helper import write_json, safe_load_yml @@ -47,7 +46,7 @@ 'max_reserved_storage_bytes': 887999692, 'max_skaled_leveldb_storage_bytes': 2663999078, }, -} # noqa +} DEFAULT_ENV_TYPE = 'devnet' @@ -118,19 +117,7 @@ def test_update_allocation_config(resource_alloc_config): assert json.load(jfile) != INITIAL_CONFIG -def test_get_static_disk_alloc_devnet(params_by_env_type, schain_allocation_data): - env_configs = params_by_env_type['envs']['devnet'] - block_device = '/dev/test' - with mock.patch('node_cli.core.resources.get_disk_size', return_value=SMALL_DISK_SIZE): - with pytest.raises(Exception): - verify_disk_size(block_device, env_configs) - - with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): - verify_disk_size(block_device, env_configs) - - with mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE): - verify_disk_size(block_device, env_configs) - +def test_get_static_disk_alloc_devnet(schain_allocation_data): assert schain_allocation_data[DEFAULT_ENV_TYPE]['disk'] == { 'large': 71039975424, 'medium': 8879996928, @@ -140,18 +127,6 @@ def test_get_static_disk_alloc_devnet(params_by_env_type, schain_allocation_data } -def test_get_static_disk_alloc_mainnet(params_by_env_type): - env_configs = params_by_env_type['envs']['mainnet'] - block_device = '/dev/test' - - with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): - with pytest.raises(Exception): - verify_disk_size(block_device, env_configs) - - with mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE): - verify_disk_size(block_device, env_configs) - - def test_get_cpu_alloc(params_by_env_type): common_config = params_by_env_type['common'] schain_cpu_alloc, ima_cpu_alloc = get_cpu_alloc(common_config) diff --git a/tests/routes_test.py b/tests/routes_test.py index 51b9c5d8..2f77dcdb 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -2,54 +2,10 @@ from node_cli.configs.routes import ( RouteNotFoundException, - get_all_available_routes, get_route, route_exists, ) -ALL_V1_ROUTES = [ - '/api/v1/node/info', - '/api/v1/node/register', - '/api/v1/node/maintenance-on', - '/api/v1/node/maintenance-off', - '/api/v1/node/signature', - '/api/v1/node/send-tg-notification', - '/api/v1/node/exit/start', - '/api/v1/node/exit/status', - '/api/v1/node/set-domain-name', - '/api/v1/node/update-safe', - '/api/v1/health/containers', - '/api/v1/health/schains', - '/api/v1/info/sgx', - '/api/v1/info/sgx-options', - '/api/v1/schains/config', - '/api/v1/schains/list', - '/api/v1/schains/dkg-statuses', - '/api/v1/schains/firewall-rules', - '/api/v1/schains/repair', - '/api/v1/schains/get', - '/api/v1/ssl/status', - '/api/v1/ssl/upload', - '/api/v1/wallet/info', - '/api/v1/wallet/send-eth', - '/api/v1/fair-node/info', - '/api/v1/fair-node/register', - '/api/v1/fair-node/set-domain-name', - '/api/v1/fair-node/change-ip', - '/api/v1/fair-node/exit', - '/api/v1/fair-chain/record', - '/api/v1/fair-chain/checks', - '/api/v1/fair-node-passive/setup', - '/api/v1/fair-staking/add-receiver', - '/api/v1/fair-staking/remove-receiver', - '/api/v1/fair-staking/set-fee-rate', - '/api/v1/fair-staking/request-fees', - '/api/v1/fair-staking/request-send-fees', - '/api/v1/fair-staking/claim-request', - '/api/v1/fair-staking/get-earned-fee-amount', - '/api/v1/fair-staking/get-exit-requests', -] - def test_route_exists(): assert route_exists('node', 'signature', 'v1') @@ -57,12 +13,7 @@ def test_route_exists(): def test_get_route(): - repair_route = get_route('schains', 'repair') - assert repair_route == '/api/v1/schains/repair' + assert get_route('info', 'containers') == '/api/v1/info/containers' with pytest.raises(RouteNotFoundException): - get_route('schains', 'refair') - - -def test_get_all_available_routes(): - assert get_all_available_routes() == ALL_V1_ROUTES + get_route('health', 'containers') diff --git a/tests/simple_container/Dockerfile b/tests/simple_container/Dockerfile index 8277910b..5df4c798 100644 --- a/tests/simple_container/Dockerfile +++ b/tests/simple_container/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-buster +FROM python:3.14.10-slim-trixie RUN mkdir /app WORKDIR /app diff --git a/tests/utils/api_auth_test.py b/tests/utils/api_auth_test.py index 0549d5b6..3224dc90 100644 --- a/tests/utils/api_auth_test.py +++ b/tests/utils/api_auth_test.py @@ -83,7 +83,7 @@ def test_missing_token_supports_upgrade_from_old_api(token_path): assert not token_path.exists() -def test_headers_sent_for_get_post_and_upload(token_path): +def test_headers_sent_for_get_and_post(token_path): api_auth.ensure_api_token() expected = f'Bearer {api_auth.read_api_token()}' with requests_mock.Mocker() as mock: @@ -93,15 +93,9 @@ def test_headers_sent_for_get_post_and_upload(token_path): mock.post( helper.construct_url('/api/v1/wallet/send-eth'), json={'status': 'ok', 'payload': {}} ) - mock.post(helper.construct_url('/api/v1/ssl/upload'), json={'status': 'ok', 'payload': {}}) assert helper.get_request('node', 'signature') == ('ok', {}) assert helper.post_request('wallet', 'send-eth', json={'amount': 1}) == ('ok', {}) - assert helper.post_request('ssl', 'upload', files={'ssl_cert': ('cert', b'cert')}) == ( - 'ok', - {}, - ) assert all(req.headers['Authorization'] == expected for req in mock.request_history) - assert 'multipart/form-data' in mock.last_request.headers['Content-Type'] def test_http_errors_reach_cli(token_path): diff --git a/tests/utils/decorators_test.py b/tests/utils/decorators_test.py index 4c659984..ff12eaf2 100644 --- a/tests/utils/decorators_test.py +++ b/tests/utils/decorators_test.py @@ -1,4 +1,4 @@ -import mock +from unittest import mock import pytest from node_cli.utils.global_config import generate_g_config_file diff --git a/tests/utils/global_config_test.py b/tests/utils/global_config_test.py index cb4fb05a..ed4a81d8 100644 --- a/tests/utils/global_config_test.py +++ b/tests/utils/global_config_test.py @@ -1,5 +1,5 @@ import os -import mock +from unittest import mock from node_cli.utils.global_config import read_g_config, generate_g_config_file from node_cli.utils.helper import write_json, get_system_user, is_user_valid, get_g_conf_user from node_cli.configs import GLOBAL_SKALE_DIR, GLOBAL_SKALE_CONF_FILEPATH diff --git a/text.yml b/text.yml index 652adf7e..a4955255 100644 --- a/text.yml +++ b/text.yml @@ -11,8 +11,6 @@ common: json: Show data in JSON format node: - base: "SKALE node commands" - info: "Info about SKALE node" registered: |- Node registered in SKALE manager. For more info run < skale node info > @@ -80,10 +78,8 @@ lvmpy: fair: node: repair: - help: Repair Fair chain node warning: Are you sure you want to repair Fair chain node? In rare cases may cause data loss and require additional maintenance snapshot: IP of the node to take snapshot from (put "any" to use any available node) - repair_requested: Repair mode is requested not_inited: Node should be initialized to proceed with operation setup: