From d16fd44e9d54e2381eafb9192f2f0d38d2223b00 Mon Sep 17 00:00:00 2001 From: "beplus-agent-claude[bot]" <300068699+beplus-agent-claude[bot]@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:09:06 +0000 Subject: [PATCH] `be` tries a download again when GitHub answers 504 (v0.12.0) GitHub answers release downloads with a 504 now and then, most often to Actions runners, and one 504 failed the whole install. Every request be makes now goes through fetch, which retries a 408, 429, 500, 502, 503 or 504, a host that does not resolve or answer, a dropped connection or a timeout: up to 5 attempts, waiting 5, 10, 20 and 40 seconds. A 404 still fails at once. BE_DOWNLOAD_ATTEMPTS and BE_DOWNLOAD_RETRY_DELAY tune it; each retry is logged on stderr, --quiet or not. With wget, be makes one try at a time (--tries=1) and reads the status from --server-response, so a 503 is retried too and a 504 is not retried twenty times by wget as well. test/tests/retries.bats runs against flaky-mirror.js, a local server that answers 504 or drops the connection on demand. Co-Authored-By: Igor Lamos --- CHANGELOG.md | 18 ++- bin/be | 207 ++++++++++++++++++++++----- package-lock.json | 4 +- package.json | 2 +- test/tests/flaky-mirror.js | 68 +++++++++ test/tests/retries.bats | 235 +++++++++++++++++++++++++++++++ test/tests/shared-functions.bash | 68 +++++++-- 7 files changed, 558 insertions(+), 44 deletions(-) create mode 100644 test/tests/flaky-mirror.js create mode 100644 test/tests/retries.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 719b4ed..58a8a6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -https://github.com/beplus/be/compare/v0.11.0...dev +https://github.com/beplus/be/compare/v0.12.0...dev + +## [v0.12.0] (11/08/2026) + +`be` tries a download again when it fails in a way that may pass, as `actions/setup-node` does +for Node: GitHub, which hosts the beplus CLI releases, answers 504 now and then, most often to +Actions runners, and one 504 used to fail the whole install. Every request `be` makes is covered: +the preflight of the tarball, its `SHA256SUMS`, the tarball itself and the release index. An HTTP +408, 429, 500, 502, 503 or 504, a host that does not resolve or answer, a dropped connection or a +timeout is tried again, up to 5 attempts in all, waiting 5, 10, 20 and then 40 seconds; anything +else, such as a 404, fails at once, as before. `BE_DOWNLOAD_ATTEMPTS` sets the number of attempts +(1 turns retrying off) and `BE_DOWNLOAD_RETRY_DELAY` the first wait in seconds, which doubles for +each later one; `be doctor` shows both. Each retry is logged on stderr, `--quiet` or not, as +`retrying : answered HTTP 504; attempt 2 of 5 in 5 seconds`, and a last failure as +`gave up`. A failed attempt leaves nothing behind: a release index or `SHA256SUMS` is output only +once it has arrived whole. With wget, `be` makes one try at a time (`--tries=1`) and does the +retrying itself, so a 503 is retried too and a 504 is not retried twenty times over. ## [v0.11.0] (10/08/2026) diff --git a/bin/be b/bin/be index b8e8059..25641dc 100755 --- a/bin/be +++ b/bin/be @@ -25,6 +25,16 @@ verbose_log() { fi } +# +# warn_log +# Like verbose_log, but shown even with --quiet: for trouble be works around, such as a download +# it is about to try again. +# + +warn_log() { + >&2 printf " ${SGR_CYAN}%10s${SGR_RESET} : ${SGR_FAINT}%s${SGR_RESET}\n" "$1" "$2" +} + # # Exit with the given # @@ -64,7 +74,7 @@ function n_grep() { # Setup and state # -VERSION="v0.11.0" +VERSION="v0.12.0" BE_PREFIX="${BE_PREFIX-/usr/local}" BE_PREFIX=${BE_PREFIX%/} @@ -103,7 +113,7 @@ if [[ -t 1 ]]; then else CURL_OPTIONS+=( "--silent" ) fi -WGET_OPTIONS=( "-q" "-O-" ) +WGET_OPTIONS=( "-q" ) # Set by set_active_beplus g_active_beplus= @@ -142,6 +152,21 @@ else readonly SGR_CYAN= fi +# A download that fails in a way the next attempt may not is tried again (see fetch), up to +# BE_DOWNLOAD_ATTEMPTS attempts in all: GitHub, which hosts the beplus CLI releases, answers 504 +# now and then, most often to Actions runners. The first retry waits BE_DOWNLOAD_RETRY_DELAY +# seconds, and each later one twice as long as the one before, so the defaults wait 5, 10, 20 and +# 40 seconds. BE_DOWNLOAD_ATTEMPTS=1 turns retrying off. + +BE_DOWNLOAD_ATTEMPTS="${BE_DOWNLOAD_ATTEMPTS:-5}" +BE_DOWNLOAD_RETRY_DELAY="${BE_DOWNLOAD_RETRY_DELAY:-5}" +# No leading zero: arithmetic would read 08 as a bad octal number. +[[ "${BE_DOWNLOAD_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]] \ + || abort "BE_DOWNLOAD_ATTEMPTS is '${BE_DOWNLOAD_ATTEMPTS}', not a whole number of 1 or more (1 turns retrying off)" +[[ "${BE_DOWNLOAD_RETRY_DELAY}" =~ ^(0|[1-9][0-9]*)$ ]] \ + || abort "BE_DOWNLOAD_RETRY_DELAY is '${BE_DOWNLOAD_RETRY_DELAY}', not a whole number of seconds" +readonly BE_DOWNLOAD_ATTEMPTS BE_DOWNLOAD_RETRY_DELAY + # # set_arch to override $(uname -a) # @@ -629,7 +654,7 @@ install() { # sha256sum writes it; v1.0.5 lists `: `. local sums_url="${url%/*}/SHA256SUMS" local sums - sums="$(do_get_index "${sums_url}")" || abort "failed to download SHA256SUMS for $version (${sums_url})" + sums="$(fetch text "${sums_url}")" || abort "failed to download SHA256SUMS for $version (${sums_url})" local checksum="$(echo "${sums}" | awk -v name="${url##*/}" '$2 == name { print $1; exit } $1 == name ":" { print $2; exit }')" [[ -n "${checksum}" ]] || abort "no checksum for ${url##*/} in ${sums_url}" @@ -641,7 +666,7 @@ install() { log fetch "$url" local archive="${dir}/${url##*/}" - if ! do_get "${url}" > "${archive}"; then + if ! fetch file "${url}" "${archive}"; then rm -f "${archive}" abort "failed to download archive for $version" fi @@ -675,19 +700,158 @@ set_quiet() { # # Synopsis: do_get [option...] url -# Call curl or wget with combination of global and passed options. +# Call curl or wget with combination of global and passed options, once: doctor shows the command. +# Downloads go through fetch, which tries again. # function do_get() { if command -v curl &> /dev/null; then curl "${CURL_OPTIONS[@]}" "$@" elif command -v wget &> /dev/null; then - wget "${WGET_OPTIONS[@]}" "$@" + wget "${WGET_OPTIONS[@]}" -O- "$@" + else + abort "curl or wget command required" + fi +} + +# +# Synopsis: fetch head|text|file url [file] +# Fetch with curl, or with wget where there is no curl: +# head ask for the headers only, to check that it is there +# text output the body, such as a release index or SHA256SUMS +# file save the body as , with a progress bar where be shows one +# An attempt that fails in a way the next one may not (is_transient_failure) is tried again, up +# to BE_DOWNLOAD_ATTEMPTS attempts in all, the waits doubling from BE_DOWNLOAD_RETRY_DELAY +# seconds. Returns the status of the last attempt. +# + +function fetch() { + local mode="$1" + local url="$2" + local body="/dev/null" + if [[ "${mode}" == "file" ]]; then + body="$3" + elif [[ "${mode}" == "text" ]]; then + # Output only once an attempt succeeds, so that a failed one leaves no part of a body behind. + body="$(mktemp)" || return 1 + fi + + local attempt=1 + local delay="${BE_DOWNLOAD_RETRY_DELAY}" + local code + local status + while true; do + # Start the body afresh: curl leaves the file as it was when an attempt receives nothing. + [[ "${body}" != "/dev/null" ]] && : > "${body}" + code="$(fetch_attempt "${mode}" "${url}" "${body}")" + status=$? + [[ "${status}" -eq 0 ]] && break + is_transient_failure "${status}" "${code}" || break + if [[ "${attempt}" -ge "${BE_DOWNLOAD_ATTEMPTS}" ]]; then + if [[ "${attempt}" -gt 1 ]]; then + warn_log "gave up" "${url} $(display_fetch_failure "${status}" "${code}") on attempt ${attempt} of ${BE_DOWNLOAD_ATTEMPTS}" + fi + break + fi + local unit="seconds" + [[ "${delay}" -eq 1 ]] && unit="second" + warn_log "retrying" "${url} $(display_fetch_failure "${status}" "${code}"); attempt $(( attempt + 1 )) of ${BE_DOWNLOAD_ATTEMPTS} in ${delay} ${unit}" + sleep "${delay}" + attempt=$(( attempt + 1 )) + delay=$(( delay * 2 )) + done + + if [[ "${mode}" == "text" ]]; then + [[ "${status}" -eq 0 ]] && cat "${body}" + rm -f "${body}" + fi + return "${status}" +} + +# +# Synopsis: fetch_attempt head|text|file url body +# One request for , as fetch describes, with the body written into the file . Outputs +# the HTTP status of the last response (000 when none came) and returns curl's or wget's status. +# + +function fetch_attempt() { + local mode="$1" + local url="$2" + local body="$3" + if command -v curl &> /dev/null; then + local options=( "${CURL_OPTIONS[@]}" ) + case "${mode}" in + head) options+=( "--silent" "--head" ) ;; + # --silent to suppress progress et al + text) options=( "--silent" "--compressed" "${options[@]}" ) ;; + esac + curl "${options[@]}" --output "${body}" --write-out '%{http_code}' "${url}" + elif command -v wget &> /dev/null; then + # One try: fetch does the trying again (GNU wget would retry a 504 twenty times itself, but + # not a 503). --server-response prints each response's status line on stderr. + local options=( "${WGET_OPTIONS[@]}" "--tries=1" "--server-response" ) + if [[ "${mode}" == "head" ]]; then + options+=( "--spider" ) + else + options+=( "-O" "${body}" ) + fi + local response + response="$(wget "${options[@]}" "${url}" 2>&1)" + local status=$? + echo "${response}" | awk '$1 ~ /^HTTP\// { code = $2 } END { print (code == "" ? "000" : code) }' + return "${status}" else abort "curl or wget command required" fi } +# +# Synopsis: is_transient_failure status http-code +# Whether an attempt that failed with curl's or wget's , after the server answered +# (000 for no answer), may succeed when made again: the server answered 408, 429, 500, +# 502, 503 or 504, or it gave no full answer, because its name did not resolve, it could not be +# reached, the connection dropped or it timed out. Any other answer, such as a 404, is final, and +# so is a failure of be's own, such as a file it cannot write. +# + +function is_transient_failure() { + case "$2" in + 408|429|500|502|503|504) return 0 ;; + [45][0-9][0-9]) return 1 ;; + esac + if command -v curl &> /dev/null; then + # https://curl.se/libcurl/c/libcurl-errors.html: the proxy or host did not resolve (5, 6), no + # connection (7), an HTTP/2 or HTTP/3 error (16, 92, 95), a partial body (18), a timeout + # (28), a failed TLS handshake (35), an empty reply (52), sending or receiving failed (55, 56). + case "$1" in + 5|6|7|16|18|28|35|52|55|56|92|95) return 0 ;; + esac + else + # GNU wget's "network failure". BusyBox wget exits 1 for any failure, so there only what the + # server answered is tried again. + [[ "$1" -eq 4 ]] && return 0 + fi + return 1 +} + +# +# Synopsis: display_fetch_failure status http-code +# How a fetch attempt failed, for a log message. +# + +function display_fetch_failure() { + case "$2" in + [45][0-9][0-9]) echo "answered HTTP $2" ;; + *) + if command -v curl &> /dev/null; then + echo "failed (curl exit $1)" + else + echo "failed (wget exit $1)" + fi + ;; + esac +} + # # Synopsis: display_sha256 file # sha256sum where it exists, otherwise shasum, which macOS has. Both hash stdin rather than a @@ -705,23 +869,6 @@ function display_sha256() { echo "${line%% *}" } -# -# Synopsis: do_get_index [option...] url -# Call curl or wget with combination of global and passed options, -# with options tweaked to be more suitable for getting index. -# - -function do_get_index() { - if command -v curl &> /dev/null; then - # --silent to suppress progress et al - curl --silent --compressed "${CURL_OPTIONS[@]}" "$@" - elif command -v wget &> /dev/null; then - wget "${WGET_OPTIONS[@]}" "$@" - else - abort "curl or wget command required" - fi -} - # # Synopsis: remove_versions version ... # @@ -819,13 +966,8 @@ function exec_with_version() { # function is_ok() { - # Note: both curl and wget can follow redirects, as present on some mirrors (e.g. https://npm.taobao.org/mirrors/node). - # The output is complicated with redirects, so keep it simple and use command status rather than parse output. - if command -v curl &> /dev/null; then - do_get --silent --head "$1" > /dev/null || return 1 - else - do_get --spider "$1" > /dev/null || return 1 - fi + # Both curl and wget follow redirects, which GitHub answers release downloads with. + fetch head "$1" } # @@ -1044,7 +1186,7 @@ function get_latest_resolved_version() { display_remote_index() { local index_url="${g_mirror_url}/index.tab" # tail to remove header line - do_get_index "${index_url}" | tail -n +2 | cut -f 1,3,10 + fetch text "${index_url}" | tail -n +2 | cut -f 1,3,10 if [[ "${PIPESTATUS[0]}" -ne 0 ]]; then # Reminder: abort will only exit subshell, but consistent error display abort "failed to download version index (${index_url})" @@ -1132,7 +1274,7 @@ function display_remote_versions() { else [0, 0, 0, 0, 0, []] end;' - do_get_index "${index_url}" \ + fetch text "${index_url}" \ | jq -r "${jq_semver_key} [ ${jq_release_filter} | .name ] | sort_by(semver_key) | reverse | .[]" \ | n_grep -E "${match}" \ | awk "NR<=${match_count}" \ @@ -1237,6 +1379,7 @@ function show_diagnostics() { echo "install destination: ${BE_PREFIX}" [[ -n "${BE_PREFIX}" ]] && echo "PATH: ${PATH}" echo "ls-remote max matches: ${BE_MAX_REMOTE_MATCHES}" + echo "download attempts: ${BE_DOWNLOAD_ATTEMPTS}, the first retry after ${BE_DOWNLOAD_RETRY_DELAY} seconds" printf "\nProxy\n" # disable "var is referenced but not assigned": https://github.com/koalaman/shellcheck/wiki/SC2154 diff --git a/package-lock.json b/package-lock.json index 519ad53..ab48681 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@beplus/be", - "version": "0.11.0", + "version": "0.12.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@beplus/be", - "version": "0.11.0", + "version": "0.12.0", "license": "MIT", "os": [ "!win32" diff --git a/package.json b/package.json index 691d67b..92a74d7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@beplus/be", "description": "Interactively Manage beplus CLI Versions", - "version": "0.11.0", + "version": "0.12.0", "author": "Igor Lamos ", "homepage": "https://github.com/beplus/be", "bugs": "https://github.com/beplus/be", diff --git a/test/tests/flaky-mirror.js b/test/tests/flaky-mirror.js new file mode 100644 index 0000000..aedc1c1 --- /dev/null +++ b/test/tests/flaky-mirror.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node +// +// A mirror that fails on purpose, for the retry tests: it serves the files under over HTTP, +// but a request for / first gets what /.plan holds. A plan has one line +// per answer still to give, each a status ("504"), a method and a status ("HEAD 504"), or "drop" +// to close the connection without answering. The first line is used up by the next request it +// applies to; once the plan is empty, the file is served. Every request is appended to +// /requests.log as " ". +// +// Usage: node flaky-mirror.js +// Listens on a free port of 127.0.0.1, and writes that port to once it is ready. + +"use strict"; + +const fs = require("fs"); +const http = require("http"); +const path = require("path"); + +const [root, portFile] = process.argv.slice(2); + +// The answer the plan holds for this request, if any, using it up. +function planned(file, method) { + const planFile = `${file}.plan`; + if (!fs.existsSync(planFile)) return null; + const lines = fs.readFileSync(planFile, "utf8").split("\n").filter((line) => line.trim() !== ""); + if (lines.length === 0) return null; + const words = lines[0].trim().split(/\s+/); + const [forMethod, answer] = words.length === 1 ? [null, words[0]] : words; + if (forMethod !== null && forMethod !== method) return null; + fs.writeFileSync(planFile, lines.slice(1).map((line) => `${line}\n`).join("")); + return answer; +} + +const server = http.createServer((request, response) => { + const pathname = decodeURIComponent(new URL(request.url, "http://127.0.0.1").pathname); + const file = path.join(root, pathname); + const log = (status) => fs.appendFileSync(path.join(root, "requests.log"), `${request.method} ${pathname} ${status}\n`); + + const answer = planned(file, request.method); + if (answer === "drop") { + log("drop"); + request.socket.destroy(); + return; + } + if (answer !== null) { + log(answer); + response.writeHead(Number(answer), { "Content-Type": "text/plain" }); + response.end(request.method === "HEAD" ? undefined : `${answer}\n`); + return; + } + + if (!fs.existsSync(file) || !fs.statSync(file).isFile()) { + log(404); + response.writeHead(404, { "Content-Type": "text/plain" }); + response.end(request.method === "HEAD" ? undefined : "404\n"); + return; + } + log(200); + const body = fs.readFileSync(file); + response.writeHead(200, { "Content-Length": body.length, "Content-Type": "application/octet-stream" }); + response.end(request.method === "HEAD" ? undefined : body); +}); + +server.listen(0, "127.0.0.1", () => { + // Written whole, then renamed, so that a reader never sees it half-written. + fs.writeFileSync(`${portFile}.tmp`, String(server.address().port)); + fs.renameSync(`${portFile}.tmp`, portFile); +}); diff --git a/test/tests/retries.bats b/test/tests/retries.bats new file mode 100644 index 0000000..d369346 --- /dev/null +++ b/test/tests/retries.bats @@ -0,0 +1,235 @@ +#!/usr/bin/env bats +# +# A download that fails in a way the next attempt may not, such as the 504 GitHub answers now and +# then, is tried again, up to BE_DOWNLOAD_ATTEMPTS attempts in all, the first retry waiting +# BE_DOWNLOAD_RETRY_DELAY seconds and each later one twice as long. Any other failure, such as a +# 404, is final at once. These run against flaky-mirror.js, which gives each request for a file +# the next answer .plan holds before serving the file, and without waiting unless a test +# sets a delay. + +load shared-functions +load '../../node_modules/bats-support/load' +load '../../node_modules/bats-assert/load' + + +function setup() { + unset_n_env + setup_tmp_prefix + setup_flaky_mirror 2.7.0 + touch "${TMP_MIRROR_DIR}/requests.log" + export BE_DOWNLOAD_RETRY_DELAY=0 + TARBALL="beplus-cli-v2.7.0-$(tarball_platform).tar.gz" + TARBALL_PATH="v2.7.0/${TARBALL}" + SUMS_PATH="v2.7.0/SHA256SUMS" + CACHED="${BE_PREFIX}/be/versions/beplus/2.7.0" +} + + +function teardown() { + stop_flaky_mirror + rm -rf "${TMP_PREFIX_DIR}" "${TMP_MIRROR_DIR}" "${TMP_SHIM_DIR}" +} + + +# Synopsis: plan ... +# The answers the mirror gives the next requests for before serving it: a status (504), a +# method and a status (HEAD 503), or drop. +function plan() { + local path="$1" + shift + printf '%s\n' "$@" > "${TMP_MIRROR_DIR}/${path}.plan" +} + + +# Synopsis: requests +# The statuses the mirror answered the requests for with, in order. +function requests() { + awk -v method="$1" -v path="/$2" '$1 == method && $2 == path { print $3 }' "${TMP_MIRROR_DIR}/requests.log" \ + | paste -s -d ' ' - +} + + +# Synopsis: setup_path_without_curl +# A directory linking every command on PATH but curl, to run be with as its PATH, so that it +# downloads with wget. +function setup_path_without_curl() { + command -v wget > /dev/null || skip "needs wget" + TMP_SHIM_DIR="$(mktemp -d)" + local dirs + local dir + IFS=':' read -ra dirs <<< "${PATH}" + for dir in "${dirs[@]}"; do + # ln links no name twice, so the command first on PATH wins, as in a lookup. + if [[ -d "${dir}" ]]; then + ln -s "${dir}"/* "${TMP_SHIM_DIR}/" 2> /dev/null || true + fi + done + rm -f "${TMP_SHIM_DIR}/curl" +} + + +@test "a tarball download answered with a 504 is tried again, and installed" { + plan "${TARBALL_PATH}" "GET 504" "GET 504" + + run be 2.7.0 + assert_success + assert_output --partial "retrying : ${BE_MIRROR}/${TARBALL_PATH} answered HTTP 504; attempt 2 of 5 in 0 seconds" + assert_output --partial "retrying : ${BE_MIRROR}/${TARBALL_PATH} answered HTTP 504; attempt 3 of 5 in 0 seconds" + assert_equal "$(requests GET "${TARBALL_PATH}")" "504 504 200" + assert_equal "$(beplus --version | cli_version)" "2.7.0" + [ ! -e "${CACHED}/${TARBALL}" ] +} + + +@test "the preflight, SHA256SUMS and the tarball are each tried again" { + plan "${TARBALL_PATH}" "HEAD 503" "GET 500" + plan "${SUMS_PATH}" 502 429 408 + + run be 2.7.0 + assert_success + assert_equal "$(requests HEAD "${TARBALL_PATH}")" "503 200" + assert_equal "$(requests GET "${SUMS_PATH}")" "502 429 408 200" + assert_equal "$(requests GET "${TARBALL_PATH}")" "500 200" + assert_equal "$(beplus --version | cli_version)" "2.7.0" +} + + +@test "a dropped connection is tried again" { + plan "${TARBALL_PATH}" "GET drop" + + run be 2.7.0 + assert_success + assert_output --regexp "retrying : ${BE_MIRROR}/${TARBALL_PATH} failed \((curl exit 52|wget exit 4)\); attempt 2 of 5" + assert_equal "$(requests GET "${TARBALL_PATH}")" "drop 200" + assert_equal "$(beplus --version | cli_version)" "2.7.0" +} + + +@test "a 404 is not tried again" { + rm "${TMP_MIRROR_DIR}/${SUMS_PATH}" + + run be 2.7.0 + assert_failure + assert_output --partial "failed to download SHA256SUMS for 2.7.0 (${BE_MIRROR}/${SUMS_PATH})" + refute_output --partial "retrying" + assert_equal "$(requests GET "${SUMS_PATH}")" "404" +} + + +@test "a version the mirror lacks fails its preflight at once" { + run be 2.8.0 + assert_failure + assert_output --partial "download preflight failed for '2.8.0'" + refute_output --partial "retrying" + assert_equal "$(requests HEAD "v2.8.0/beplus-cli-v2.8.0-$(tarball_platform).tar.gz")" "404" +} + + +@test "be gives up after BE_DOWNLOAD_ATTEMPTS attempts" { + export BE_DOWNLOAD_ATTEMPTS=3 + plan "${SUMS_PATH}" 504 504 504 504 + + run be 2.7.0 + assert_failure + assert_output --partial "gave up : ${BE_MIRROR}/${SUMS_PATH} answered HTTP 504 on attempt 3 of 3" + assert_output --partial "failed to download SHA256SUMS for 2.7.0" + assert_equal "$(requests GET "${SUMS_PATH}")" "504 504 504" + [ ! -e "${BE_PREFIX}/bin/beplus" ] +} + + +@test "a tarball download that fails for good leaves no part of it behind, and a later install works" { + export BE_DOWNLOAD_ATTEMPTS=2 + plan "${TARBALL_PATH}" "GET drop" "GET 504" + + run be 2.7.0 + assert_failure + assert_output --partial "failed to download archive for 2.7.0" + [ ! -e "${CACHED}/${TARBALL}" ] + [ ! -e "${BE_PREFIX}/bin/beplus" ] + + run be 2.7.0 + assert_success + assert_equal "$(beplus --version | cli_version)" "2.7.0" +} + + +@test "BE_DOWNLOAD_ATTEMPTS=1 makes one attempt" { + export BE_DOWNLOAD_ATTEMPTS=1 + plan "${SUMS_PATH}" 504 + + run be 2.7.0 + assert_failure + refute_output --partial "retrying" + refute_output --partial "gave up" + assert_equal "$(requests GET "${SUMS_PATH}")" "504" +} + + +@test "the wait doubles from BE_DOWNLOAD_RETRY_DELAY" { + export BE_DOWNLOAD_RETRY_DELAY=1 + plan "${SUMS_PATH}" 504 504 + + run be 2.7.0 + assert_success + assert_line --regexp "attempt 2 of 5 in 1 second$" + assert_line --regexp "attempt 3 of 5 in 2 seconds$" +} + + +@test "retries are shown with --quiet too" { + plan "${SUMS_PATH}" 503 + + run be --quiet 2.7.0 + assert_success + assert_output --partial "retrying : ${BE_MIRROR}/${SUMS_PATH} answered HTTP 503" +} + + +@test "the release index is tried again, and be --latest outputs only the version" { + echo '[{"name": "v2.7.0", "tag_name": "v2.7.0", "prerelease": false, "draft": false}]' > "${TMP_MIRROR_DIR}/releases.json" + plan releases.json 503 drop + export BE_RELEASE_INDEX_URL="${BE_MIRROR}/releases.json" + + assert_equal "$(be --latest 2> "${TMP_PREFIX_DIR}/stderr")" "2.7.0" + run cat "${TMP_PREFIX_DIR}/stderr" + assert_output --partial "retrying : ${BE_MIRROR}/releases.json answered HTTP 503; attempt 2 of 5" + assert_output --partial "retrying : ${BE_MIRROR}/releases.json failed (" + assert_equal "$(requests GET releases.json)" "503 drop 200" +} + + +@test "with wget, a 503, a 504 and a dropped connection are tried again, and a 404 is not" { + setup_path_without_curl + plan "${TARBALL_PATH}" "HEAD 503" "GET 504" + plan "${SUMS_PATH}" drop + + run env PATH="${TMP_SHIM_DIR}" be 2.7.0 + assert_success + assert_output --partial "retrying : ${BE_MIRROR}/${TARBALL_PATH} answered HTTP 503; attempt 2 of 5" + assert_output --partial "retrying : ${BE_MIRROR}/${TARBALL_PATH} answered HTTP 504; attempt 2 of 5" + assert_output --partial "retrying : ${BE_MIRROR}/${SUMS_PATH} failed (wget exit 4); attempt 2 of 5" + # One request each: wget does not retry them itself as well. + assert_equal "$(requests HEAD "${TARBALL_PATH}")" "503 200" + assert_equal "$(requests GET "${SUMS_PATH}")" "drop 200" + assert_equal "$(requests GET "${TARBALL_PATH}")" "504 200" + assert_equal "$(beplus --version | cli_version)" "2.7.0" + + run env PATH="${TMP_SHIM_DIR}" be 2.8.0 + assert_failure + refute_output --partial "retrying" + assert_equal "$(requests HEAD "v2.8.0/beplus-cli-v2.8.0-$(tarball_platform).tar.gz")" "404" +} + + +@test "a BE_DOWNLOAD_ATTEMPTS or BE_DOWNLOAD_RETRY_DELAY that is not a whole number is refused" { + run env BE_DOWNLOAD_ATTEMPTS=0 be 2.7.0 + assert_failure + assert_output --partial "BE_DOWNLOAD_ATTEMPTS is '0', not a whole number of 1 or more" + + run env BE_DOWNLOAD_RETRY_DELAY=08 be 2.7.0 + assert_failure + assert_output --partial "BE_DOWNLOAD_RETRY_DELAY is '08', not a whole number of seconds" + + assert_equal "$(cat "${TMP_MIRROR_DIR}/requests.log")" "" +} diff --git a/test/tests/shared-functions.bash b/test/tests/shared-functions.bash index 067301e..3e00185 100644 --- a/test/tests/shared-functions.bash +++ b/test/tests/shared-functions.bash @@ -12,6 +12,8 @@ function unset_n_env(){ unset BE_DOWNLOAD_MIRROR unset BE_MAX_REMOTE_MATCHES unset BE_RELEASE_INDEX_URL + unset BE_DOWNLOAD_ATTEMPTS + unset BE_DOWNLOAD_RETRY_DELAY unset HTTP_USER unset HTTP_PASSWORD unset GREP_OPTIONS @@ -35,17 +37,14 @@ function setup_tmp_prefix() { } -# Create a local mirror laid out as the beplus CLI release publishes one, and point be at it: -# for each given, v/ holds this machine's tarball, with a stub beplus that -# prints the version banner, and SHA256SUMS beside it. The mirror is a file:// URL, which be reads -# with curl only (wget fetches http, https and ftp), so without curl the test is skipped. -# Usage: setup_tmp_mirror version... +# Create a local mirror laid out as the beplus CLI release publishes one: for each +# given, v/ holds this machine's tarball, with a stub beplus that prints the version +# banner, and SHA256SUMS beside it. +# Usage: make_tmp_mirror version... # Globals: # TMP_MIRROR_DIR -# BE_MIRROR -function setup_tmp_mirror() { - command -v curl > /dev/null || skip "the local mirror is a file:// URL, which needs curl" +function make_tmp_mirror() { TMP_MIRROR_DIR="$(mktemp -d)" [ -d "${TMP_MIRROR_DIR}" ] || exit 2 export TMP_MIRROR_DIR @@ -64,10 +63,63 @@ function setup_tmp_mirror() { (cd "${release}" && shasum -a 256 "${tarball}" > SHA256SUMS) fi done +} + +# make_tmp_mirror, with be pointed at it as a file:// URL. be reads that with curl only (wget +# fetches http, https and ftp), so without curl the test is skipped. +# Usage: setup_tmp_mirror version... +# Globals: +# TMP_MIRROR_DIR +# BE_MIRROR + +function setup_tmp_mirror() { + command -v curl > /dev/null || skip "the local mirror is a file:// URL, which needs curl" + make_tmp_mirror "$@" export BE_MIRROR="file://${TMP_MIRROR_DIR}" } + +# make_tmp_mirror, served on 127.0.0.1 by flaky-mirror.js and with be pointed at it: a request for +# a file whose .plan lists answers gets those first, one per request, such as a 504 (see +# flaky-mirror.js). Each request is logged to ${TMP_MIRROR_DIR}/requests.log. Needs node. Stop it +# with stop_flaky_mirror. +# Usage: setup_flaky_mirror version... +# Globals: +# TMP_MIRROR_DIR +# FLAKY_MIRROR_PID +# BE_MIRROR + +function setup_flaky_mirror() { + command -v node > /dev/null || skip "the flaky mirror runs on node" + make_tmp_mirror "$@" + local port_file="${TMP_MIRROR_DIR}/port" + # With fd 3 open, bats would wait for the server to exit. + node "${BATS_TEST_DIRNAME}/flaky-mirror.js" "${TMP_MIRROR_DIR}" "${port_file}" > /dev/null 2>&1 3>&- & + FLAKY_MIRROR_PID=$! + local waited=0 + while [[ ! -s "${port_file}" ]]; do + if (( waited++ >= 100 )); then + stop_flaky_mirror + echo "flaky-mirror.js did not start within 10 seconds" >&2 + return 1 + fi + sleep 0.1 + done + export BE_MIRROR="http://127.0.0.1:$(cat "${port_file}")" + # Not through a proxy that may be set for the tests. + export no_proxy="127.0.0.1${no_proxy:+,${no_proxy}}" + export NO_PROXY="127.0.0.1${NO_PROXY:+,${NO_PROXY}}" +} + + +function stop_flaky_mirror() { + if [[ -n "${FLAKY_MIRROR_PID}" ]]; then + kill "${FLAKY_MIRROR_PID}" 2> /dev/null || true + fi + FLAKY_MIRROR_PID= +} + # # @todo Duplicate # Synopsis: tarball_platform