diff --git a/plugin.sh b/plugin.sh index 7674bfb..dcd0b47 100755 --- a/plugin.sh +++ b/plugin.sh @@ -248,6 +248,8 @@ if [ "$COMMAND" = "remove" ]; then overlay_http="$(json_read "$manifest" install.overlays.http)" overlay_https="$(json_read "$manifest" install.overlays.https)" + gateway_http="$(json_read "$manifest" install.gateway.overlays.http)" + gateway_https="$(json_read "$manifest" install.gateway.overlays.https)" setup_kustomize step "Removing $SOURCE" @@ -256,10 +258,13 @@ if [ "$COMMAND" = "remove" ]; then [ "$ASSUME_YES" = true ] && confirm_default=y if [ "$(ask "Continue? (y/n)" "$confirm_default")" != "y" ]; then die "cancelled."; fi - for overlay in "$overlay_https" "$overlay_http"; do - [ -n "$overlay" ] && [ -d "$dir/$overlay" ] || continue - ./kustomize build "$dir/$overlay" | kubectl delete --ignore-not-found -f - || true - break + # Either install may be the one on the cluster, so both go. + for pair in "$gateway_https $gateway_http" "$overlay_https $overlay_http"; do + for overlay in $pair; do + [ -d "$dir/$overlay" ] || continue + ./kustomize build "$dir/$overlay" | kubectl delete --ignore-not-found -f - || true + break + done done ok "removed from the cluster" echo " Files are still at $dir — delete them by hand if you're done with it." @@ -267,6 +272,49 @@ if [ "$COMMAND" = "remove" ]; then exit 0 fi +# kind/name of each object an overlay renders. +rendered_objects() { + ./kustomize build "$1" | awk '/^kind: /{ kind = $2; next } kind && /^ name: /{ print kind "/" $2; kind = "" }' +} + +# Accepted by every parent, with its backends resolved, at the route's +# current generation. +route_ready() { + local state gen parents parent + state="$(kubectl -n 5stack get httproute "$1" -o jsonpath='{.metadata.generation}{" "}{range .status.parents[*]}{.conditions[?(@.type=="Accepted")].status}/{.conditions[?(@.type=="ResolvedRefs")].status}/{.conditions[?(@.type=="Accepted")].observedGeneration}{" "}{end}' 2>/dev/null)" || return 1 + gen="${state%% *}" + parents="${state#"$gen"}" + [ -n "$gen" ] && [ -n "${parents// /}" ] || return 1 + for parent in $parents; do + [ "$parent" = "True/True/$gen" ] || return 1 + done +} + +# The Ingresses an earlier, Ingress install of this plugin made, deleted by +# the names that install renders -- never by guess -- once every route the +# Gateway API install renders is serving. Until then they stay and serve. +retire_ingresses() { + local routes ingresses route ingress deadline + routes="$(rendered_objects "$1" | sed -n 's#^HTTPRoute/##p')" + ingresses="$(rendered_objects "$2" | sed -n 's#^Ingress/##p')" + [ -n "$ingresses" ] || return 0 + deadline=$((SECONDS + ${ROUTE_WAIT_TIMEOUT:-120})) + for route in $routes; do + until route_ready "$route"; do + if [ "$SECONDS" -ge "$deadline" ]; then + warn "HTTPRoute $route isn't serving yet, so $NAME's Ingresses stay — kubectl -n 5stack describe httproute $route" + return 0 + fi + sleep 3 + done + done + for ingress in $ingresses; do + if kubectl -n 5stack delete ingress "$ingress" --ignore-not-found >/dev/null; then + ok "retired Ingress $ingress" + fi + done +} + # --------------------------------------------------------------------------- # Fetch # --------------------------------------------------------------------------- @@ -347,7 +395,9 @@ NAME="$(json_read "$MANIFEST" name)" OVERLAY_HTTP="$(json_read "$MANIFEST" install.overlays.http)" OVERLAY_HTTPS="$(json_read "$MANIFEST" install.overlays.https)" -[ -n "$OVERLAY_HTTP$OVERLAY_HTTPS" ] \ +GATEWAY_HTTP="$(json_read "$MANIFEST" install.gateway.overlays.http)" +GATEWAY_HTTPS="$(json_read "$MANIFEST" install.gateway.overlays.https)" +[ -n "$OVERLAY_HTTP$OVERLAY_HTTPS$GATEWAY_HTTP$GATEWAY_HTTPS" ] \ || die "$NAME ships no \"install\" block, so it can't be deployed to this cluster. It is a static build — host it anywhere and register it in the panel. See https://docs.5stack.gg/plugins/deploying" @@ -407,6 +457,24 @@ done < <(find "$TARGET" -name '*.env.example' 2>/dev/null) step "Choosing the install" +# A panel on Traefik has the shared Gateway. There, a plugin that ships +# Gateway API routes installs those, and the Ingress install it replaces is +# retired once they are serving; everywhere else, the Ingress install. +USE_GATEWAY=false +LEGACY_HTTP="" +LEGACY_HTTPS="" +if [ -n "$GATEWAY_HTTP$GATEWAY_HTTPS" ] && kubectl -n traefik get gateway traefik-gateway >/dev/null 2>&1; then + USE_GATEWAY=true + LEGACY_HTTP="$OVERLAY_HTTP" + LEGACY_HTTPS="$OVERLAY_HTTPS" + OVERLAY_HTTP="$GATEWAY_HTTP" + OVERLAY_HTTPS="$GATEWAY_HTTPS" + ok "Gateway API routes — this panel routes through Traefik" +elif [ -z "$OVERLAY_HTTP$OVERLAY_HTTPS" ]; then + die "$NAME only ships Gateway API routes, and this panel has no shared Gateway yet. + Run ./update.sh first." +fi + if [ -z "$TLS_MODE" ]; then if [ "$REVERSE_PROXY" = "true" ]; then TLS_MODE=http; else TLS_MODE=https; fi echo " ${C_DIM}from this site's REVERSE_PROXY=$REVERSE_PROXY${C_RESET}" @@ -432,6 +500,15 @@ fi [ -d "$TARGET/$OVERLAY" ] || die "the manifest points at $OVERLAY, which isn't in the repo." +LEGACY_OVERLAY="" +if [ "$USE_GATEWAY" = true ]; then + if [ "$TLS_MODE" = "https" ] && [ -n "$LEGACY_HTTPS" ]; then + LEGACY_OVERLAY="$LEGACY_HTTPS" + else + LEGACY_OVERLAY="$LEGACY_HTTP" + fi +fi + setup_kustomize # --------------------------------------------------------------------------- @@ -594,6 +671,10 @@ for deployment in $DEPLOYMENTS; do || warn "$deployment isn't ready — kubectl -n 5stack describe deployment/$deployment" done +if [ -n "$LEGACY_OVERLAY" ] && [ -d "$TARGET/$LEGACY_OVERLAY" ]; then + retire_ingresses "$TARGET/$OVERLAY" "$TARGET/$LEGACY_OVERLAY" +fi + # --------------------------------------------------------------------------- banner "5Stack : $NAME Installed" diff --git a/tests/scripts/plugin_test.sh b/tests/scripts/plugin_test.sh index 6ab416c..bb4365f 100644 --- a/tests/scripts/plugin_test.sh +++ b/tests/scripts/plugin_test.sh @@ -39,6 +39,35 @@ EOF printf 'HELLO_DOMAIN=hello.example.org\n' >"$PLUGIN/k8s/base/hello.env.example" } +# A plugin that also ships Gateway API routes, beside its Ingress install. +make_gateway_plugin() { + make_plugin + mkdir -p "$PLUGIN/k8s/gateway/http" + sed -i.bak 's#"overlays": { "http": "k8s/overlays/http" },#"overlays": { "http": "k8s/overlays/http" },\n "gateway": { "overlays": { "http": "k8s/gateway/http" } },#' "$PLUGIN/public/5stack-plugin.json" + rm -f "$PLUGIN/public/5stack-plugin.json.bak" + # Renders the hello route for the Gateway API install and the hello + # Ingress for the other, the way kustomize would. + cat >"$PANEL/kustomize" <<'STUB' +#!/bin/sh +case "$2" in + */k8s/gateway/*) printf 'apiVersion: gateway.networking.k8s.io/v1\nkind: HTTPRoute\nmetadata:\n name: hello\n namespace: 5stack\n' ;; + */k8s/overlays/*) printf 'apiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: hello\n namespace: 5stack\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: hello\n namespace: 5stack\n' ;; +esac +STUB + chmod +x "$PANEL/kustomize" +} + +# Without --dry-run: applies, then retires what it replaces. +run_plugin_for_real() { + (cd "$WORK_DIR" && PATH="$STUB_BIN:$PATH" ROUTE_WAIT_TIMEOUT=0 bash "$PANEL/plugin.sh" "$PLUGIN" -y "$@") \ + >"$WORK_DIR/plugin-output" 2>&1 \ + || fail "plugin.sh exited $?: $(tail -n 20 "$WORK_DIR/plugin-output")" +} + +ingress_deletes() { + grep ' delete ingress ' "$WORK_DIR/calls" 2>/dev/null || true +} + run_plugin() { (cd "$WORK_DIR" && PATH="$STUB_BIN:$PATH" bash "$PANEL/plugin.sh" "$PLUGIN" --dry-run -y "$@") \ >"$WORK_DIR/plugin-output" 2>&1 \ @@ -171,4 +200,59 @@ test_a_host_under_the_panel_is_not_warned() { assert_not_contains "$(cat "$WORK_DIR/plugin-output")" "outside the session cookie's domain" } + +test_a_panel_on_traefik_installs_the_plugins_gateway_routes() { + make_panel panel.example.com + make_gateway_plugin + touch "$WORK_DIR/shared-gateway" + run_plugin --domain hello.example.com + assert_contains "$(cat "$WORK_DIR/plugin-output")" "custom/hello/k8s/gateway/http" + assert_not_contains "$(cat "$WORK_DIR/plugin-output")" "custom/hello/k8s/overlays/http" +} + +test_a_panel_still_on_ingress_nginx_installs_the_plugins_ingress() { + make_panel panel.example.com + make_gateway_plugin + run_plugin --domain hello.example.com + assert_contains "$(cat "$WORK_DIR/plugin-output")" "custom/hello/k8s/overlays/http" + assert_not_contains "$(cat "$WORK_DIR/plugin-output")" "custom/hello/k8s/gateway/http" +} + +test_a_plugin_without_gateway_routes_keeps_its_ingress_on_traefik() { + make_panel panel.example.com + make_plugin + touch "$WORK_DIR/shared-gateway" + run_plugin --domain hello.example.com + assert_contains "$(cat "$WORK_DIR/plugin-output")" "custom/hello/k8s/overlays/http" +} + +test_routes_that_serve_retire_the_ingress_install() { + make_panel panel.example.com + make_gateway_plugin + touch "$WORK_DIR/shared-gateway" + echo "3 True/True/3 True/True/3 " >"$WORK_DIR/route-hello" + run_plugin_for_real --domain hello.example.com + assert_eq "$(ingress_deletes)" "-n 5stack delete ingress hello --ignore-not-found" + assert_contains "$(cat "$WORK_DIR/plugin-output")" "retired Ingress hello" +} + +test_routes_whose_backends_do_not_resolve_keep_the_ingresses() { + make_panel panel.example.com + make_gateway_plugin + touch "$WORK_DIR/shared-gateway" + echo "3 True/False/3 True/False/3 " >"$WORK_DIR/route-hello" + run_plugin_for_real --domain hello.example.com + assert_eq "$(ingress_deletes)" "" + assert_contains "$(cat "$WORK_DIR/plugin-output")" "Ingresses stay" +} + +test_a_route_status_from_an_older_generation_keeps_the_ingresses() { + make_panel panel.example.com + make_gateway_plugin + touch "$WORK_DIR/shared-gateway" + echo "4 True/True/3 True/True/3 " >"$WORK_DIR/route-hello" + run_plugin_for_real --domain hello.example.com + assert_eq "$(ingress_deletes)" "" +} + run_tests diff --git a/tests/scripts/stub/kubectl b/tests/scripts/stub/kubectl index 3f59315..c88086a 100755 --- a/tests/scripts/stub/kubectl +++ b/tests/scripts/stub/kubectl @@ -23,6 +23,12 @@ case "$args" in *" apply -f - "*) cat >"$WORK_DIR/applied" ;; *" get service traefik "*) if [ -f "$WORK_DIR/traefik-address" ]; then cat "$WORK_DIR/traefik-address"; fi ;; *" get helmchart "*) echo 1 ;; + *" get gateway traefik-gateway "*) [ -f "$WORK_DIR/shared-gateway" ] || exit 1 ;; + *" get httproute "*) + route="$(printf '%s' "$args" | sed -n 's/.* get httproute \([^ ]*\).*/\1/p')" + [ -f "$WORK_DIR/route-$route" ] || exit 1 + cat "$WORK_DIR/route-$route" + ;; *" get customresourcedefinitions,"*) if [ -f "$WORK_DIR/cluster-objects" ]; then cat "$WORK_DIR/cluster-objects"; fi ;; *" get helmcharts.helm.cattle.io "*) name="$(printf '%s' "$args" | sed -n 's/.*metadata\.name=\([^ ]*\).*/\1/p')"