From dde9e32a4b171c3d7643155ce472f3ae34f2da96 Mon Sep 17 00:00:00 2001 From: Rithika Narayan Date: Wed, 18 Nov 2026 15:43:03 -0500 Subject: [PATCH 1/3] centralize version list --- .github/workflows/build.yml | 18 +- .github/workflows/build_layer.yml | 20 +- ci/config.yaml | 4 +- ci/datasources/runtimes.yaml | 65 ------ ci/input_files/build.yaml.tpl | 62 +++--- ci/publish_layers.sh | 82 +------- python_versions.yaml | 32 +++ scripts/add_new_region.sh | 40 +--- scripts/build_layers.sh | 20 +- scripts/list_layers.sh | 21 +- scripts/publish_govcloud.sh | 5 +- scripts/publish_layers.sh | 59 +----- scripts/python_versions.sh | 333 ++++++++++++++++++++++++++++++ scripts/run_integration_tests.sh | 27 +-- scripts/run_tests.sh | 5 +- scripts/sign_layers.sh | 22 +- 16 files changed, 497 insertions(+), 318 deletions(-) delete mode 100644 ci/datasources/runtimes.yaml create mode 100644 python_versions.yaml create mode 100755 scripts/python_versions.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 828c3082e..79dfae69a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,12 +38,28 @@ jobs: source venv/bin/activate flake8 datadog_lambda/ + python-versions: + runs-on: ubuntu-24.04 + outputs: + versions: ${{ steps.load.outputs.versions }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Load Python versions + id: load + run: | + source ./scripts/python_versions.sh + load_python_versions + echo "versions=$(python_versions_json)" >> "$GITHUB_OUTPUT" + unit-test: + needs: python-versions runs-on: ubuntu-24.04 strategy: max-parallel: 4 matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15'] + python-version: ${{ fromJSON(needs.python-versions.outputs.versions) }} steps: - name: Checkout diff --git a/.github/workflows/build_layer.yml b/.github/workflows/build_layer.yml index 4bbe073eb..44b64281a 100644 --- a/.github/workflows/build_layer.yml +++ b/.github/workflows/build_layer.yml @@ -8,14 +8,32 @@ on: - "main" jobs: + python-versions: + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.load.outputs.versions }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load Python versions + id: load + run: | + # system_tests: false versions are skipped. dd-trace-py main no longer + # publishes wheels for them (currently 3.8). + source ./scripts/python_versions.sh + load_python_versions + echo "versions=$(python_versions_json system_tests)" >> "$GITHUB_OUTPUT" + build: + needs: python-versions runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} strategy: fail-fast: false matrix: arch: [arm64, amd64] - python_version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] + python_version: ${{ fromJSON(needs.python-versions.outputs.versions) }} steps: - name: Checkout diff --git a/ci/config.yaml b/ci/config.yaml index d37a0f316..749a96c65 100644 --- a/ci/config.yaml +++ b/ci/config.yaml @@ -5,8 +5,8 @@ outputFiles: - ci/build-pipeline.yaml datasources: - runtimes: - url: ci/datasources/runtimes.yaml + python_versions: + url: python_versions.yaml regions: url: ci/datasources/regions.yaml environments: diff --git a/ci/datasources/runtimes.yaml b/ci/datasources/runtimes.yaml deleted file mode 100644 index b750be675..000000000 --- a/ci/datasources/runtimes.yaml +++ /dev/null @@ -1,65 +0,0 @@ -runtimes: - - name: "python38" - python_version: "3.8" - arch: "amd64" - image: "3.8" - - name: "python38" - python_version: "3.8" - arch: "arm64" - image: "3.8" - - name: "python39" - python_version: "3.9" - arch: "amd64" - image: "3.9" - - name: "python39" - python_version: "3.9" - arch: "arm64" - image: "3.9" - - name: "python310" - python_version: "3.10" - arch: "amd64" - image: "3.10" - - name: "python310" - python_version: "3.10" - arch: "arm64" - image: "3.10" - - name: "python311" - python_version: "3.11" - arch: "amd64" - image: "3.11.6" - - name: "python311" - python_version: "3.11" - arch: "arm64" - image: "3.11.6" - - name: "python312" - python_version: "3.12" - arch: "amd64" - image: "3.12.0" - - name: "python312" - python_version: "3.12" - arch: "arm64" - image: "3.12.0" - - name: "python313" - python_version: "3.13" - arch: "amd64" - image: "3.13.0" - - name: "python313" - python_version: "3.13" - arch: "arm64" - image: "3.13.0" - - name: "python314" - python_version: "3.14" - arch: "amd64" - image: "3.14.0" - - name: "python314" - python_version: "3.14" - arch: "arm64" - image: "3.14.0" - - name: "python315" - python_version: "3.15" - arch: "amd64" - image: "3.15.0" - - name: "python315" - python_version: "3.15" - arch: "arm64" - image: "3.15.0" diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl index cf89c5eac..7c03cce97 100644 --- a/ci/input_files/build.yaml.tpl +++ b/ci/input_files/build.yaml.tpl @@ -1,4 +1,6 @@ {{- $e2e_region := "us-west-2" -}} +{{- $archs := coll.Slice "amd64" "arm64" -}} +{{- $runtime := dict -}} stages: - build @@ -26,7 +28,9 @@ default: - apt-get install nodejs -y - npm install --global yarn -{{ range $runtime := (ds "runtimes").runtimes }} +{{ range $py := (ds "python_versions").versions }} +{{ range $arch := $archs }} +{{ $runtime = dict "name" (printf "python%s" ($py.version | strings.ReplaceAll "." "")) "arch" $arch "python_version" $py.version "image" $py.image }} # TODO(astuyve) - figure out python build cache .{{ $runtime.name }}-{{ $runtime.arch }}-cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache @@ -196,6 +200,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}): {{- end }} +{{- end }} {{- end }} publish-pypi-package: @@ -208,9 +213,12 @@ publish-pypi-package: rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' when: manual - needs: {{ range $runtime := (ds "runtimes").runtimes }} - - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}}) - {{- end }} + needs: + {{- range $py := (ds "python_versions").versions }} + {{- range $arch := $archs }} + - sign-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) + {{- end }} + {{- end }} script: - ./ci/publish_pypi.sh @@ -219,13 +227,17 @@ layer bundle: tags: ["arch:amd64"] image: registry.ddbuild.io/images/docker:20.10 needs: - {{ range (ds "runtimes").runtimes }} - - build-layer ({{ .name }}-{{ .arch }}) - {{ end }} + {{- range $py := (ds "python_versions").versions }} + {{- range $arch := $archs }} + - build-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) + {{- end }} + {{- end }} dependencies: - {{ range (ds "runtimes").runtimes }} - - build-layer ({{ .name }}-{{ .arch }}) - {{ end }} + {{- range $py := (ds "python_versions").versions }} + {{- range $arch := $archs }} + - build-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) + {{- end }} + {{- end }} artifacts: expire_in: 1 day paths: @@ -243,13 +255,17 @@ signed layer bundle: rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' needs: - {{ range (ds "runtimes").runtimes }} - - sign-layer ({{ .name }}-{{ .arch }}) - {{ end }} + {{- range $py := (ds "python_versions").versions }} + {{- range $arch := $archs }} + - sign-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) + {{- end }} + {{- end }} dependencies: - {{ range (ds "runtimes").runtimes }} - - sign-layer ({{ .name }}-{{ .arch }}) - {{ end }} + {{- range $py := (ds "python_versions").versions }} + {{- range $arch := $archs }} + - sign-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) + {{- end }} + {{- end }} artifacts: expire_in: 1 day paths: @@ -272,16 +288,12 @@ e2e-test: variables: LANGUAGES_SUBSET: python # These env vars are inherited from the dotenv reports of the publish-layer jobs - {{- range (ds "runtimes").runtimes }} - {{- if eq .arch "amd64" }} - {{- $version := print (.name | strings.Trim "python") }} - PYTHON_{{ $version }}_VERSION: $PYTHON_{{ $version }}_VERSION - {{- end }} - {{- end }} - needs: {{ range (ds "runtimes").runtimes }} - {{- if eq .arch "amd64" }} - - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]" + {{- range $py := (ds "python_versions").versions }} + PYTHON_{{ $py.version | strings.ReplaceAll "." "" }}_VERSION: $PYTHON_{{ $py.version | strings.ReplaceAll "." "" }}_VERSION {{- end }} + needs: + {{- range $py := (ds "python_versions").versions }} + - "publish-layer-sandbox (python{{ $py.version | strings.ReplaceAll "." "" }}-amd64): [{{ $e2e_region }}]" {{- end }} e2e-test-status: diff --git a/ci/publish_layers.sh b/ci/publish_layers.sh index 573b8028b..51bada705 100755 --- a/ci/publish_layers.sh +++ b/ci/publish_layers.sh @@ -10,78 +10,16 @@ set -e # Available runtimes: https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html -AWS_CLI_PYTHON_VERSIONS=( - "python3.8" - "python3.8" - "python3.9" - "python3.9" - "python3.10" - "python3.10" - "python3.11" - "python3.11" - "python3.12" - "python3.12" - "python3.13" - "python3.13" - "python3.14" - "python3.14" - "python3.15" - "python3.15" -) -PYTHON_VERSIONS=( - "3.8-amd64" - "3.8-arm64" - "3.9-amd64" - "3.9-arm64" - "3.10-amd64" - "3.10-arm64" - "3.11-amd64" - "3.11-arm64" - "3.12-amd64" - "3.12-arm64" - "3.13-amd64" - "3.13-arm64" - "3.14-amd64" - "3.14-arm64" - "3.15-amd64" - "3.15-arm64" -) -LAYER_PATHS=( - ".layers/datadog_lambda_py-amd64-3.8.zip" - ".layers/datadog_lambda_py-arm64-3.8.zip" - ".layers/datadog_lambda_py-amd64-3.9.zip" - ".layers/datadog_lambda_py-arm64-3.9.zip" - ".layers/datadog_lambda_py-amd64-3.10.zip" - ".layers/datadog_lambda_py-arm64-3.10.zip" - ".layers/datadog_lambda_py-amd64-3.11.zip" - ".layers/datadog_lambda_py-arm64-3.11.zip" - ".layers/datadog_lambda_py-amd64-3.12.zip" - ".layers/datadog_lambda_py-arm64-3.12.zip" - ".layers/datadog_lambda_py-amd64-3.13.zip" - ".layers/datadog_lambda_py-arm64-3.13.zip" - ".layers/datadog_lambda_py-amd64-3.14.zip" - ".layers/datadog_lambda_py-arm64-3.14.zip" - ".layers/datadog_lambda_py-amd64-3.15.zip" - ".layers/datadog_lambda_py-arm64-3.15.zip" -) -LAYERS=( - "Datadog-Python38" - "Datadog-Python38-ARM" - "Datadog-Python39" - "Datadog-Python39-ARM" - "Datadog-Python310" - "Datadog-Python310-ARM" - "Datadog-Python311" - "Datadog-Python311-ARM" - "Datadog-Python312" - "Datadog-Python312-ARM" - "Datadog-Python313" - "Datadog-Python313-ARM" - "Datadog-Python314" - "Datadog-Python314-ARM" - "Datadog-Python315" - "Datadog-Python315-ARM" -) +source "$(dirname "${BASH_SOURCE[0]}")/../scripts/python_versions.sh" +load_python_versions +load_python_layers +AWS_CLI_PYTHON_VERSIONS=("${LAYER_RUNTIMES[@]}") +LAYERS=("${LAYER_NAMES[@]}") +PYTHON_VERSIONS=() +for _i in "${!LAYER_VERSIONS[@]}"; do + PYTHON_VERSIONS+=("${LAYER_VERSIONS[$_i]}-${LAYER_ARCHS[$_i]}") +done +unset _i STAGES=('prod', 'sandbox', 'staging', 'gov-staging', 'gov-prod') printf "Starting script...\n\n" diff --git a/python_versions.yaml b/python_versions.yaml new file mode 100644 index 000000000..32615c930 --- /dev/null +++ b/python_versions.yaml @@ -0,0 +1,32 @@ +# Supported Python versions for the Lambda library. +# +# Layer builds, publish scripts, tests, and CI read this file. Add a version +# here and those pick it up. The README and the pyproject.toml classifiers are +# the lists still maintained by hand. +# +# version: CPython minor version. +# image: GitLab CI image tag (registry.ddbuild.io/images/mirror/python). +# preview: AWS has not published a SAM build image yet. Layer builds use +# public.ecr.aws/lambda/python:-preview. Default false. +# system_tests: Set false to skip the GitHub system-tests layer build. That +# job installs a dd-trace-py main-branch wheel, which no longer +# publishes 3.8 wheels. Default true. +versions: + - version: "3.8" + image: "3.8" + system_tests: false + - version: "3.9" + image: "3.9" + - version: "3.10" + image: "3.10" + - version: "3.11" + image: "3.11.6" + - version: "3.12" + image: "3.12.0" + - version: "3.13" + image: "3.13.0" + - version: "3.14" + image: "3.14.0" + - version: "3.15" + image: "3.15.0" + preview: true diff --git a/scripts/add_new_region.sh b/scripts/add_new_region.sh index 62367e5f0..704ceb484 100755 --- a/scripts/add_new_region.sh +++ b/scripts/add_new_region.sh @@ -12,42 +12,10 @@ set -e OLD_REGION='us-east-1' -LAYER_NAMES=( - "Datadog-Python38" - "Datadog-Python38-ARM" - "Datadog-Python39" - "Datadog-Python39-ARM" - "Datadog-Python310" - "Datadog-Python310-ARM" - "Datadog-Python311" - "Datadog-Python311-ARM" - "Datadog-Python312" - "Datadog-Python312-ARM" - "Datadog-Python313" - "Datadog-Python313-ARM" - "Datadog-Python314" - "Datadog-Python314-ARM" - "Datadog-Python315" - "Datadog-Python315-ARM" -) -PYTHON_VERSIONS_FOR_AWS_CLI=( - "python3.8" - "python3.8" - "python3.9" - "python3.9" - "python3.10" - "python3.10" - "python3.11" - "python3.11" - "python3.12" - "python3.12" - "python3.13" - "python3.13" - "python3.14" - "python3.14" - "python3.15" - "python3.15" -) +source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" +load_python_versions +load_python_layers +PYTHON_VERSIONS_FOR_AWS_CLI=("${LAYER_RUNTIMES[@]}") NEW_REGION=$1 publish_layer() { diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index a6be62ff6..b250ad9fa 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -35,12 +35,9 @@ set -e LAYER_DIR=".layers" LAYER_FILES_PREFIX="datadog_lambda_py" -AVAILABLE_PYTHON_VERSIONS=("3.8" "3.9" "3.10" "3.11" "3.12" "3.13" "3.14") -# Preview runtimes. AWS has not published a SAM build image for these, so the -# Lambda preview base image is used instead. The Dockerfile installs the -# compiler toolchain that image is missing. -PREVIEW_PYTHON_VERSIONS=("3.15") -ALL_PYTHON_VERSIONS=("${AVAILABLE_PYTHON_VERSIONS[@]}" "${PREVIEW_PYTHON_VERSIONS[@]}") +# Versions and the preview flag live in python_versions.yaml. +source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" +load_python_versions AVAILABLE_ARCHS=("arm64" "amd64") if [ -n "$ARCH" ]; then @@ -74,11 +71,11 @@ fi # Determine which Python versions to build layers for if [ -z "$PYTHON_VERSION" ]; then echo "Python version not specified, building layers for all versions." - PYTHON_VERSIONS=("${ALL_PYTHON_VERSIONS[@]}") + PYTHON_VERSIONS=("${SUPPORTED_PYTHON_VERSIONS[@]}") else echo "Python version specified: $PYTHON_VERSION" - if [[ ! " ${ALL_PYTHON_VERSIONS[@]} " =~ " ${PYTHON_VERSION} " ]]; then - echo "Python version $PYTHON_VERSION is not a valid option. Choose from: ${ALL_PYTHON_VERSIONS[@]}" + if ! python_version_known "$PYTHON_VERSION"; then + echo "Python version $PYTHON_VERSION is not a valid option. Choose from: ${SUPPORTED_PYTHON_VERSIONS[*]}" echo "" echo "EXITING SCRIPT." exit 1 @@ -108,8 +105,9 @@ function docker_build_zip { destination=$(make_path_absolute $2) arch=$3 - # Preview runtimes have no SAM build image yet. See PREVIEW_PYTHON_VERSIONS. - if [[ " ${PREVIEW_PYTHON_VERSIONS[*]} " =~ " ${1} " ]]; then + # Preview runtimes have no SAM build image yet. The Dockerfile installs + # the compiler toolchain that the Lambda preview image is missing. + if python_version_is_preview "$1"; then image="public.ecr.aws/lambda/python:${1}-preview" else image="public.ecr.aws/sam/build-python${1}:1" diff --git a/scripts/list_layers.sh b/scripts/list_layers.sh index 352c5b7a6..6fa1dfeb9 100755 --- a/scripts/list_layers.sh +++ b/scripts/list_layers.sh @@ -10,24 +10,9 @@ set -e -LAYER_NAMES=( - "Datadog-Python38" - "Datadog-Python38-ARM" - "Datadog-Python39" - "Datadog-Python39-ARM" - "Datadog-Python310" - "Datadog-Python310-ARM" - "Datadog-Python311" - "Datadog-Python311-ARM" - "Datadog-Python312" - "Datadog-Python312-ARM" - "Datadog-Python313" - "Datadog-Python313-ARM" - "Datadog-Python314" - "Datadog-Python314-ARM" - "Datadog-Python315" - "Datadog-Python315-ARM" -) +source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" +load_python_versions +load_python_layers AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') LAYERS_MISSING_REGIONS=() diff --git a/scripts/publish_govcloud.sh b/scripts/publish_govcloud.sh index db79a2009..9217b068a 100755 --- a/scripts/publish_govcloud.sh +++ b/scripts/publish_govcloud.sh @@ -90,13 +90,16 @@ else fi fi +source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" +load_python_versions + for region in $REGIONS do echo "Starting publishing layers for region $region..." export REGION=$region - for python_version in "3.8" "3.9" "3.10" "3.11" "3.12" "3.13" "3.14" "3.15"; do + for python_version in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do for arch in "amd64" "arm64"; do export PYTHON_VERSION=$python_version export ARCH=$arch diff --git a/scripts/publish_layers.sh b/scripts/publish_layers.sh index 12ba05a40..bf9a06f58 100755 --- a/scripts/publish_layers.sh +++ b/scripts/publish_layers.sh @@ -13,60 +13,11 @@ set -e # Makes sure any subprocesses will be terminated with this process trap "pkill -P $$; exit 1;" INT -PYTHON_VERSIONS_FOR_AWS_CLI=( - "python3.8" - "python3.8" - "python3.9" - "python3.9" - "python3.10" - "python3.10" - "python3.11" - "python3.11" - "python3.12" - "python3.12" - "python3.13" - "python3.13" - "python3.14" - "python3.14" - "python3.15" - "python3.15" -) -LAYER_PATHS=( - ".layers/datadog_lambda_py-amd64-3.8.zip" - ".layers/datadog_lambda_py-arm64-3.8.zip" - ".layers/datadog_lambda_py-amd64-3.9.zip" - ".layers/datadog_lambda_py-arm64-3.9.zip" - ".layers/datadog_lambda_py-amd64-3.10.zip" - ".layers/datadog_lambda_py-arm64-3.10.zip" - ".layers/datadog_lambda_py-amd64-3.11.zip" - ".layers/datadog_lambda_py-arm64-3.11.zip" - ".layers/datadog_lambda_py-amd64-3.12.zip" - ".layers/datadog_lambda_py-arm64-3.12.zip" - ".layers/datadog_lambda_py-amd64-3.13.zip" - ".layers/datadog_lambda_py-arm64-3.13.zip" - ".layers/datadog_lambda_py-amd64-3.14.zip" - ".layers/datadog_lambda_py-arm64-3.14.zip" - ".layers/datadog_lambda_py-amd64-3.15.zip" - ".layers/datadog_lambda_py-arm64-3.15.zip" -) -AVAILABLE_LAYERS=( - "Datadog-Python38" - "Datadog-Python38-ARM" - "Datadog-Python39" - "Datadog-Python39-ARM" - "Datadog-Python310" - "Datadog-Python310-ARM" - "Datadog-Python311" - "Datadog-Python311-ARM" - "Datadog-Python312" - "Datadog-Python312-ARM" - "Datadog-Python313" - "Datadog-Python313-ARM" - "Datadog-Python314" - "Datadog-Python314-ARM" - "Datadog-Python315" - "Datadog-Python315-ARM" -) +source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" +load_python_versions +load_python_layers +PYTHON_VERSIONS_FOR_AWS_CLI=("${LAYER_RUNTIMES[@]}") +AVAILABLE_LAYERS=("${LAYER_NAMES[@]}") AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') BATCH_SIZE=60 diff --git a/scripts/python_versions.sh b/scripts/python_versions.sh new file mode 100755 index 000000000..d971ef7a0 --- /dev/null +++ b/scripts/python_versions.sh @@ -0,0 +1,333 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2026 Datadog, Inc. + +# Reads python_versions.yaml. Source this file; do not execute it from other +# scripts. Pure bash (3.2+) so layer builds do not need jq, yq, or Python. +# +# source scripts/python_versions.sh +# load_python_versions +# +# After load_python_versions: +# SUPPORTED_PYTHON_VERSIONS "3.8" "3.9" ... +# SUPPORTED_PYTHON_VERSION_IMAGES parallel to versions +# SUPPORTED_PYTHON_VERSION_PREVIEW "true" / "false" +# SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS "true" / "false" +# +# python_version_known +# python_version_is_preview +# python_versions_json [system_tests|preview] +# +# After load_python_layers (calls load_python_versions if needed), one entry +# per version for amd64 then arm64: +# LAYER_NAMES Datadog-Python38, Datadog-Python38-ARM, ... +# LAYER_RUNTIMES python3.8, python3.8, ... +# LAYER_ZIPS datadog_lambda_py-amd64-3.8.zip, ... +# LAYER_PATHS .layers/datadog_lambda_py-amd64-3.8.zip, ... +# LAYER_VERSIONS 3.8, 3.8, ... +# LAYER_ARCHS amd64, arm64, ... + +_PYTHON_VERSIONS_FILE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/python_versions.yaml" + +_pv_trim() { + local s="$1" + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +_pv_unquote() { + local s="$1" + local first last + first="${s%"${s#?}"}" + last="${s#"${s%?}"}" + if [ ${#s} -ge 2 ] && [ "$first" = "$last" ] && { [ "$first" = '"' ] || [ "$first" = "'" ]; }; then + s="${s#?}" + s="${s%?}" + fi + printf '%s' "$s" +} + +_pv_reset_entry() { + _pv_version="" + _pv_image="" + _pv_preview="" + _pv_system_tests="" +} + +_pv_flush_entry() { + if [ -z "${_pv_version}" ]; then + return 0 + fi + local existing + if [ "${#SUPPORTED_PYTHON_VERSIONS[@]}" -gt 0 ]; then + for existing in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do + if [ "$existing" = "$_pv_version" ]; then + echo "duplicate python version in ${PYTHON_VERSIONS_FILE}: ${_pv_version}" >&2 + return 1 + fi + done + fi + if [ -z "${_pv_image}" ]; then + echo "python version ${_pv_version} is missing image in ${PYTHON_VERSIONS_FILE}" >&2 + return 1 + fi + if [ -z "${_pv_preview}" ]; then + _pv_preview="false" + fi + if [ -z "${_pv_system_tests}" ]; then + _pv_system_tests="true" + fi + SUPPORTED_PYTHON_VERSIONS+=("$_pv_version") + SUPPORTED_PYTHON_VERSION_IMAGES+=("$_pv_image") + SUPPORTED_PYTHON_VERSION_PREVIEW+=("$_pv_preview") + SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS+=("$_pv_system_tests") + _pv_reset_entry +} + +_pv_set_field() { + local key="$1" + local value="$2" + case "$key" in + image) _pv_image="$value" ;; + preview) + case "$value" in + true|false) _pv_preview="$value" ;; + *) + echo "python version ${_pv_version}: preview must be true or false, got '${value}'" >&2 + return 1 + ;; + esac + ;; + system_tests) + case "$value" in + true|false) _pv_system_tests="$value" ;; + *) + echo "python version ${_pv_version}: system_tests must be true or false, got '${value}'" >&2 + return 1 + ;; + esac + ;; + *) + echo "unknown field '${key}' in ${PYTHON_VERSIONS_FILE}" >&2 + return 1 + ;; + esac +} + +load_python_versions() { + PYTHON_VERSIONS_FILE="${PYTHON_VERSIONS_FILE:-$_PYTHON_VERSIONS_FILE}" + if [ ! -f "$PYTHON_VERSIONS_FILE" ]; then + echo "python versions config not found: $PYTHON_VERSIONS_FILE" >&2 + return 1 + fi + + SUPPORTED_PYTHON_VERSIONS=() + SUPPORTED_PYTHON_VERSION_IMAGES=() + SUPPORTED_PYTHON_VERSION_PREVIEW=() + SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS=() + _python_versions_loaded=0 + _pv_reset_entry + + local line key value seen_versions=0 + while IFS= read -r line || [ -n "$line" ]; do + line="${line%%#*}" + line="${line%$'\r'}" + line="$(_pv_trim "$line")" + if [ -z "$line" ]; then + continue + fi + if [ "$line" = "versions:" ]; then + seen_versions=1 + continue + fi + if [ "$seen_versions" -ne 1 ]; then + echo "unexpected line in ${PYTHON_VERSIONS_FILE}: ${line}" >&2 + return 1 + fi + if [[ "$line" =~ ^-[[:space:]]+version:[[:space:]]*(.*)$ ]]; then + _pv_flush_entry || return 1 + value="$(_pv_unquote "$(_pv_trim "${BASH_REMATCH[1]}")")" + if [[ ! "$value" =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "invalid python version '${value}' in ${PYTHON_VERSIONS_FILE}" >&2 + return 1 + fi + _pv_version="$value" + continue + fi + if [ -z "${_pv_version}" ]; then + echo "field before version in ${PYTHON_VERSIONS_FILE}: ${line}" >&2 + return 1 + fi + if [[ "$line" =~ ^([A-Za-z_]+):[[:space:]]*(.*)$ ]]; then + key="${BASH_REMATCH[1]}" + value="$(_pv_unquote "$(_pv_trim "${BASH_REMATCH[2]}")")" + _pv_set_field "$key" "$value" || return 1 + continue + fi + echo "could not parse line in ${PYTHON_VERSIONS_FILE}: ${line}" >&2 + return 1 + done < "$PYTHON_VERSIONS_FILE" + + _pv_flush_entry || return 1 + if [ "${#SUPPORTED_PYTHON_VERSIONS[@]}" -eq 0 ]; then + echo "no python versions found in ${PYTHON_VERSIONS_FILE}" >&2 + return 1 + fi + _python_versions_loaded=1 +} + +_python_version_index() { + local target="$1" + local i + if [ "${_python_versions_loaded:-0}" -ne 1 ]; then + echo "load_python_versions must be called first" >&2 + return 1 + fi + for i in "${!SUPPORTED_PYTHON_VERSIONS[@]}"; do + if [ "${SUPPORTED_PYTHON_VERSIONS[$i]}" = "$target" ]; then + printf '%s' "$i" + return 0 + fi + done + return 1 +} + +python_version_known() { + _python_version_index "$1" >/dev/null +} + +python_version_is_preview() { + local i + i="$(_python_version_index "$1")" || return 1 + [ "${SUPPORTED_PYTHON_VERSION_PREVIEW[$i]}" = "true" ] +} + +python_runtime_name() { + printf 'python%s' "$1" +} + +python_layer_name() { + local version="$1" + local arch="$2" + local suffix="${version//./}" + case "$arch" in + amd64) printf 'Datadog-Python%s' "$suffix" ;; + arm64) printf 'Datadog-Python%s-ARM' "$suffix" ;; + *) + echo "unknown architecture '${arch}'" >&2 + return 1 + ;; + esac +} + +python_layer_zip() { + printf 'datadog_lambda_py-%s-%s.zip' "$2" "$1" +} + +python_layer_path() { + printf '.layers/%s' "$(python_layer_zip "$1" "$2")" +} + +python_layer_matrix() { + if [ "${_python_versions_loaded:-0}" -ne 1 ]; then + echo "load_python_versions must be called first" >&2 + return 1 + fi + local version + for version in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do + printf '%s %s\n' "$version" "amd64" + printf '%s %s\n' "$version" "arm64" + done +} + +load_python_layers() { + if [ "${_python_versions_loaded:-0}" -ne 1 ]; then + load_python_versions || return 1 + fi + LAYER_NAMES=() + LAYER_PATHS=() + LAYER_ZIPS=() + LAYER_RUNTIMES=() + LAYER_VERSIONS=() + LAYER_ARCHS=() + local version arch + while IFS=' ' read -r version arch; do + if [ -z "$version" ]; then + continue + fi + LAYER_VERSIONS+=("$version") + LAYER_ARCHS+=("$arch") + LAYER_NAMES+=("$(python_layer_name "$version" "$arch")") + LAYER_ZIPS+=("$(python_layer_zip "$version" "$arch")") + LAYER_PATHS+=("$(python_layer_path "$version" "$arch")") + LAYER_RUNTIMES+=("$(python_runtime_name "$version")") + done <&2 + return 1 + fi + local json="[" + local first=1 + local i version include + for i in "${!SUPPORTED_PYTHON_VERSIONS[@]}"; do + version="${SUPPORTED_PYTHON_VERSIONS[$i]}" + include=1 + case "$only" in + "") ;; + system_tests) + if [ "${SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS[$i]}" != "true" ]; then + include=0 + fi + ;; + preview) + if [ "${SUPPORTED_PYTHON_VERSION_PREVIEW[$i]}" != "true" ]; then + include=0 + fi + ;; + *) + echo "unknown python_versions_json filter: ${only}" >&2 + return 1 + ;; + esac + if [ "$include" -eq 1 ]; then + if [ "$first" -eq 0 ]; then + json="${json}," + fi + first=0 + json="${json}\"${version}\"" + fi + done + json="${json}]" + printf '%s' "$json" +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + load_python_versions + load_python_layers + i=0 + while [ "$i" -lt "${#SUPPORTED_PYTHON_VERSIONS[@]}" ]; do + extra="" + if [ "${SUPPORTED_PYTHON_VERSION_PREVIEW[$i]}" = "true" ]; then + extra="${extra} preview" + fi + if [ "${SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS[$i]}" != "true" ]; then + extra="${extra} system_tests=false" + fi + printf '%s image=%s%s\n' \ + "${SUPPORTED_PYTHON_VERSIONS[$i]}" \ + "${SUPPORTED_PYTHON_VERSION_IMAGES[$i]}" \ + "$extra" + i=$((i + 1)) + done +fi diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index e2c2f7369..06c0e83b7 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -23,26 +23,29 @@ script_utc_start_time=$(date -u +"%Y%m%dT%H%M%S") mismatch_found=false -# Format : +# Format of each python array: # [0]: serverless runtime name # [1]: python version # [2]: random 8-character ID to avoid collisions with other runs -python38=("python3.8" "3.8" $(xxd -l 4 -c 4 -p < /dev/random)) -python39=("python3.9" "3.9" $(xxd -l 4 -c 4 -p < /dev/random)) -python310=("python3.10" "3.10" $(xxd -l 4 -c 4 -p < /dev/random)) -python311=("python3.11" "3.11" $(xxd -l 4 -c 4 -p < /dev/random)) -python312=("python3.12" "3.12" $(xxd -l 4 -c 4 -p < /dev/random)) -python313=("python3.13" "3.13" $(xxd -l 4 -c 4 -p < /dev/random)) -python314=("python3.14" "3.14" $(xxd -l 4 -c 4 -p < /dev/random)) -python315=("python3.15" "3.15" $(xxd -l 4 -c 4 -p < /dev/random)) - -PARAMETERS_SETS=("python38" "python39" "python310" "python311" "python312" "python313" "python314" "python315") +source "$scripts_dir/python_versions.sh" +load_python_versions + +PARAMETERS_SETS=() +for version in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do + key="python${version//./}" + eval "${key}=(\"python${version}\" \"${version}\" \"$(xxd -l 4 -c 4 -p < /dev/random)\")" + PARAMETERS_SETS+=("$key") +done if [ -z "$RUNTIME_PARAM" ]; then echo "Python version not specified, running for all python versions." else - RUNTIME_PARAM_NO_DOT=$(echo $RUNTIME_PARAM | sed 's/\.//') echo "Python version is specified: $RUNTIME_PARAM" + if ! python_version_known "$RUNTIME_PARAM"; then + echo "Python version $RUNTIME_PARAM is not a valid option. Choose from: ${SUPPORTED_PYTHON_VERSIONS[*]}" + exit 1 + fi + RUNTIME_PARAM_NO_DOT=$(echo $RUNTIME_PARAM | sed 's/\.//') PARAMETERS_SETS=(python${RUNTIME_PARAM_NO_DOT}) BUILD_LAYER_VERSION=python$RUNTIME_PARAM_NO_DOT[1] fi diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index d9a0b2ba0..59f697be2 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -8,9 +8,10 @@ # Run unit tests in Docker set -e -PYTHON_VERSIONS=("3.8" "3.9" "3.10" "3.11" "3.12" "3.13" "3.14" "3.15") +source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" +load_python_versions -for python_version in "${PYTHON_VERSIONS[@]}" +for python_version in "${SUPPORTED_PYTHON_VERSIONS[@]}" do echo "Running tests against python${python_version}" docker build -t datadog-lambda-python-test:$python_version \ diff --git a/scripts/sign_layers.sh b/scripts/sign_layers.sh index c014e3a67..23164cf87 100755 --- a/scripts/sign_layers.sh +++ b/scripts/sign_layers.sh @@ -8,24 +8,10 @@ set -e LAYER_DIR=".layers" -LAYER_FILES=( - "datadog_lambda_py-amd64-3.8.zip" - "datadog_lambda_py-arm64-3.8.zip" - "datadog_lambda_py-amd64-3.9.zip" - "datadog_lambda_py-arm64-3.9.zip" - "datadog_lambda_py-amd64-3.10.zip" - "datadog_lambda_py-arm64-3.10.zip" - "datadog_lambda_py-amd64-3.11.zip" - "datadog_lambda_py-arm64-3.11.zip" - "datadog_lambda_py-amd64-3.12.zip" - "datadog_lambda_py-arm64-3.12.zip" - "datadog_lambda_py-amd64-3.13.zip" - "datadog_lambda_py-arm64-3.13.zip" - "datadog_lambda_py-amd64-3.14.zip" - "datadog_lambda_py-arm64-3.14.zip" - "datadog_lambda_py-amd64-3.15.zip" - "datadog_lambda_py-arm64-3.15.zip" -) +source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" +load_python_versions +load_python_layers +LAYER_FILES=("${LAYER_ZIPS[@]}") SIGNING_PROFILE_NAME="DatadogLambdaSigningProfile" # Check account parameter From 418f7beff473fa3656c5ccbe9be945f61a6b8ed0 Mon Sep 17 00:00:00 2001 From: Rithika Narayan Date: Wed, 18 Nov 2026 16:01:48 -0500 Subject: [PATCH 2/3] reducing diff --- .github/workflows/build.yml | 6 +- .github/workflows/build_layer.yml | 4 +- ci/build-pipeline.yaml | 4333 +++++++++++++++++++++++++++++ ci/input_files/build.yaml.tpl | 66 +- ci/publish_layers.sh | 8 +- scripts/add_new_region.sh | 2 - scripts/build_layers.sh | 5 +- scripts/list_layers.sh | 2 - scripts/publish_govcloud.sh | 1 - scripts/publish_layers.sh | 2 - scripts/python_versions.sh | 353 +-- scripts/run_integration_tests.sh | 7 +- scripts/run_tests.sh | 1 - scripts/sign_layers.sh | 2 - 14 files changed, 4404 insertions(+), 388 deletions(-) create mode 100644 ci/build-pipeline.yaml mode change 100755 => 100644 scripts/python_versions.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 79dfae69a..1e96634a5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,14 +44,12 @@ jobs: versions: ${{ steps.load.outputs.versions }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Load Python versions id: load run: | - source ./scripts/python_versions.sh - load_python_versions - echo "versions=$(python_versions_json)" >> "$GITHUB_OUTPUT" + echo "versions=$(yq -o=json -I=0 '[.versions[] | .version]' python_versions.yaml)" >> "$GITHUB_OUTPUT" unit-test: needs: python-versions diff --git a/.github/workflows/build_layer.yml b/.github/workflows/build_layer.yml index 44b64281a..fee2593be 100644 --- a/.github/workflows/build_layer.yml +++ b/.github/workflows/build_layer.yml @@ -21,9 +21,7 @@ jobs: run: | # system_tests: false versions are skipped. dd-trace-py main no longer # publishes wheels for them (currently 3.8). - source ./scripts/python_versions.sh - load_python_versions - echo "versions=$(python_versions_json system_tests)" >> "$GITHUB_OUTPUT" + echo "versions=$(yq -o=json -I=0 '[.versions[] | select(.system_tests != false) | .version]' python_versions.yaml)" >> "$GITHUB_OUTPUT" build: needs: python-versions diff --git a/ci/build-pipeline.yaml b/ci/build-pipeline.yaml new file mode 100644 index 000000000..3ee140e46 --- /dev/null +++ b/ci/build-pipeline.yaml @@ -0,0 +1,4333 @@ +stages: + - build + - test + - sign + - publish + - e2e + +default: + retry: + max: 1 + when: + # Retry when the runner fails to start + - runner_system_failure + +# This is for serverless framework +.install-node: &install-node + - apt-get update + - apt-get install -y ca-certificates curl gnupg xxd + - mkdir -p /etc/apt/keyrings + - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + # We are explicitly setting the node_20.x version for the installation + - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + - apt-get update + - apt-get install nodejs -y + - npm install --global yarn + + + + + +# TODO(astuyve) - figure out python build cache +.python38-amd64-cache: &python38-amd64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python38-amd64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.8.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.8 ARCH=amd64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python38-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python38-amd64) + dependencies: + - build-layer (python38-amd64) + script: + - PYTHON_VERSION=3.8 ARCH=amd64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.8 + cache: &python38-amd64-cache + before_script: + - PYTHON_VERSION=3.8 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python38-amd64): + stage: test + tags: + + - "arch:amd64" + + image: registry.ddbuild.io/images/mirror/python:3.8 + cache: &python38-amd64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.8,arch:amd64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.8 ARCH=amd64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python38-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python38-amd64) + dependencies: + - build-layer (python38-amd64) + cache: &python38-amd64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.8 ARCH=amd64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python38-amd64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python38-amd64) + - check-layer-size (python38-amd64) + - lint python + - unit-test (python38-amd64) + - integration-test (python38-amd64) + dependencies: + - build-layer (python38-amd64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.8.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-amd64-3.8.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python38-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python38_amd64_prod.env + needs: + + - sign-layer (python38-amd64) + + dependencies: + + - sign-layer (python38-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.8 ARCH=amd64 DOTENV=python38_amd64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python38-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python38_amd64_sandbox.env + needs: + + - build-layer (python38-amd64) + - check-layer-size (python38-amd64) + - lint python + - unit-test (python38-amd64) + - integration-test (python38-amd64) + + dependencies: + + - build-layer (python38-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.8 ARCH=amd64 DOTENV=python38_amd64_sandbox.env ./ci/publish_layers.sh + + +# TODO(astuyve) - figure out python build cache +.python38-arm64-cache: &python38-arm64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python38-arm64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.8.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.8 ARCH=arm64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python38-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python38-arm64) + dependencies: + - build-layer (python38-arm64) + script: + - PYTHON_VERSION=3.8 ARCH=arm64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.8 + cache: &python38-arm64-cache + before_script: + - PYTHON_VERSION=3.8 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python38-arm64): + stage: test + tags: + + - "arch:arm64" + + image: registry.ddbuild.io/images/mirror/python:3.8 + cache: &python38-arm64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.8,arch:arm64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.8 ARCH=arm64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python38-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python38-arm64) + dependencies: + - build-layer (python38-arm64) + cache: &python38-arm64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.8 ARCH=arm64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python38-arm64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python38-arm64) + - check-layer-size (python38-arm64) + - lint python + - unit-test (python38-arm64) + - integration-test (python38-arm64) + dependencies: + - build-layer (python38-arm64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.8.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-arm64-3.8.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python38-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python38_arm64_prod.env + needs: + + - sign-layer (python38-arm64) + + dependencies: + + - sign-layer (python38-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.8 ARCH=arm64 DOTENV=python38_arm64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python38-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python38_arm64_sandbox.env + needs: + + - build-layer (python38-arm64) + - check-layer-size (python38-arm64) + - lint python + - unit-test (python38-arm64) + - integration-test (python38-arm64) + + dependencies: + + - build-layer (python38-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.8 ARCH=arm64 DOTENV=python38_arm64_sandbox.env ./ci/publish_layers.sh + + + +# TODO(astuyve) - figure out python build cache +.python39-amd64-cache: &python39-amd64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python39-amd64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.9.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.9 ARCH=amd64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python39-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python39-amd64) + dependencies: + - build-layer (python39-amd64) + script: + - PYTHON_VERSION=3.9 ARCH=amd64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.9 + cache: &python39-amd64-cache + before_script: + - PYTHON_VERSION=3.9 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python39-amd64): + stage: test + tags: + + - "arch:amd64" + + image: registry.ddbuild.io/images/mirror/python:3.9 + cache: &python39-amd64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.9,arch:amd64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.9 ARCH=amd64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python39-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python39-amd64) + dependencies: + - build-layer (python39-amd64) + cache: &python39-amd64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.9 ARCH=amd64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python39-amd64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python39-amd64) + - check-layer-size (python39-amd64) + - lint python + - unit-test (python39-amd64) + - integration-test (python39-amd64) + dependencies: + - build-layer (python39-amd64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.9.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-amd64-3.9.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python39-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python39_amd64_prod.env + needs: + + - sign-layer (python39-amd64) + + dependencies: + + - sign-layer (python39-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.9 ARCH=amd64 DOTENV=python39_amd64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python39-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python39_amd64_sandbox.env + needs: + + - build-layer (python39-amd64) + - check-layer-size (python39-amd64) + - lint python + - unit-test (python39-amd64) + - integration-test (python39-amd64) + + dependencies: + + - build-layer (python39-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.9 ARCH=amd64 DOTENV=python39_amd64_sandbox.env ./ci/publish_layers.sh + + +# TODO(astuyve) - figure out python build cache +.python39-arm64-cache: &python39-arm64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python39-arm64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.9.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.9 ARCH=arm64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python39-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python39-arm64) + dependencies: + - build-layer (python39-arm64) + script: + - PYTHON_VERSION=3.9 ARCH=arm64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.9 + cache: &python39-arm64-cache + before_script: + - PYTHON_VERSION=3.9 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python39-arm64): + stage: test + tags: + + - "arch:arm64" + + image: registry.ddbuild.io/images/mirror/python:3.9 + cache: &python39-arm64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.9,arch:arm64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.9 ARCH=arm64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python39-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python39-arm64) + dependencies: + - build-layer (python39-arm64) + cache: &python39-arm64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.9 ARCH=arm64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python39-arm64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python39-arm64) + - check-layer-size (python39-arm64) + - lint python + - unit-test (python39-arm64) + - integration-test (python39-arm64) + dependencies: + - build-layer (python39-arm64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.9.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-arm64-3.9.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python39-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python39_arm64_prod.env + needs: + + - sign-layer (python39-arm64) + + dependencies: + + - sign-layer (python39-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.9 ARCH=arm64 DOTENV=python39_arm64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python39-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python39_arm64_sandbox.env + needs: + + - build-layer (python39-arm64) + - check-layer-size (python39-arm64) + - lint python + - unit-test (python39-arm64) + - integration-test (python39-arm64) + + dependencies: + + - build-layer (python39-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.9 ARCH=arm64 DOTENV=python39_arm64_sandbox.env ./ci/publish_layers.sh + + + +# TODO(astuyve) - figure out python build cache +.python310-amd64-cache: &python310-amd64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python310-amd64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.10.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.10 ARCH=amd64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python310-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python310-amd64) + dependencies: + - build-layer (python310-amd64) + script: + - PYTHON_VERSION=3.10 ARCH=amd64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.10 + cache: &python310-amd64-cache + before_script: + - PYTHON_VERSION=3.10 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python310-amd64): + stage: test + tags: + + - "arch:amd64" + + image: registry.ddbuild.io/images/mirror/python:3.10 + cache: &python310-amd64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.10,arch:amd64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.10 ARCH=amd64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python310-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python310-amd64) + dependencies: + - build-layer (python310-amd64) + cache: &python310-amd64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.10 ARCH=amd64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python310-amd64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python310-amd64) + - check-layer-size (python310-amd64) + - lint python + - unit-test (python310-amd64) + - integration-test (python310-amd64) + dependencies: + - build-layer (python310-amd64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.10.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-amd64-3.10.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python310-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python310_amd64_prod.env + needs: + + - sign-layer (python310-amd64) + + dependencies: + + - sign-layer (python310-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.10 ARCH=amd64 DOTENV=python310_amd64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python310-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python310_amd64_sandbox.env + needs: + + - build-layer (python310-amd64) + - check-layer-size (python310-amd64) + - lint python + - unit-test (python310-amd64) + - integration-test (python310-amd64) + + dependencies: + + - build-layer (python310-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.10 ARCH=amd64 DOTENV=python310_amd64_sandbox.env ./ci/publish_layers.sh + + +# TODO(astuyve) - figure out python build cache +.python310-arm64-cache: &python310-arm64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python310-arm64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.10.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.10 ARCH=arm64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python310-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python310-arm64) + dependencies: + - build-layer (python310-arm64) + script: + - PYTHON_VERSION=3.10 ARCH=arm64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.10 + cache: &python310-arm64-cache + before_script: + - PYTHON_VERSION=3.10 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python310-arm64): + stage: test + tags: + + - "arch:arm64" + + image: registry.ddbuild.io/images/mirror/python:3.10 + cache: &python310-arm64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.10,arch:arm64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.10 ARCH=arm64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python310-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python310-arm64) + dependencies: + - build-layer (python310-arm64) + cache: &python310-arm64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.10 ARCH=arm64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python310-arm64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python310-arm64) + - check-layer-size (python310-arm64) + - lint python + - unit-test (python310-arm64) + - integration-test (python310-arm64) + dependencies: + - build-layer (python310-arm64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.10.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-arm64-3.10.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python310-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python310_arm64_prod.env + needs: + + - sign-layer (python310-arm64) + + dependencies: + + - sign-layer (python310-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.10 ARCH=arm64 DOTENV=python310_arm64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python310-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python310_arm64_sandbox.env + needs: + + - build-layer (python310-arm64) + - check-layer-size (python310-arm64) + - lint python + - unit-test (python310-arm64) + - integration-test (python310-arm64) + + dependencies: + + - build-layer (python310-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.10 ARCH=arm64 DOTENV=python310_arm64_sandbox.env ./ci/publish_layers.sh + + + +# TODO(astuyve) - figure out python build cache +.python311-amd64-cache: &python311-amd64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python311-amd64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.11.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.11 ARCH=amd64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python311-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python311-amd64) + dependencies: + - build-layer (python311-amd64) + script: + - PYTHON_VERSION=3.11 ARCH=amd64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.11.6 + cache: &python311-amd64-cache + before_script: + - PYTHON_VERSION=3.11 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python311-amd64): + stage: test + tags: + + - "arch:amd64" + + image: registry.ddbuild.io/images/mirror/python:3.11.6 + cache: &python311-amd64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.11,arch:amd64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.11 ARCH=amd64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python311-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python311-amd64) + dependencies: + - build-layer (python311-amd64) + cache: &python311-amd64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.11 ARCH=amd64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python311-amd64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python311-amd64) + - check-layer-size (python311-amd64) + - lint python + - unit-test (python311-amd64) + - integration-test (python311-amd64) + dependencies: + - build-layer (python311-amd64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.11.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-amd64-3.11.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python311-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python311_amd64_prod.env + needs: + + - sign-layer (python311-amd64) + + dependencies: + + - sign-layer (python311-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.11 ARCH=amd64 DOTENV=python311_amd64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python311-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python311_amd64_sandbox.env + needs: + + - build-layer (python311-amd64) + - check-layer-size (python311-amd64) + - lint python + - unit-test (python311-amd64) + - integration-test (python311-amd64) + + dependencies: + + - build-layer (python311-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.11 ARCH=amd64 DOTENV=python311_amd64_sandbox.env ./ci/publish_layers.sh + + +# TODO(astuyve) - figure out python build cache +.python311-arm64-cache: &python311-arm64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python311-arm64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.11.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.11 ARCH=arm64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python311-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python311-arm64) + dependencies: + - build-layer (python311-arm64) + script: + - PYTHON_VERSION=3.11 ARCH=arm64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.11.6 + cache: &python311-arm64-cache + before_script: + - PYTHON_VERSION=3.11 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python311-arm64): + stage: test + tags: + + - "arch:arm64" + + image: registry.ddbuild.io/images/mirror/python:3.11.6 + cache: &python311-arm64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.11,arch:arm64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.11 ARCH=arm64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python311-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python311-arm64) + dependencies: + - build-layer (python311-arm64) + cache: &python311-arm64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.11 ARCH=arm64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python311-arm64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python311-arm64) + - check-layer-size (python311-arm64) + - lint python + - unit-test (python311-arm64) + - integration-test (python311-arm64) + dependencies: + - build-layer (python311-arm64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.11.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-arm64-3.11.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python311-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python311_arm64_prod.env + needs: + + - sign-layer (python311-arm64) + + dependencies: + + - sign-layer (python311-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.11 ARCH=arm64 DOTENV=python311_arm64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python311-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python311_arm64_sandbox.env + needs: + + - build-layer (python311-arm64) + - check-layer-size (python311-arm64) + - lint python + - unit-test (python311-arm64) + - integration-test (python311-arm64) + + dependencies: + + - build-layer (python311-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.11 ARCH=arm64 DOTENV=python311_arm64_sandbox.env ./ci/publish_layers.sh + + + +# TODO(astuyve) - figure out python build cache +.python312-amd64-cache: &python312-amd64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python312-amd64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.12.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.12 ARCH=amd64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python312-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python312-amd64) + dependencies: + - build-layer (python312-amd64) + script: + - PYTHON_VERSION=3.12 ARCH=amd64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.12.0 + cache: &python312-amd64-cache + before_script: + - PYTHON_VERSION=3.12 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python312-amd64): + stage: test + tags: + + - "arch:amd64" + + image: registry.ddbuild.io/images/mirror/python:3.12.0 + cache: &python312-amd64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.12,arch:amd64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.12 ARCH=amd64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python312-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python312-amd64) + dependencies: + - build-layer (python312-amd64) + cache: &python312-amd64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.12 ARCH=amd64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python312-amd64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python312-amd64) + - check-layer-size (python312-amd64) + - lint python + - unit-test (python312-amd64) + - integration-test (python312-amd64) + dependencies: + - build-layer (python312-amd64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.12.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-amd64-3.12.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python312-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python312_amd64_prod.env + needs: + + - sign-layer (python312-amd64) + + dependencies: + + - sign-layer (python312-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.12 ARCH=amd64 DOTENV=python312_amd64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python312-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python312_amd64_sandbox.env + needs: + + - build-layer (python312-amd64) + - check-layer-size (python312-amd64) + - lint python + - unit-test (python312-amd64) + - integration-test (python312-amd64) + + dependencies: + + - build-layer (python312-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.12 ARCH=amd64 DOTENV=python312_amd64_sandbox.env ./ci/publish_layers.sh + + +# TODO(astuyve) - figure out python build cache +.python312-arm64-cache: &python312-arm64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python312-arm64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.12.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.12 ARCH=arm64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python312-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python312-arm64) + dependencies: + - build-layer (python312-arm64) + script: + - PYTHON_VERSION=3.12 ARCH=arm64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.12.0 + cache: &python312-arm64-cache + before_script: + - PYTHON_VERSION=3.12 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python312-arm64): + stage: test + tags: + + - "arch:arm64" + + image: registry.ddbuild.io/images/mirror/python:3.12.0 + cache: &python312-arm64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.12,arch:arm64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.12 ARCH=arm64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python312-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python312-arm64) + dependencies: + - build-layer (python312-arm64) + cache: &python312-arm64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.12 ARCH=arm64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python312-arm64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python312-arm64) + - check-layer-size (python312-arm64) + - lint python + - unit-test (python312-arm64) + - integration-test (python312-arm64) + dependencies: + - build-layer (python312-arm64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.12.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-arm64-3.12.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python312-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python312_arm64_prod.env + needs: + + - sign-layer (python312-arm64) + + dependencies: + + - sign-layer (python312-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.12 ARCH=arm64 DOTENV=python312_arm64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python312-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python312_arm64_sandbox.env + needs: + + - build-layer (python312-arm64) + - check-layer-size (python312-arm64) + - lint python + - unit-test (python312-arm64) + - integration-test (python312-arm64) + + dependencies: + + - build-layer (python312-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.12 ARCH=arm64 DOTENV=python312_arm64_sandbox.env ./ci/publish_layers.sh + + + +# TODO(astuyve) - figure out python build cache +.python313-amd64-cache: &python313-amd64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python313-amd64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.13.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.13 ARCH=amd64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python313-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python313-amd64) + dependencies: + - build-layer (python313-amd64) + script: + - PYTHON_VERSION=3.13 ARCH=amd64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.13.0 + cache: &python313-amd64-cache + before_script: + - PYTHON_VERSION=3.13 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python313-amd64): + stage: test + tags: + + - "arch:amd64" + + image: registry.ddbuild.io/images/mirror/python:3.13.0 + cache: &python313-amd64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.13,arch:amd64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.13 ARCH=amd64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python313-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python313-amd64) + dependencies: + - build-layer (python313-amd64) + cache: &python313-amd64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.13 ARCH=amd64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python313-amd64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python313-amd64) + - check-layer-size (python313-amd64) + - lint python + - unit-test (python313-amd64) + - integration-test (python313-amd64) + dependencies: + - build-layer (python313-amd64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.13.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-amd64-3.13.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python313-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python313_amd64_prod.env + needs: + + - sign-layer (python313-amd64) + + dependencies: + + - sign-layer (python313-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.13 ARCH=amd64 DOTENV=python313_amd64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python313-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python313_amd64_sandbox.env + needs: + + - build-layer (python313-amd64) + - check-layer-size (python313-amd64) + - lint python + - unit-test (python313-amd64) + - integration-test (python313-amd64) + + dependencies: + + - build-layer (python313-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.13 ARCH=amd64 DOTENV=python313_amd64_sandbox.env ./ci/publish_layers.sh + + +# TODO(astuyve) - figure out python build cache +.python313-arm64-cache: &python313-arm64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python313-arm64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.13.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.13 ARCH=arm64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python313-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python313-arm64) + dependencies: + - build-layer (python313-arm64) + script: + - PYTHON_VERSION=3.13 ARCH=arm64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.13.0 + cache: &python313-arm64-cache + before_script: + - PYTHON_VERSION=3.13 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python313-arm64): + stage: test + tags: + + - "arch:arm64" + + image: registry.ddbuild.io/images/mirror/python:3.13.0 + cache: &python313-arm64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.13,arch:arm64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.13 ARCH=arm64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python313-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python313-arm64) + dependencies: + - build-layer (python313-arm64) + cache: &python313-arm64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.13 ARCH=arm64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python313-arm64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python313-arm64) + - check-layer-size (python313-arm64) + - lint python + - unit-test (python313-arm64) + - integration-test (python313-arm64) + dependencies: + - build-layer (python313-arm64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.13.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-arm64-3.13.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python313-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python313_arm64_prod.env + needs: + + - sign-layer (python313-arm64) + + dependencies: + + - sign-layer (python313-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.13 ARCH=arm64 DOTENV=python313_arm64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python313-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python313_arm64_sandbox.env + needs: + + - build-layer (python313-arm64) + - check-layer-size (python313-arm64) + - lint python + - unit-test (python313-arm64) + - integration-test (python313-arm64) + + dependencies: + + - build-layer (python313-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.13 ARCH=arm64 DOTENV=python313_arm64_sandbox.env ./ci/publish_layers.sh + + + +# TODO(astuyve) - figure out python build cache +.python314-amd64-cache: &python314-amd64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python314-amd64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.14.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.14 ARCH=amd64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python314-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python314-amd64) + dependencies: + - build-layer (python314-amd64) + script: + - PYTHON_VERSION=3.14 ARCH=amd64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.14.0 + cache: &python314-amd64-cache + before_script: + - PYTHON_VERSION=3.14 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python314-amd64): + stage: test + tags: + + - "arch:amd64" + + image: registry.ddbuild.io/images/mirror/python:3.14.0 + cache: &python314-amd64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.14,arch:amd64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.14 ARCH=amd64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python314-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python314-amd64) + dependencies: + - build-layer (python314-amd64) + cache: &python314-amd64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.14 ARCH=amd64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python314-amd64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python314-amd64) + - check-layer-size (python314-amd64) + - lint python + - unit-test (python314-amd64) + - integration-test (python314-amd64) + dependencies: + - build-layer (python314-amd64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.14.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-amd64-3.14.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python314-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python314_amd64_prod.env + needs: + + - sign-layer (python314-amd64) + + dependencies: + + - sign-layer (python314-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.14 ARCH=amd64 DOTENV=python314_amd64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python314-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python314_amd64_sandbox.env + needs: + + - build-layer (python314-amd64) + - check-layer-size (python314-amd64) + - lint python + - unit-test (python314-amd64) + - integration-test (python314-amd64) + + dependencies: + + - build-layer (python314-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.14 ARCH=amd64 DOTENV=python314_amd64_sandbox.env ./ci/publish_layers.sh + + +# TODO(astuyve) - figure out python build cache +.python314-arm64-cache: &python314-arm64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python314-arm64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.14.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.14 ARCH=arm64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python314-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python314-arm64) + dependencies: + - build-layer (python314-arm64) + script: + - PYTHON_VERSION=3.14 ARCH=arm64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.14.0 + cache: &python314-arm64-cache + before_script: + - PYTHON_VERSION=3.14 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python314-arm64): + stage: test + tags: + + - "arch:arm64" + + image: registry.ddbuild.io/images/mirror/python:3.14.0 + cache: &python314-arm64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.14,arch:arm64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.14 ARCH=arm64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python314-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python314-arm64) + dependencies: + - build-layer (python314-arm64) + cache: &python314-arm64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.14 ARCH=arm64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python314-arm64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python314-arm64) + - check-layer-size (python314-arm64) + - lint python + - unit-test (python314-arm64) + - integration-test (python314-arm64) + dependencies: + - build-layer (python314-arm64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.14.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-arm64-3.14.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python314-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python314_arm64_prod.env + needs: + + - sign-layer (python314-arm64) + + dependencies: + + - sign-layer (python314-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.14 ARCH=arm64 DOTENV=python314_arm64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python314-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python314_arm64_sandbox.env + needs: + + - build-layer (python314-arm64) + - check-layer-size (python314-arm64) + - lint python + - unit-test (python314-arm64) + - integration-test (python314-arm64) + + dependencies: + + - build-layer (python314-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.14 ARCH=arm64 DOTENV=python314_arm64_sandbox.env ./ci/publish_layers.sh + + + +# TODO(astuyve) - figure out python build cache +.python315-amd64-cache: &python315-amd64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python315-amd64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.15.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.15 ARCH=amd64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python315-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python315-amd64) + dependencies: + - build-layer (python315-amd64) + script: + - PYTHON_VERSION=3.15 ARCH=amd64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.15.0 + cache: &python315-amd64-cache + before_script: + - PYTHON_VERSION=3.15 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python315-amd64): + stage: test + tags: + + - "arch:amd64" + + image: registry.ddbuild.io/images/mirror/python:3.15.0 + cache: &python315-amd64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.15,arch:amd64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.15 ARCH=amd64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python315-amd64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python315-amd64) + dependencies: + - build-layer (python315-amd64) + cache: &python315-amd64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.15 ARCH=amd64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python315-amd64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python315-amd64) + - check-layer-size (python315-amd64) + - lint python + - unit-test (python315-amd64) + - integration-test (python315-amd64) + dependencies: + - build-layer (python315-amd64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-amd64-3.15.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-amd64-3.15.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python315-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python315_amd64_prod.env + needs: + + - sign-layer (python315-amd64) + + dependencies: + + - sign-layer (python315-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.15 ARCH=amd64 DOTENV=python315_amd64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python315-amd64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python315_amd64_sandbox.env + needs: + + - build-layer (python315-amd64) + - check-layer-size (python315-amd64) + - lint python + - unit-test (python315-amd64) + - integration-test (python315-amd64) + + dependencies: + + - build-layer (python315-amd64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.15 ARCH=amd64 DOTENV=python315_amd64_sandbox.env ./ci/publish_layers.sh + + +# TODO(astuyve) - figure out python build cache +.python315-arm64-cache: &python315-arm64-cache + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + - $CI_PROJECT_DIR/.yarn-cache + policy: pull + +build-layer (python315-arm64): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + artifacts: + expire_in: 1 day # Unsigned zips expire in 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.15.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + script: + - PYTHON_VERSION=3.15 ARCH=arm64 ./scripts/build_layers.sh + timeout: 15m + retry: 2 + +check-layer-size (python315-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python315-arm64) + dependencies: + - build-layer (python315-arm64) + script: + - PYTHON_VERSION=3.15 ARCH=arm64 ./scripts/check_layer_size.sh + +lint python: + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/mirror/python:3.15.0 + cache: &python315-arm64-cache + before_script: + - PYTHON_VERSION=3.15 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh + script: + - source venv/bin/activate + - ./scripts/check_format.sh + +unit-test (python315-arm64): + stage: test + tags: + + - "arch:arm64" + + image: registry.ddbuild.io/images/mirror/python:3.15.0 + cache: &python315-arm64-cache + variables: + DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" + DD_SERVICE: "datadog-lambda-python" + DD_ENV: "ci" + DD_TAGS: "python_version:3.15,arch:arm64" + before_script: + - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh + - PYTHON_VERSION=3.15 ARCH=arm64 ./scripts/setup_python_env.sh + script: + - set -e + - source venv/bin/activate + - pytest -vv --ddtrace + retry: 2 + +integration-test (python315-arm64): + stage: test + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + needs: + - build-layer (python315-arm64) + dependencies: + - build-layer (python315-arm64) + cache: &python315-arm64-cache + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" + before_script: + - *install-node + - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + - yarn global add serverless@^3.38.0 --prefix /usr/local + - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local + - cd integration_tests && yarn install && cd .. + script: + - RUNTIME_PARAM=3.15 ARCH=arm64 ./scripts/run_integration_tests.sh + retry: 2 + +sign-layer (python315-arm64): + stage: sign + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build-layer (python315-arm64) + - check-layer-size (python315-arm64) + - lint python + - unit-test (python315-arm64) + - integration-test (python315-arm64) + dependencies: + - build-layer (python315-arm64) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_lambda_py-arm64-3.15.zip + before_script: + - apt-get update + - apt-get install -y uuid-runtime + + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + + script: + - LAYER_FILE=datadog_lambda_py-arm64-3.15.zip ./scripts/sign_layers.sh prod + + + + +publish-layer-prod (python315-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"prod" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python315_arm64_prod.env + needs: + + - sign-layer (python315-arm64) + + dependencies: + + - sign-layer (python315-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh + script: + - STAGE=prod PYTHON_VERSION=3.15 ARCH=arm64 DOTENV=python315_arm64_prod.env ./ci/publish_layers.sh + + +publish-layer-sandbox (python315-arm64): + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' + when: on_success + - if: '"sandbox" == "sandbox"' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: python315_arm64_sandbox.env + needs: + + - build-layer (python315-arm64) + - check-layer-size (python315-arm64) + - lint python + - unit-test (python315-arm64) + - integration-test (python315-arm64) + + dependencies: + + - build-layer (python315-arm64) + + parallel: + matrix: + - REGION: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - af-south-1 + - ap-east-1 + - ap-east-2 + - ap-south-1 + - ap-south-2 + - ap-southeast-1 + - ap-southeast-2 + - ap-southeast-3 + - ap-southeast-4 + - ap-southeast-5 + - ap-southeast-6 + - ap-southeast-7 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ca-central-1 + - ca-west-1 + - eu-central-1 + - eu-central-2 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-south-1 + - eu-south-2 + - il-central-1 + - me-central-1 + - mx-central-1 + - sa-east-1 + before_script: + - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh + script: + - STAGE=sandbox PYTHON_VERSION=3.15 ARCH=arm64 DOTENV=python315_arm64_sandbox.env ./ci/publish_layers.sh + +publish-pypi-package: + stage: publish + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10-py3 + before_script: + - ./scripts/setup_python_env.sh + cache: [] + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - sign-layer (python38-amd64) + - sign-layer (python38-arm64) + - sign-layer (python39-amd64) + - sign-layer (python39-arm64) + - sign-layer (python310-amd64) + - sign-layer (python310-arm64) + - sign-layer (python311-amd64) + - sign-layer (python311-arm64) + - sign-layer (python312-amd64) + - sign-layer (python312-arm64) + - sign-layer (python313-amd64) + - sign-layer (python313-arm64) + - sign-layer (python314-amd64) + - sign-layer (python314-arm64) + - sign-layer (python315-amd64) + - sign-layer (python315-arm64) + script: + - ./ci/publish_pypi.sh + +layer bundle: + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + - build-layer (python38-amd64) + - build-layer (python38-arm64) + - build-layer (python39-amd64) + - build-layer (python39-arm64) + - build-layer (python310-amd64) + - build-layer (python310-arm64) + - build-layer (python311-amd64) + - build-layer (python311-arm64) + - build-layer (python312-amd64) + - build-layer (python312-arm64) + - build-layer (python313-amd64) + - build-layer (python313-arm64) + - build-layer (python314-amd64) + - build-layer (python314-arm64) + - build-layer (python315-amd64) + - build-layer (python315-arm64) + dependencies: + - build-layer (python38-amd64) + - build-layer (python38-arm64) + - build-layer (python39-amd64) + - build-layer (python39-arm64) + - build-layer (python310-amd64) + - build-layer (python310-arm64) + - build-layer (python311-amd64) + - build-layer (python311-arm64) + - build-layer (python312-amd64) + - build-layer (python312-arm64) + - build-layer (python313-amd64) + - build-layer (python313-arm64) + - build-layer (python314-amd64) + - build-layer (python314-arm64) + - build-layer (python315-amd64) + - build-layer (python315-arm64) + artifacts: + expire_in: 1 day + paths: + - datadog_lambda_py-bundle-${CI_JOB_ID}/ + name: datadog_lambda_py-bundle-${CI_JOB_ID} + script: + - rm -rf datadog_lambda_py-bundle-${CI_JOB_ID} + - mkdir -p datadog_lambda_py-bundle-${CI_JOB_ID} + - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-bundle-${CI_JOB_ID} + +signed layer bundle: + stage: sign + image: registry.ddbuild.io/images/docker:20.10-py3 + tags: ["arch:amd64"] + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: + - sign-layer (python38-amd64) + - sign-layer (python38-arm64) + - sign-layer (python39-amd64) + - sign-layer (python39-arm64) + - sign-layer (python310-amd64) + - sign-layer (python310-arm64) + - sign-layer (python311-amd64) + - sign-layer (python311-arm64) + - sign-layer (python312-amd64) + - sign-layer (python312-arm64) + - sign-layer (python313-amd64) + - sign-layer (python313-arm64) + - sign-layer (python314-amd64) + - sign-layer (python314-arm64) + - sign-layer (python315-amd64) + - sign-layer (python315-arm64) + dependencies: + - sign-layer (python38-amd64) + - sign-layer (python38-arm64) + - sign-layer (python39-amd64) + - sign-layer (python39-arm64) + - sign-layer (python310-amd64) + - sign-layer (python310-arm64) + - sign-layer (python311-amd64) + - sign-layer (python311-arm64) + - sign-layer (python312-amd64) + - sign-layer (python312-arm64) + - sign-layer (python313-amd64) + - sign-layer (python313-arm64) + - sign-layer (python314-amd64) + - sign-layer (python314-arm64) + - sign-layer (python315-amd64) + - sign-layer (python315-arm64) + artifacts: + expire_in: 1 day + paths: + - datadog_lambda_py-signed-bundle-${CI_JOB_ID}/ + name: datadog_lambda_py-signed-bundle-${CI_JOB_ID} + script: + - rm -rf datadog_lambda_py-signed-bundle-${CI_JOB_ID} + - mkdir -p datadog_lambda_py-signed-bundle-${CI_JOB_ID} + - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-signed-bundle-${CI_JOB_ID} + +e2e-test: + stage: e2e + trigger: + project: DataDog/serverless-e2e-tests + strategy: depend + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + variables: + LANGUAGES_SUBSET: python + # These env vars are inherited from the dotenv reports of the publish-layer jobs + PYTHON_38_VERSION: $PYTHON_38_VERSION + PYTHON_39_VERSION: $PYTHON_39_VERSION + PYTHON_310_VERSION: $PYTHON_310_VERSION + PYTHON_311_VERSION: $PYTHON_311_VERSION + PYTHON_312_VERSION: $PYTHON_312_VERSION + PYTHON_313_VERSION: $PYTHON_313_VERSION + PYTHON_314_VERSION: $PYTHON_314_VERSION + PYTHON_315_VERSION: $PYTHON_315_VERSION + needs: + - "publish-layer-sandbox (python38-amd64): [us-west-2]" + - "publish-layer-sandbox (python39-amd64): [us-west-2]" + - "publish-layer-sandbox (python310-amd64): [us-west-2]" + - "publish-layer-sandbox (python311-amd64): [us-west-2]" + - "publish-layer-sandbox (python312-amd64): [us-west-2]" + - "publish-layer-sandbox (python313-amd64): [us-west-2]" + - "publish-layer-sandbox (python314-amd64): [us-west-2]" + - "publish-layer-sandbox (python315-amd64): [us-west-2]" + +e2e-test-status: + stage: e2e + image: registry.ddbuild.io/images/docker:20.10-py3 + tags: ["arch:amd64"] + timeout: 3h + rules: + - if: '$SKIP_E2E_TESTS == "true"' + when: never + - when: on_success + script: + - ci/poll_e2e.sh diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl index 7c03cce97..8e37420af 100644 --- a/ci/input_files/build.yaml.tpl +++ b/ci/input_files/build.yaml.tpl @@ -1,6 +1,8 @@ {{- $e2e_region := "us-west-2" -}} -{{- $archs := coll.Slice "amd64" "arm64" -}} -{{- $runtime := dict -}} +{{- $runtimes := coll.Slice -}} +{{- range $py := (ds "python_versions").versions }}{{ range $arch := coll.Slice "amd64" "arm64" }} +{{- $runtimes = $runtimes | coll.Append (dict "name" (printf "python%s" ($py.version | strings.ReplaceAll "." "")) "python_version" $py.version "arch" $arch "image" $py.image) }} +{{- end }}{{ end -}} stages: - build @@ -28,9 +30,7 @@ default: - apt-get install nodejs -y - npm install --global yarn -{{ range $py := (ds "python_versions").versions }} -{{ range $arch := $archs }} -{{ $runtime = dict "name" (printf "python%s" ($py.version | strings.ReplaceAll "." "")) "arch" $arch "python_version" $py.version "image" $py.image }} +{{ range $runtime := $runtimes }} # TODO(astuyve) - figure out python build cache .{{ $runtime.name }}-{{ $runtime.arch }}-cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache @@ -200,7 +200,6 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}): {{- end }} -{{- end }} {{- end }} publish-pypi-package: @@ -213,12 +212,9 @@ publish-pypi-package: rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' when: manual - needs: - {{- range $py := (ds "python_versions").versions }} - {{- range $arch := $archs }} - - sign-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) - {{- end }} - {{- end }} + needs: {{ range $runtime := $runtimes }} + - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}}) + {{- end }} script: - ./ci/publish_pypi.sh @@ -227,17 +223,13 @@ layer bundle: tags: ["arch:amd64"] image: registry.ddbuild.io/images/docker:20.10 needs: - {{- range $py := (ds "python_versions").versions }} - {{- range $arch := $archs }} - - build-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) - {{- end }} - {{- end }} + {{ range $runtimes }} + - build-layer ({{ .name }}-{{ .arch }}) + {{ end }} dependencies: - {{- range $py := (ds "python_versions").versions }} - {{- range $arch := $archs }} - - build-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) - {{- end }} - {{- end }} + {{ range $runtimes }} + - build-layer ({{ .name }}-{{ .arch }}) + {{ end }} artifacts: expire_in: 1 day paths: @@ -255,17 +247,13 @@ signed layer bundle: rules: - if: '$CI_COMMIT_TAG =~ /^v.*/' needs: - {{- range $py := (ds "python_versions").versions }} - {{- range $arch := $archs }} - - sign-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) - {{- end }} - {{- end }} + {{ range $runtimes }} + - sign-layer ({{ .name }}-{{ .arch }}) + {{ end }} dependencies: - {{- range $py := (ds "python_versions").versions }} - {{- range $arch := $archs }} - - sign-layer (python{{ $py.version | strings.ReplaceAll "." "" }}-{{ $arch }}) - {{- end }} - {{- end }} + {{ range $runtimes }} + - sign-layer ({{ .name }}-{{ .arch }}) + {{ end }} artifacts: expire_in: 1 day paths: @@ -288,12 +276,16 @@ e2e-test: variables: LANGUAGES_SUBSET: python # These env vars are inherited from the dotenv reports of the publish-layer jobs - {{- range $py := (ds "python_versions").versions }} - PYTHON_{{ $py.version | strings.ReplaceAll "." "" }}_VERSION: $PYTHON_{{ $py.version | strings.ReplaceAll "." "" }}_VERSION + {{- range $runtimes }} + {{- if eq .arch "amd64" }} + {{- $version := print (.name | strings.Trim "python") }} + PYTHON_{{ $version }}_VERSION: $PYTHON_{{ $version }}_VERSION + {{- end }} + {{- end }} + needs: {{ range $runtimes }} + {{- if eq .arch "amd64" }} + - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]" {{- end }} - needs: - {{- range $py := (ds "python_versions").versions }} - - "publish-layer-sandbox (python{{ $py.version | strings.ReplaceAll "." "" }}-amd64): [{{ $e2e_region }}]" {{- end }} e2e-test-status: diff --git a/ci/publish_layers.sh b/ci/publish_layers.sh index 51bada705..06f289bc1 100755 --- a/ci/publish_layers.sh +++ b/ci/publish_layers.sh @@ -11,15 +11,9 @@ set -e # Available runtimes: https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html source "$(dirname "${BASH_SOURCE[0]}")/../scripts/python_versions.sh" -load_python_versions -load_python_layers AWS_CLI_PYTHON_VERSIONS=("${LAYER_RUNTIMES[@]}") LAYERS=("${LAYER_NAMES[@]}") -PYTHON_VERSIONS=() -for _i in "${!LAYER_VERSIONS[@]}"; do - PYTHON_VERSIONS+=("${LAYER_VERSIONS[$_i]}-${LAYER_ARCHS[$_i]}") -done -unset _i +PYTHON_VERSIONS=("${LAYER_KEYS[@]}") STAGES=('prod', 'sandbox', 'staging', 'gov-staging', 'gov-prod') printf "Starting script...\n\n" diff --git a/scripts/add_new_region.sh b/scripts/add_new_region.sh index 704ceb484..411b3aba4 100755 --- a/scripts/add_new_region.sh +++ b/scripts/add_new_region.sh @@ -13,8 +13,6 @@ set -e OLD_REGION='us-east-1' source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" -load_python_versions -load_python_layers PYTHON_VERSIONS_FOR_AWS_CLI=("${LAYER_RUNTIMES[@]}") NEW_REGION=$1 diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index b250ad9fa..6954184a0 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -37,7 +37,6 @@ LAYER_DIR=".layers" LAYER_FILES_PREFIX="datadog_lambda_py" # Versions and the preview flag live in python_versions.yaml. source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" -load_python_versions AVAILABLE_ARCHS=("arm64" "amd64") if [ -n "$ARCH" ]; then @@ -74,7 +73,7 @@ if [ -z "$PYTHON_VERSION" ]; then PYTHON_VERSIONS=("${SUPPORTED_PYTHON_VERSIONS[@]}") else echo "Python version specified: $PYTHON_VERSION" - if ! python_version_known "$PYTHON_VERSION"; then + if [[ ! " ${SUPPORTED_PYTHON_VERSIONS[@]} " =~ " ${PYTHON_VERSION} " ]]; then echo "Python version $PYTHON_VERSION is not a valid option. Choose from: ${SUPPORTED_PYTHON_VERSIONS[*]}" echo "" echo "EXITING SCRIPT." @@ -107,7 +106,7 @@ function docker_build_zip { arch=$3 # Preview runtimes have no SAM build image yet. The Dockerfile installs # the compiler toolchain that the Lambda preview image is missing. - if python_version_is_preview "$1"; then + if [[ " ${PREVIEW_PYTHON_VERSIONS[*]} " =~ " ${1} " ]]; then image="public.ecr.aws/lambda/python:${1}-preview" else image="public.ecr.aws/sam/build-python${1}:1" diff --git a/scripts/list_layers.sh b/scripts/list_layers.sh index 6fa1dfeb9..68617283a 100755 --- a/scripts/list_layers.sh +++ b/scripts/list_layers.sh @@ -11,8 +11,6 @@ set -e source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" -load_python_versions -load_python_layers AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') LAYERS_MISSING_REGIONS=() diff --git a/scripts/publish_govcloud.sh b/scripts/publish_govcloud.sh index 9217b068a..1ac6d0eb1 100755 --- a/scripts/publish_govcloud.sh +++ b/scripts/publish_govcloud.sh @@ -91,7 +91,6 @@ else fi source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" -load_python_versions for region in $REGIONS do diff --git a/scripts/publish_layers.sh b/scripts/publish_layers.sh index bf9a06f58..ac76fed24 100755 --- a/scripts/publish_layers.sh +++ b/scripts/publish_layers.sh @@ -14,8 +14,6 @@ set -e trap "pkill -P $$; exit 1;" INT source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" -load_python_versions -load_python_layers PYTHON_VERSIONS_FOR_AWS_CLI=("${LAYER_RUNTIMES[@]}") AVAILABLE_LAYERS=("${LAYER_NAMES[@]}") AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') diff --git a/scripts/python_versions.sh b/scripts/python_versions.sh old mode 100755 new mode 100644 index d971ef7a0..4a8e704ab --- a/scripts/python_versions.sh +++ b/scripts/python_versions.sh @@ -5,329 +5,42 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2026 Datadog, Inc. -# Reads python_versions.yaml. Source this file; do not execute it from other -# scripts. Pure bash (3.2+) so layer builds do not need jq, yq, or Python. +# Reads python_versions.yaml. Source this file: # # source scripts/python_versions.sh -# load_python_versions # -# After load_python_versions: -# SUPPORTED_PYTHON_VERSIONS "3.8" "3.9" ... -# SUPPORTED_PYTHON_VERSION_IMAGES parallel to versions -# SUPPORTED_PYTHON_VERSION_PREVIEW "true" / "false" -# SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS "true" / "false" -# -# python_version_known -# python_version_is_preview -# python_versions_json [system_tests|preview] -# -# After load_python_layers (calls load_python_versions if needed), one entry -# per version for amd64 then arm64: -# LAYER_NAMES Datadog-Python38, Datadog-Python38-ARM, ... -# LAYER_RUNTIMES python3.8, python3.8, ... -# LAYER_ZIPS datadog_lambda_py-amd64-3.8.zip, ... -# LAYER_PATHS .layers/datadog_lambda_py-amd64-3.8.zip, ... -# LAYER_VERSIONS 3.8, 3.8, ... -# LAYER_ARCHS amd64, arm64, ... - -_PYTHON_VERSIONS_FILE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/python_versions.yaml" - -_pv_trim() { - local s="$1" - s="${s#"${s%%[![:space:]]*}"}" - s="${s%"${s##*[![:space:]]}"}" - printf '%s' "$s" -} - -_pv_unquote() { - local s="$1" - local first last - first="${s%"${s#?}"}" - last="${s#"${s%?}"}" - if [ ${#s} -ge 2 ] && [ "$first" = "$last" ] && { [ "$first" = '"' ] || [ "$first" = "'" ]; }; then - s="${s#?}" - s="${s%?}" - fi - printf '%s' "$s" -} - -_pv_reset_entry() { - _pv_version="" - _pv_image="" - _pv_preview="" - _pv_system_tests="" -} - -_pv_flush_entry() { - if [ -z "${_pv_version}" ]; then - return 0 - fi - local existing - if [ "${#SUPPORTED_PYTHON_VERSIONS[@]}" -gt 0 ]; then - for existing in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do - if [ "$existing" = "$_pv_version" ]; then - echo "duplicate python version in ${PYTHON_VERSIONS_FILE}: ${_pv_version}" >&2 - return 1 - fi - done - fi - if [ -z "${_pv_image}" ]; then - echo "python version ${_pv_version} is missing image in ${PYTHON_VERSIONS_FILE}" >&2 - return 1 - fi - if [ -z "${_pv_preview}" ]; then - _pv_preview="false" - fi - if [ -z "${_pv_system_tests}" ]; then - _pv_system_tests="true" - fi - SUPPORTED_PYTHON_VERSIONS+=("$_pv_version") - SUPPORTED_PYTHON_VERSION_IMAGES+=("$_pv_image") - SUPPORTED_PYTHON_VERSION_PREVIEW+=("$_pv_preview") - SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS+=("$_pv_system_tests") - _pv_reset_entry -} - -_pv_set_field() { - local key="$1" - local value="$2" - case "$key" in - image) _pv_image="$value" ;; - preview) - case "$value" in - true|false) _pv_preview="$value" ;; - *) - echo "python version ${_pv_version}: preview must be true or false, got '${value}'" >&2 - return 1 - ;; - esac - ;; - system_tests) - case "$value" in - true|false) _pv_system_tests="$value" ;; - *) - echo "python version ${_pv_version}: system_tests must be true or false, got '${value}'" >&2 - return 1 - ;; - esac - ;; - *) - echo "unknown field '${key}' in ${PYTHON_VERSIONS_FILE}" >&2 - return 1 - ;; - esac -} - -load_python_versions() { - PYTHON_VERSIONS_FILE="${PYTHON_VERSIONS_FILE:-$_PYTHON_VERSIONS_FILE}" - if [ ! -f "$PYTHON_VERSIONS_FILE" ]; then - echo "python versions config not found: $PYTHON_VERSIONS_FILE" >&2 - return 1 - fi - - SUPPORTED_PYTHON_VERSIONS=() - SUPPORTED_PYTHON_VERSION_IMAGES=() - SUPPORTED_PYTHON_VERSION_PREVIEW=() - SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS=() - _python_versions_loaded=0 - _pv_reset_entry - - local line key value seen_versions=0 - while IFS= read -r line || [ -n "$line" ]; do - line="${line%%#*}" - line="${line%$'\r'}" - line="$(_pv_trim "$line")" - if [ -z "$line" ]; then - continue - fi - if [ "$line" = "versions:" ]; then - seen_versions=1 - continue - fi - if [ "$seen_versions" -ne 1 ]; then - echo "unexpected line in ${PYTHON_VERSIONS_FILE}: ${line}" >&2 - return 1 - fi - if [[ "$line" =~ ^-[[:space:]]+version:[[:space:]]*(.*)$ ]]; then - _pv_flush_entry || return 1 - value="$(_pv_unquote "$(_pv_trim "${BASH_REMATCH[1]}")")" - if [[ ! "$value" =~ ^[0-9]+\.[0-9]+$ ]]; then - echo "invalid python version '${value}' in ${PYTHON_VERSIONS_FILE}" >&2 - return 1 - fi - _pv_version="$value" - continue - fi - if [ -z "${_pv_version}" ]; then - echo "field before version in ${PYTHON_VERSIONS_FILE}: ${line}" >&2 - return 1 - fi - if [[ "$line" =~ ^([A-Za-z_]+):[[:space:]]*(.*)$ ]]; then - key="${BASH_REMATCH[1]}" - value="$(_pv_unquote "$(_pv_trim "${BASH_REMATCH[2]}")")" - _pv_set_field "$key" "$value" || return 1 - continue - fi - echo "could not parse line in ${PYTHON_VERSIONS_FILE}: ${line}" >&2 - return 1 - done < "$PYTHON_VERSIONS_FILE" - - _pv_flush_entry || return 1 - if [ "${#SUPPORTED_PYTHON_VERSIONS[@]}" -eq 0 ]; then - echo "no python versions found in ${PYTHON_VERSIONS_FILE}" >&2 - return 1 - fi - _python_versions_loaded=1 -} - -_python_version_index() { - local target="$1" - local i - if [ "${_python_versions_loaded:-0}" -ne 1 ]; then - echo "load_python_versions must be called first" >&2 - return 1 - fi - for i in "${!SUPPORTED_PYTHON_VERSIONS[@]}"; do - if [ "${SUPPORTED_PYTHON_VERSIONS[$i]}" = "$target" ]; then - printf '%s' "$i" - return 0 - fi - done +# Sets: +# SUPPORTED_PYTHON_VERSIONS 3.8 3.9 ... +# PREVIEW_PYTHON_VERSIONS versions with preview: true +# and, one entry per version for amd64 then arm64: +# LAYER_NAMES Datadog-Python38, Datadog-Python38-ARM, ... +# LAYER_RUNTIMES python3.8, python3.8, ... +# LAYER_KEYS 3.8-amd64, 3.8-arm64, ... +# LAYER_ZIPS datadog_lambda_py-amd64-3.8.zip, ... +# LAYER_PATHS .layers/datadog_lambda_py-amd64-3.8.zip, ... + +_pv_file="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/python_versions.yaml" + +SUPPORTED_PYTHON_VERSIONS=($(sed -nE 's/^[[:space:]]*- version:[[:space:]]*"?([0-9]+\.[0-9]+)"?.*/\1/p' "$_pv_file")) +PREVIEW_PYTHON_VERSIONS=($(awk -F'"' '/- version:/{v=$2} /^[[:space:]]*preview:[[:space:]]*true/{print v}' "$_pv_file")) +if [ ${#SUPPORTED_PYTHON_VERSIONS[@]} -eq 0 ]; then + echo "no python versions found in $_pv_file" >&2 return 1 -} - -python_version_known() { - _python_version_index "$1" >/dev/null -} - -python_version_is_preview() { - local i - i="$(_python_version_index "$1")" || return 1 - [ "${SUPPORTED_PYTHON_VERSION_PREVIEW[$i]}" = "true" ] -} - -python_runtime_name() { - printf 'python%s' "$1" -} - -python_layer_name() { - local version="$1" - local arch="$2" - local suffix="${version//./}" - case "$arch" in - amd64) printf 'Datadog-Python%s' "$suffix" ;; - arm64) printf 'Datadog-Python%s-ARM' "$suffix" ;; - *) - echo "unknown architecture '${arch}'" >&2 - return 1 - ;; - esac -} - -python_layer_zip() { - printf 'datadog_lambda_py-%s-%s.zip' "$2" "$1" -} - -python_layer_path() { - printf '.layers/%s' "$(python_layer_zip "$1" "$2")" -} - -python_layer_matrix() { - if [ "${_python_versions_loaded:-0}" -ne 1 ]; then - echo "load_python_versions must be called first" >&2 - return 1 - fi - local version - for version in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do - printf '%s %s\n' "$version" "amd64" - printf '%s %s\n' "$version" "arm64" - done -} - -load_python_layers() { - if [ "${_python_versions_loaded:-0}" -ne 1 ]; then - load_python_versions || return 1 - fi - LAYER_NAMES=() - LAYER_PATHS=() - LAYER_ZIPS=() - LAYER_RUNTIMES=() - LAYER_VERSIONS=() - LAYER_ARCHS=() - local version arch - while IFS=' ' read -r version arch; do - if [ -z "$version" ]; then - continue - fi - LAYER_VERSIONS+=("$version") - LAYER_ARCHS+=("$arch") - LAYER_NAMES+=("$(python_layer_name "$version" "$arch")") - LAYER_ZIPS+=("$(python_layer_zip "$version" "$arch")") - LAYER_PATHS+=("$(python_layer_path "$version" "$arch")") - LAYER_RUNTIMES+=("$(python_runtime_name "$version")") - done <&2 - return 1 - fi - local json="[" - local first=1 - local i version include - for i in "${!SUPPORTED_PYTHON_VERSIONS[@]}"; do - version="${SUPPORTED_PYTHON_VERSIONS[$i]}" - include=1 - case "$only" in - "") ;; - system_tests) - if [ "${SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS[$i]}" != "true" ]; then - include=0 - fi - ;; - preview) - if [ "${SUPPORTED_PYTHON_VERSION_PREVIEW[$i]}" != "true" ]; then - include=0 - fi - ;; - *) - echo "unknown python_versions_json filter: ${only}" >&2 - return 1 - ;; - esac - if [ "$include" -eq 1 ]; then - if [ "$first" -eq 0 ]; then - json="${json}," - fi - first=0 - json="${json}\"${version}\"" - fi - done - json="${json}]" - printf '%s' "$json" -} +fi -if [ "${BASH_SOURCE[0]}" = "$0" ]; then - load_python_versions - load_python_layers - i=0 - while [ "$i" -lt "${#SUPPORTED_PYTHON_VERSIONS[@]}" ]; do - extra="" - if [ "${SUPPORTED_PYTHON_VERSION_PREVIEW[$i]}" = "true" ]; then - extra="${extra} preview" - fi - if [ "${SUPPORTED_PYTHON_VERSION_SYSTEM_TESTS[$i]}" != "true" ]; then - extra="${extra} system_tests=false" - fi - printf '%s image=%s%s\n' \ - "${SUPPORTED_PYTHON_VERSIONS[$i]}" \ - "${SUPPORTED_PYTHON_VERSION_IMAGES[$i]}" \ - "$extra" - i=$((i + 1)) +LAYER_NAMES=() +LAYER_RUNTIMES=() +LAYER_KEYS=() +LAYER_ZIPS=() +for _pv_version in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do + for _pv_arch in amd64 arm64; do + _pv_suffix="" + [ "$_pv_arch" = arm64 ] && _pv_suffix="-ARM" + LAYER_NAMES+=("Datadog-Python${_pv_version//./}${_pv_suffix}") + LAYER_RUNTIMES+=("python${_pv_version}") + LAYER_KEYS+=("${_pv_version}-${_pv_arch}") + LAYER_ZIPS+=("datadog_lambda_py-${_pv_arch}-${_pv_version}.zip") done -fi +done +LAYER_PATHS=("${LAYER_ZIPS[@]/#/.layers/}") +unset _pv_file _pv_version _pv_arch _pv_suffix diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 06c0e83b7..dff7ac130 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -28,24 +28,23 @@ mismatch_found=false # [1]: python version # [2]: random 8-character ID to avoid collisions with other runs source "$scripts_dir/python_versions.sh" -load_python_versions PARAMETERS_SETS=() for version in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do key="python${version//./}" - eval "${key}=(\"python${version}\" \"${version}\" \"$(xxd -l 4 -c 4 -p < /dev/random)\")" + declare -a "${key}=(python${version} ${version} $(xxd -l 4 -c 4 -p < /dev/random))" PARAMETERS_SETS+=("$key") done if [ -z "$RUNTIME_PARAM" ]; then echo "Python version not specified, running for all python versions." else + RUNTIME_PARAM_NO_DOT=${RUNTIME_PARAM//./} echo "Python version is specified: $RUNTIME_PARAM" - if ! python_version_known "$RUNTIME_PARAM"; then + if [[ ! " ${SUPPORTED_PYTHON_VERSIONS[*]} " =~ " ${RUNTIME_PARAM} " ]]; then echo "Python version $RUNTIME_PARAM is not a valid option. Choose from: ${SUPPORTED_PYTHON_VERSIONS[*]}" exit 1 fi - RUNTIME_PARAM_NO_DOT=$(echo $RUNTIME_PARAM | sed 's/\.//') PARAMETERS_SETS=(python${RUNTIME_PARAM_NO_DOT}) BUILD_LAYER_VERSION=python$RUNTIME_PARAM_NO_DOT[1] fi diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 59f697be2..ee18fce29 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -9,7 +9,6 @@ set -e source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" -load_python_versions for python_version in "${SUPPORTED_PYTHON_VERSIONS[@]}" do diff --git a/scripts/sign_layers.sh b/scripts/sign_layers.sh index 23164cf87..340e86a78 100755 --- a/scripts/sign_layers.sh +++ b/scripts/sign_layers.sh @@ -9,8 +9,6 @@ set -e LAYER_DIR=".layers" source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh" -load_python_versions -load_python_layers LAYER_FILES=("${LAYER_ZIPS[@]}") SIGNING_PROFILE_NAME="DatadogLambdaSigningProfile" From 45b747b5f4a14009ac466487144092414a22bb4a Mon Sep 17 00:00:00 2001 From: Rithika Narayan Date: Wed, 18 Nov 2026 16:03:00 -0500 Subject: [PATCH 3/3] clean diff --- ci/build-pipeline.yaml | 4333 ---------------------------------------- 1 file changed, 4333 deletions(-) delete mode 100644 ci/build-pipeline.yaml diff --git a/ci/build-pipeline.yaml b/ci/build-pipeline.yaml deleted file mode 100644 index 3ee140e46..000000000 --- a/ci/build-pipeline.yaml +++ /dev/null @@ -1,4333 +0,0 @@ -stages: - - build - - test - - sign - - publish - - e2e - -default: - retry: - max: 1 - when: - # Retry when the runner fails to start - - runner_system_failure - -# This is for serverless framework -.install-node: &install-node - - apt-get update - - apt-get install -y ca-certificates curl gnupg xxd - - mkdir -p /etc/apt/keyrings - - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg - # We are explicitly setting the node_20.x version for the installation - - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list - - apt-get update - - apt-get install nodejs -y - - npm install --global yarn - - - - - -# TODO(astuyve) - figure out python build cache -.python38-amd64-cache: &python38-amd64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python38-amd64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.8.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.8 ARCH=amd64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python38-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python38-amd64) - dependencies: - - build-layer (python38-amd64) - script: - - PYTHON_VERSION=3.8 ARCH=amd64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.8 - cache: &python38-amd64-cache - before_script: - - PYTHON_VERSION=3.8 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python38-amd64): - stage: test - tags: - - - "arch:amd64" - - image: registry.ddbuild.io/images/mirror/python:3.8 - cache: &python38-amd64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.8,arch:amd64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.8 ARCH=amd64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python38-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python38-amd64) - dependencies: - - build-layer (python38-amd64) - cache: &python38-amd64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.8 ARCH=amd64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python38-amd64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python38-amd64) - - check-layer-size (python38-amd64) - - lint python - - unit-test (python38-amd64) - - integration-test (python38-amd64) - dependencies: - - build-layer (python38-amd64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.8.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-amd64-3.8.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python38-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python38_amd64_prod.env - needs: - - - sign-layer (python38-amd64) - - dependencies: - - - sign-layer (python38-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.8 ARCH=amd64 DOTENV=python38_amd64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python38-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python38_amd64_sandbox.env - needs: - - - build-layer (python38-amd64) - - check-layer-size (python38-amd64) - - lint python - - unit-test (python38-amd64) - - integration-test (python38-amd64) - - dependencies: - - - build-layer (python38-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.8 ARCH=amd64 DOTENV=python38_amd64_sandbox.env ./ci/publish_layers.sh - - -# TODO(astuyve) - figure out python build cache -.python38-arm64-cache: &python38-arm64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python38-arm64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.8.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.8 ARCH=arm64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python38-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python38-arm64) - dependencies: - - build-layer (python38-arm64) - script: - - PYTHON_VERSION=3.8 ARCH=arm64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.8 - cache: &python38-arm64-cache - before_script: - - PYTHON_VERSION=3.8 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python38-arm64): - stage: test - tags: - - - "arch:arm64" - - image: registry.ddbuild.io/images/mirror/python:3.8 - cache: &python38-arm64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.8,arch:arm64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.8 ARCH=arm64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python38-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python38-arm64) - dependencies: - - build-layer (python38-arm64) - cache: &python38-arm64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.8 ARCH=arm64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python38-arm64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python38-arm64) - - check-layer-size (python38-arm64) - - lint python - - unit-test (python38-arm64) - - integration-test (python38-arm64) - dependencies: - - build-layer (python38-arm64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.8.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-arm64-3.8.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python38-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python38_arm64_prod.env - needs: - - - sign-layer (python38-arm64) - - dependencies: - - - sign-layer (python38-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.8 ARCH=arm64 DOTENV=python38_arm64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python38-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python38_arm64_sandbox.env - needs: - - - build-layer (python38-arm64) - - check-layer-size (python38-arm64) - - lint python - - unit-test (python38-arm64) - - integration-test (python38-arm64) - - dependencies: - - - build-layer (python38-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.8 ARCH=arm64 DOTENV=python38_arm64_sandbox.env ./ci/publish_layers.sh - - - -# TODO(astuyve) - figure out python build cache -.python39-amd64-cache: &python39-amd64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python39-amd64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.9.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.9 ARCH=amd64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python39-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python39-amd64) - dependencies: - - build-layer (python39-amd64) - script: - - PYTHON_VERSION=3.9 ARCH=amd64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.9 - cache: &python39-amd64-cache - before_script: - - PYTHON_VERSION=3.9 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python39-amd64): - stage: test - tags: - - - "arch:amd64" - - image: registry.ddbuild.io/images/mirror/python:3.9 - cache: &python39-amd64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.9,arch:amd64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.9 ARCH=amd64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python39-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python39-amd64) - dependencies: - - build-layer (python39-amd64) - cache: &python39-amd64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.9 ARCH=amd64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python39-amd64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python39-amd64) - - check-layer-size (python39-amd64) - - lint python - - unit-test (python39-amd64) - - integration-test (python39-amd64) - dependencies: - - build-layer (python39-amd64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.9.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-amd64-3.9.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python39-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python39_amd64_prod.env - needs: - - - sign-layer (python39-amd64) - - dependencies: - - - sign-layer (python39-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.9 ARCH=amd64 DOTENV=python39_amd64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python39-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python39_amd64_sandbox.env - needs: - - - build-layer (python39-amd64) - - check-layer-size (python39-amd64) - - lint python - - unit-test (python39-amd64) - - integration-test (python39-amd64) - - dependencies: - - - build-layer (python39-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.9 ARCH=amd64 DOTENV=python39_amd64_sandbox.env ./ci/publish_layers.sh - - -# TODO(astuyve) - figure out python build cache -.python39-arm64-cache: &python39-arm64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python39-arm64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.9.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.9 ARCH=arm64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python39-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python39-arm64) - dependencies: - - build-layer (python39-arm64) - script: - - PYTHON_VERSION=3.9 ARCH=arm64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.9 - cache: &python39-arm64-cache - before_script: - - PYTHON_VERSION=3.9 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python39-arm64): - stage: test - tags: - - - "arch:arm64" - - image: registry.ddbuild.io/images/mirror/python:3.9 - cache: &python39-arm64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.9,arch:arm64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.9 ARCH=arm64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python39-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python39-arm64) - dependencies: - - build-layer (python39-arm64) - cache: &python39-arm64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.9 ARCH=arm64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python39-arm64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python39-arm64) - - check-layer-size (python39-arm64) - - lint python - - unit-test (python39-arm64) - - integration-test (python39-arm64) - dependencies: - - build-layer (python39-arm64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.9.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-arm64-3.9.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python39-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python39_arm64_prod.env - needs: - - - sign-layer (python39-arm64) - - dependencies: - - - sign-layer (python39-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.9 ARCH=arm64 DOTENV=python39_arm64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python39-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python39_arm64_sandbox.env - needs: - - - build-layer (python39-arm64) - - check-layer-size (python39-arm64) - - lint python - - unit-test (python39-arm64) - - integration-test (python39-arm64) - - dependencies: - - - build-layer (python39-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.9 ARCH=arm64 DOTENV=python39_arm64_sandbox.env ./ci/publish_layers.sh - - - -# TODO(astuyve) - figure out python build cache -.python310-amd64-cache: &python310-amd64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python310-amd64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.10.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.10 ARCH=amd64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python310-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python310-amd64) - dependencies: - - build-layer (python310-amd64) - script: - - PYTHON_VERSION=3.10 ARCH=amd64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.10 - cache: &python310-amd64-cache - before_script: - - PYTHON_VERSION=3.10 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python310-amd64): - stage: test - tags: - - - "arch:amd64" - - image: registry.ddbuild.io/images/mirror/python:3.10 - cache: &python310-amd64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.10,arch:amd64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.10 ARCH=amd64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python310-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python310-amd64) - dependencies: - - build-layer (python310-amd64) - cache: &python310-amd64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.10 ARCH=amd64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python310-amd64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python310-amd64) - - check-layer-size (python310-amd64) - - lint python - - unit-test (python310-amd64) - - integration-test (python310-amd64) - dependencies: - - build-layer (python310-amd64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.10.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-amd64-3.10.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python310-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python310_amd64_prod.env - needs: - - - sign-layer (python310-amd64) - - dependencies: - - - sign-layer (python310-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.10 ARCH=amd64 DOTENV=python310_amd64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python310-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python310_amd64_sandbox.env - needs: - - - build-layer (python310-amd64) - - check-layer-size (python310-amd64) - - lint python - - unit-test (python310-amd64) - - integration-test (python310-amd64) - - dependencies: - - - build-layer (python310-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.10 ARCH=amd64 DOTENV=python310_amd64_sandbox.env ./ci/publish_layers.sh - - -# TODO(astuyve) - figure out python build cache -.python310-arm64-cache: &python310-arm64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python310-arm64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.10.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.10 ARCH=arm64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python310-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python310-arm64) - dependencies: - - build-layer (python310-arm64) - script: - - PYTHON_VERSION=3.10 ARCH=arm64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.10 - cache: &python310-arm64-cache - before_script: - - PYTHON_VERSION=3.10 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python310-arm64): - stage: test - tags: - - - "arch:arm64" - - image: registry.ddbuild.io/images/mirror/python:3.10 - cache: &python310-arm64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.10,arch:arm64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.10 ARCH=arm64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python310-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python310-arm64) - dependencies: - - build-layer (python310-arm64) - cache: &python310-arm64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.10 ARCH=arm64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python310-arm64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python310-arm64) - - check-layer-size (python310-arm64) - - lint python - - unit-test (python310-arm64) - - integration-test (python310-arm64) - dependencies: - - build-layer (python310-arm64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.10.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-arm64-3.10.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python310-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python310_arm64_prod.env - needs: - - - sign-layer (python310-arm64) - - dependencies: - - - sign-layer (python310-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.10 ARCH=arm64 DOTENV=python310_arm64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python310-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python310_arm64_sandbox.env - needs: - - - build-layer (python310-arm64) - - check-layer-size (python310-arm64) - - lint python - - unit-test (python310-arm64) - - integration-test (python310-arm64) - - dependencies: - - - build-layer (python310-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.10 ARCH=arm64 DOTENV=python310_arm64_sandbox.env ./ci/publish_layers.sh - - - -# TODO(astuyve) - figure out python build cache -.python311-amd64-cache: &python311-amd64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python311-amd64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.11.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.11 ARCH=amd64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python311-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python311-amd64) - dependencies: - - build-layer (python311-amd64) - script: - - PYTHON_VERSION=3.11 ARCH=amd64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.11.6 - cache: &python311-amd64-cache - before_script: - - PYTHON_VERSION=3.11 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python311-amd64): - stage: test - tags: - - - "arch:amd64" - - image: registry.ddbuild.io/images/mirror/python:3.11.6 - cache: &python311-amd64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.11,arch:amd64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.11 ARCH=amd64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python311-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python311-amd64) - dependencies: - - build-layer (python311-amd64) - cache: &python311-amd64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.11 ARCH=amd64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python311-amd64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python311-amd64) - - check-layer-size (python311-amd64) - - lint python - - unit-test (python311-amd64) - - integration-test (python311-amd64) - dependencies: - - build-layer (python311-amd64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.11.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-amd64-3.11.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python311-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python311_amd64_prod.env - needs: - - - sign-layer (python311-amd64) - - dependencies: - - - sign-layer (python311-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.11 ARCH=amd64 DOTENV=python311_amd64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python311-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python311_amd64_sandbox.env - needs: - - - build-layer (python311-amd64) - - check-layer-size (python311-amd64) - - lint python - - unit-test (python311-amd64) - - integration-test (python311-amd64) - - dependencies: - - - build-layer (python311-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.11 ARCH=amd64 DOTENV=python311_amd64_sandbox.env ./ci/publish_layers.sh - - -# TODO(astuyve) - figure out python build cache -.python311-arm64-cache: &python311-arm64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python311-arm64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.11.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.11 ARCH=arm64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python311-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python311-arm64) - dependencies: - - build-layer (python311-arm64) - script: - - PYTHON_VERSION=3.11 ARCH=arm64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.11.6 - cache: &python311-arm64-cache - before_script: - - PYTHON_VERSION=3.11 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python311-arm64): - stage: test - tags: - - - "arch:arm64" - - image: registry.ddbuild.io/images/mirror/python:3.11.6 - cache: &python311-arm64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.11,arch:arm64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.11 ARCH=arm64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python311-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python311-arm64) - dependencies: - - build-layer (python311-arm64) - cache: &python311-arm64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.11 ARCH=arm64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python311-arm64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python311-arm64) - - check-layer-size (python311-arm64) - - lint python - - unit-test (python311-arm64) - - integration-test (python311-arm64) - dependencies: - - build-layer (python311-arm64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.11.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-arm64-3.11.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python311-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python311_arm64_prod.env - needs: - - - sign-layer (python311-arm64) - - dependencies: - - - sign-layer (python311-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.11 ARCH=arm64 DOTENV=python311_arm64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python311-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python311_arm64_sandbox.env - needs: - - - build-layer (python311-arm64) - - check-layer-size (python311-arm64) - - lint python - - unit-test (python311-arm64) - - integration-test (python311-arm64) - - dependencies: - - - build-layer (python311-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.11 ARCH=arm64 DOTENV=python311_arm64_sandbox.env ./ci/publish_layers.sh - - - -# TODO(astuyve) - figure out python build cache -.python312-amd64-cache: &python312-amd64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python312-amd64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.12.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.12 ARCH=amd64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python312-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python312-amd64) - dependencies: - - build-layer (python312-amd64) - script: - - PYTHON_VERSION=3.12 ARCH=amd64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.12.0 - cache: &python312-amd64-cache - before_script: - - PYTHON_VERSION=3.12 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python312-amd64): - stage: test - tags: - - - "arch:amd64" - - image: registry.ddbuild.io/images/mirror/python:3.12.0 - cache: &python312-amd64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.12,arch:amd64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.12 ARCH=amd64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python312-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python312-amd64) - dependencies: - - build-layer (python312-amd64) - cache: &python312-amd64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.12 ARCH=amd64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python312-amd64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python312-amd64) - - check-layer-size (python312-amd64) - - lint python - - unit-test (python312-amd64) - - integration-test (python312-amd64) - dependencies: - - build-layer (python312-amd64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.12.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-amd64-3.12.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python312-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python312_amd64_prod.env - needs: - - - sign-layer (python312-amd64) - - dependencies: - - - sign-layer (python312-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.12 ARCH=amd64 DOTENV=python312_amd64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python312-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python312_amd64_sandbox.env - needs: - - - build-layer (python312-amd64) - - check-layer-size (python312-amd64) - - lint python - - unit-test (python312-amd64) - - integration-test (python312-amd64) - - dependencies: - - - build-layer (python312-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.12 ARCH=amd64 DOTENV=python312_amd64_sandbox.env ./ci/publish_layers.sh - - -# TODO(astuyve) - figure out python build cache -.python312-arm64-cache: &python312-arm64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python312-arm64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.12.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.12 ARCH=arm64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python312-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python312-arm64) - dependencies: - - build-layer (python312-arm64) - script: - - PYTHON_VERSION=3.12 ARCH=arm64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.12.0 - cache: &python312-arm64-cache - before_script: - - PYTHON_VERSION=3.12 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python312-arm64): - stage: test - tags: - - - "arch:arm64" - - image: registry.ddbuild.io/images/mirror/python:3.12.0 - cache: &python312-arm64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.12,arch:arm64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.12 ARCH=arm64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python312-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python312-arm64) - dependencies: - - build-layer (python312-arm64) - cache: &python312-arm64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.12 ARCH=arm64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python312-arm64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python312-arm64) - - check-layer-size (python312-arm64) - - lint python - - unit-test (python312-arm64) - - integration-test (python312-arm64) - dependencies: - - build-layer (python312-arm64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.12.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-arm64-3.12.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python312-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python312_arm64_prod.env - needs: - - - sign-layer (python312-arm64) - - dependencies: - - - sign-layer (python312-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.12 ARCH=arm64 DOTENV=python312_arm64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python312-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python312_arm64_sandbox.env - needs: - - - build-layer (python312-arm64) - - check-layer-size (python312-arm64) - - lint python - - unit-test (python312-arm64) - - integration-test (python312-arm64) - - dependencies: - - - build-layer (python312-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.12 ARCH=arm64 DOTENV=python312_arm64_sandbox.env ./ci/publish_layers.sh - - - -# TODO(astuyve) - figure out python build cache -.python313-amd64-cache: &python313-amd64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python313-amd64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.13.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.13 ARCH=amd64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python313-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python313-amd64) - dependencies: - - build-layer (python313-amd64) - script: - - PYTHON_VERSION=3.13 ARCH=amd64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.13.0 - cache: &python313-amd64-cache - before_script: - - PYTHON_VERSION=3.13 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python313-amd64): - stage: test - tags: - - - "arch:amd64" - - image: registry.ddbuild.io/images/mirror/python:3.13.0 - cache: &python313-amd64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.13,arch:amd64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.13 ARCH=amd64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python313-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python313-amd64) - dependencies: - - build-layer (python313-amd64) - cache: &python313-amd64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.13 ARCH=amd64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python313-amd64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python313-amd64) - - check-layer-size (python313-amd64) - - lint python - - unit-test (python313-amd64) - - integration-test (python313-amd64) - dependencies: - - build-layer (python313-amd64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.13.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-amd64-3.13.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python313-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python313_amd64_prod.env - needs: - - - sign-layer (python313-amd64) - - dependencies: - - - sign-layer (python313-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.13 ARCH=amd64 DOTENV=python313_amd64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python313-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python313_amd64_sandbox.env - needs: - - - build-layer (python313-amd64) - - check-layer-size (python313-amd64) - - lint python - - unit-test (python313-amd64) - - integration-test (python313-amd64) - - dependencies: - - - build-layer (python313-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.13 ARCH=amd64 DOTENV=python313_amd64_sandbox.env ./ci/publish_layers.sh - - -# TODO(astuyve) - figure out python build cache -.python313-arm64-cache: &python313-arm64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python313-arm64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.13.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.13 ARCH=arm64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python313-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python313-arm64) - dependencies: - - build-layer (python313-arm64) - script: - - PYTHON_VERSION=3.13 ARCH=arm64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.13.0 - cache: &python313-arm64-cache - before_script: - - PYTHON_VERSION=3.13 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python313-arm64): - stage: test - tags: - - - "arch:arm64" - - image: registry.ddbuild.io/images/mirror/python:3.13.0 - cache: &python313-arm64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.13,arch:arm64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.13 ARCH=arm64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python313-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python313-arm64) - dependencies: - - build-layer (python313-arm64) - cache: &python313-arm64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.13 ARCH=arm64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python313-arm64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python313-arm64) - - check-layer-size (python313-arm64) - - lint python - - unit-test (python313-arm64) - - integration-test (python313-arm64) - dependencies: - - build-layer (python313-arm64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.13.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-arm64-3.13.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python313-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python313_arm64_prod.env - needs: - - - sign-layer (python313-arm64) - - dependencies: - - - sign-layer (python313-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.13 ARCH=arm64 DOTENV=python313_arm64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python313-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python313_arm64_sandbox.env - needs: - - - build-layer (python313-arm64) - - check-layer-size (python313-arm64) - - lint python - - unit-test (python313-arm64) - - integration-test (python313-arm64) - - dependencies: - - - build-layer (python313-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.13 ARCH=arm64 DOTENV=python313_arm64_sandbox.env ./ci/publish_layers.sh - - - -# TODO(astuyve) - figure out python build cache -.python314-amd64-cache: &python314-amd64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python314-amd64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.14.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.14 ARCH=amd64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python314-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python314-amd64) - dependencies: - - build-layer (python314-amd64) - script: - - PYTHON_VERSION=3.14 ARCH=amd64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.14.0 - cache: &python314-amd64-cache - before_script: - - PYTHON_VERSION=3.14 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python314-amd64): - stage: test - tags: - - - "arch:amd64" - - image: registry.ddbuild.io/images/mirror/python:3.14.0 - cache: &python314-amd64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.14,arch:amd64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.14 ARCH=amd64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python314-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python314-amd64) - dependencies: - - build-layer (python314-amd64) - cache: &python314-amd64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.14 ARCH=amd64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python314-amd64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python314-amd64) - - check-layer-size (python314-amd64) - - lint python - - unit-test (python314-amd64) - - integration-test (python314-amd64) - dependencies: - - build-layer (python314-amd64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.14.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-amd64-3.14.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python314-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python314_amd64_prod.env - needs: - - - sign-layer (python314-amd64) - - dependencies: - - - sign-layer (python314-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.14 ARCH=amd64 DOTENV=python314_amd64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python314-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python314_amd64_sandbox.env - needs: - - - build-layer (python314-amd64) - - check-layer-size (python314-amd64) - - lint python - - unit-test (python314-amd64) - - integration-test (python314-amd64) - - dependencies: - - - build-layer (python314-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.14 ARCH=amd64 DOTENV=python314_amd64_sandbox.env ./ci/publish_layers.sh - - -# TODO(astuyve) - figure out python build cache -.python314-arm64-cache: &python314-arm64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python314-arm64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.14.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.14 ARCH=arm64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python314-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python314-arm64) - dependencies: - - build-layer (python314-arm64) - script: - - PYTHON_VERSION=3.14 ARCH=arm64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.14.0 - cache: &python314-arm64-cache - before_script: - - PYTHON_VERSION=3.14 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python314-arm64): - stage: test - tags: - - - "arch:arm64" - - image: registry.ddbuild.io/images/mirror/python:3.14.0 - cache: &python314-arm64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.14,arch:arm64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.14 ARCH=arm64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python314-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python314-arm64) - dependencies: - - build-layer (python314-arm64) - cache: &python314-arm64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.14 ARCH=arm64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python314-arm64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python314-arm64) - - check-layer-size (python314-arm64) - - lint python - - unit-test (python314-arm64) - - integration-test (python314-arm64) - dependencies: - - build-layer (python314-arm64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.14.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-arm64-3.14.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python314-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python314_arm64_prod.env - needs: - - - sign-layer (python314-arm64) - - dependencies: - - - sign-layer (python314-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.14 ARCH=arm64 DOTENV=python314_arm64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python314-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python314_arm64_sandbox.env - needs: - - - build-layer (python314-arm64) - - check-layer-size (python314-arm64) - - lint python - - unit-test (python314-arm64) - - integration-test (python314-arm64) - - dependencies: - - - build-layer (python314-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.14 ARCH=arm64 DOTENV=python314_arm64_sandbox.env ./ci/publish_layers.sh - - - -# TODO(astuyve) - figure out python build cache -.python315-amd64-cache: &python315-amd64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python315-amd64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.15.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.15 ARCH=amd64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python315-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python315-amd64) - dependencies: - - build-layer (python315-amd64) - script: - - PYTHON_VERSION=3.15 ARCH=amd64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.15.0 - cache: &python315-amd64-cache - before_script: - - PYTHON_VERSION=3.15 ARCH=amd64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python315-amd64): - stage: test - tags: - - - "arch:amd64" - - image: registry.ddbuild.io/images/mirror/python:3.15.0 - cache: &python315-amd64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.15,arch:amd64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.15 ARCH=amd64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python315-amd64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python315-amd64) - dependencies: - - build-layer (python315-amd64) - cache: &python315-amd64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.15 ARCH=amd64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python315-amd64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python315-amd64) - - check-layer-size (python315-amd64) - - lint python - - unit-test (python315-amd64) - - integration-test (python315-amd64) - dependencies: - - build-layer (python315-amd64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-amd64-3.15.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-amd64-3.15.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python315-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python315_amd64_prod.env - needs: - - - sign-layer (python315-amd64) - - dependencies: - - - sign-layer (python315-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.15 ARCH=amd64 DOTENV=python315_amd64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python315-amd64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "amd64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python315_amd64_sandbox.env - needs: - - - build-layer (python315-amd64) - - check-layer-size (python315-amd64) - - lint python - - unit-test (python315-amd64) - - integration-test (python315-amd64) - - dependencies: - - - build-layer (python315-amd64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.15 ARCH=amd64 DOTENV=python315_amd64_sandbox.env ./ci/publish_layers.sh - - -# TODO(astuyve) - figure out python build cache -.python315-arm64-cache: &python315-arm64-cache - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" - paths: - - $CI_PROJECT_DIR/.yarn-cache - policy: pull - -build-layer (python315-arm64): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 day # Unsigned zips expire in 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.15.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - PYTHON_VERSION=3.15 ARCH=arm64 ./scripts/build_layers.sh - timeout: 15m - retry: 2 - -check-layer-size (python315-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python315-arm64) - dependencies: - - build-layer (python315-arm64) - script: - - PYTHON_VERSION=3.15 ARCH=arm64 ./scripts/check_layer_size.sh - -lint python: - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/mirror/python:3.15.0 - cache: &python315-arm64-cache - before_script: - - PYTHON_VERSION=3.15 ARCH=arm64 IS_LINT="1" ./scripts/setup_python_env.sh - script: - - source venv/bin/activate - - ./scripts/check_format.sh - -unit-test (python315-arm64): - stage: test - tags: - - - "arch:arm64" - - image: registry.ddbuild.io/images/mirror/python:3.15.0 - cache: &python315-arm64-cache - variables: - DD_CIVISIBILITY_AGENTLESS_ENABLED: "true" - DD_SERVICE: "datadog-lambda-python" - DD_ENV: "ci" - DD_TAGS: "python_version:3.15,arch:arm64" - before_script: - - GET_SECRETS_API_KEY_ONLY=1 source ./ci/get_secrets.sh - - PYTHON_VERSION=3.15 ARCH=arm64 ./scripts/setup_python_env.sh - script: - - set -e - - source venv/bin/activate - - pytest -vv --ddtrace - retry: 2 - -integration-test (python315-arm64): - stage: test - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - needs: - - build-layer (python315-arm64) - dependencies: - - build-layer (python315-arm64) - cache: &python315-arm64-cache - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - before_script: - - *install-node - - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - - yarn global add serverless@^3.38.0 --prefix /usr/local - - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local - - cd integration_tests && yarn install && cd .. - script: - - RUNTIME_PARAM=3.15 ARCH=arm64 ./scripts/run_integration_tests.sh - retry: 2 - -sign-layer (python315-arm64): - stage: sign - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build-layer (python315-arm64) - - check-layer-size (python315-arm64) - - lint python - - unit-test (python315-arm64) - - integration-test (python315-arm64) - dependencies: - - build-layer (python315-arm64) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_lambda_py-arm64-3.15.zip - before_script: - - apt-get update - - apt-get install -y uuid-runtime - - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - - script: - - LAYER_FILE=datadog_lambda_py-arm64-3.15.zip ./scripts/sign_layers.sh prod - - - - -publish-layer-prod (python315-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"prod" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"prod" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python315_arm64_prod.env - needs: - - - sign-layer (python315-arm64) - - dependencies: - - - sign-layer (python315-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=prod-publish-externalid ROLE_TO_ASSUME=dd-serverless-layer-deployer-role AWS_ACCOUNT=464622532012 source ./ci/get_secrets.sh - script: - - STAGE=prod PYTHON_VERSION=3.15 ARCH=arm64 DOTENV=python315_arm64_prod.env ./ci/publish_layers.sh - - -publish-layer-sandbox (python315-arm64): - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - if: '"sandbox" == "sandbox" && $REGION == "us-west-2" && "arm64" == "amd64"' - when: on_success - - if: '"sandbox" == "sandbox"' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - artifacts: - reports: - dotenv: python315_arm64_sandbox.env - needs: - - - build-layer (python315-arm64) - - check-layer-size (python315-arm64) - - lint python - - unit-test (python315-arm64) - - integration-test (python315-arm64) - - dependencies: - - - build-layer (python315-arm64) - - parallel: - matrix: - - REGION: - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 - - af-south-1 - - ap-east-1 - - ap-east-2 - - ap-south-1 - - ap-south-2 - - ap-southeast-1 - - ap-southeast-2 - - ap-southeast-3 - - ap-southeast-4 - - ap-southeast-5 - - ap-southeast-6 - - ap-southeast-7 - - ap-northeast-1 - - ap-northeast-2 - - ap-northeast-3 - - ca-central-1 - - ca-west-1 - - eu-central-1 - - eu-central-2 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - eu-south-1 - - eu-south-2 - - il-central-1 - - me-central-1 - - mx-central-1 - - sa-east-1 - before_script: - - EXTERNAL_ID_NAME=sandbox-publish-externalid ROLE_TO_ASSUME=sandbox-layer-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh - script: - - STAGE=sandbox PYTHON_VERSION=3.15 ARCH=arm64 DOTENV=python315_arm64_sandbox.env ./ci/publish_layers.sh - -publish-pypi-package: - stage: publish - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10-py3 - before_script: - - ./scripts/setup_python_env.sh - cache: [] - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - sign-layer (python38-amd64) - - sign-layer (python38-arm64) - - sign-layer (python39-amd64) - - sign-layer (python39-arm64) - - sign-layer (python310-amd64) - - sign-layer (python310-arm64) - - sign-layer (python311-amd64) - - sign-layer (python311-arm64) - - sign-layer (python312-amd64) - - sign-layer (python312-arm64) - - sign-layer (python313-amd64) - - sign-layer (python313-arm64) - - sign-layer (python314-amd64) - - sign-layer (python314-arm64) - - sign-layer (python315-amd64) - - sign-layer (python315-arm64) - script: - - ./ci/publish_pypi.sh - -layer bundle: - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - needs: - - build-layer (python38-amd64) - - build-layer (python38-arm64) - - build-layer (python39-amd64) - - build-layer (python39-arm64) - - build-layer (python310-amd64) - - build-layer (python310-arm64) - - build-layer (python311-amd64) - - build-layer (python311-arm64) - - build-layer (python312-amd64) - - build-layer (python312-arm64) - - build-layer (python313-amd64) - - build-layer (python313-arm64) - - build-layer (python314-amd64) - - build-layer (python314-arm64) - - build-layer (python315-amd64) - - build-layer (python315-arm64) - dependencies: - - build-layer (python38-amd64) - - build-layer (python38-arm64) - - build-layer (python39-amd64) - - build-layer (python39-arm64) - - build-layer (python310-amd64) - - build-layer (python310-arm64) - - build-layer (python311-amd64) - - build-layer (python311-arm64) - - build-layer (python312-amd64) - - build-layer (python312-arm64) - - build-layer (python313-amd64) - - build-layer (python313-arm64) - - build-layer (python314-amd64) - - build-layer (python314-arm64) - - build-layer (python315-amd64) - - build-layer (python315-arm64) - artifacts: - expire_in: 1 day - paths: - - datadog_lambda_py-bundle-${CI_JOB_ID}/ - name: datadog_lambda_py-bundle-${CI_JOB_ID} - script: - - rm -rf datadog_lambda_py-bundle-${CI_JOB_ID} - - mkdir -p datadog_lambda_py-bundle-${CI_JOB_ID} - - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-bundle-${CI_JOB_ID} - -signed layer bundle: - stage: sign - image: registry.ddbuild.io/images/docker:20.10-py3 - tags: ["arch:amd64"] - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - needs: - - sign-layer (python38-amd64) - - sign-layer (python38-arm64) - - sign-layer (python39-amd64) - - sign-layer (python39-arm64) - - sign-layer (python310-amd64) - - sign-layer (python310-arm64) - - sign-layer (python311-amd64) - - sign-layer (python311-arm64) - - sign-layer (python312-amd64) - - sign-layer (python312-arm64) - - sign-layer (python313-amd64) - - sign-layer (python313-arm64) - - sign-layer (python314-amd64) - - sign-layer (python314-arm64) - - sign-layer (python315-amd64) - - sign-layer (python315-arm64) - dependencies: - - sign-layer (python38-amd64) - - sign-layer (python38-arm64) - - sign-layer (python39-amd64) - - sign-layer (python39-arm64) - - sign-layer (python310-amd64) - - sign-layer (python310-arm64) - - sign-layer (python311-amd64) - - sign-layer (python311-arm64) - - sign-layer (python312-amd64) - - sign-layer (python312-arm64) - - sign-layer (python313-amd64) - - sign-layer (python313-arm64) - - sign-layer (python314-amd64) - - sign-layer (python314-arm64) - - sign-layer (python315-amd64) - - sign-layer (python315-arm64) - artifacts: - expire_in: 1 day - paths: - - datadog_lambda_py-signed-bundle-${CI_JOB_ID}/ - name: datadog_lambda_py-signed-bundle-${CI_JOB_ID} - script: - - rm -rf datadog_lambda_py-signed-bundle-${CI_JOB_ID} - - mkdir -p datadog_lambda_py-signed-bundle-${CI_JOB_ID} - - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-signed-bundle-${CI_JOB_ID} - -e2e-test: - stage: e2e - trigger: - project: DataDog/serverless-e2e-tests - strategy: depend - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - variables: - LANGUAGES_SUBSET: python - # These env vars are inherited from the dotenv reports of the publish-layer jobs - PYTHON_38_VERSION: $PYTHON_38_VERSION - PYTHON_39_VERSION: $PYTHON_39_VERSION - PYTHON_310_VERSION: $PYTHON_310_VERSION - PYTHON_311_VERSION: $PYTHON_311_VERSION - PYTHON_312_VERSION: $PYTHON_312_VERSION - PYTHON_313_VERSION: $PYTHON_313_VERSION - PYTHON_314_VERSION: $PYTHON_314_VERSION - PYTHON_315_VERSION: $PYTHON_315_VERSION - needs: - - "publish-layer-sandbox (python38-amd64): [us-west-2]" - - "publish-layer-sandbox (python39-amd64): [us-west-2]" - - "publish-layer-sandbox (python310-amd64): [us-west-2]" - - "publish-layer-sandbox (python311-amd64): [us-west-2]" - - "publish-layer-sandbox (python312-amd64): [us-west-2]" - - "publish-layer-sandbox (python313-amd64): [us-west-2]" - - "publish-layer-sandbox (python314-amd64): [us-west-2]" - - "publish-layer-sandbox (python315-amd64): [us-west-2]" - -e2e-test-status: - stage: e2e - image: registry.ddbuild.io/images/docker:20.10-py3 - tags: ["arch:amd64"] - timeout: 3h - rules: - - if: '$SKIP_E2E_TESTS == "true"' - when: never - - when: on_success - script: - - ci/poll_e2e.sh