Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,26 @@ jobs:
source venv/bin/activate
flake8 datadog_lambda/

python-versions:
runs-on: ubuntu-24.04
outputs:
versions: ${{ steps.load.outputs.versions }}
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Load Python versions
id: load
run: |
echo "versions=$(yq -o=json -I=0 '[.versions[] | .version]' python_versions.yaml)" >> "$GITHUB_OUTPUT"

unit-test:
needs: python-versions
runs-on: ubuntu-24.04
strategy:
max-parallel: 4
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15']
python-version: ${{ fromJSON(needs.python-versions.outputs.versions) }}

steps:
- name: Checkout
Expand Down
18 changes: 17 additions & 1 deletion .github/workflows/build_layer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,30 @@ on:
- "main"

jobs:
python-versions:
runs-on: ubuntu-latest
outputs:
versions: ${{ steps.load.outputs.versions }}
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Load Python versions
id: load
run: |
# system_tests: false versions are skipped. dd-trace-py main no longer
# publishes wheels for them (currently 3.8).
echo "versions=$(yq -o=json -I=0 '[.versions[] | select(.system_tests != false) | .version]' python_versions.yaml)" >> "$GITHUB_OUTPUT"

build:
needs: python-versions
runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}

strategy:
fail-fast: false
matrix:
arch: [arm64, amd64]
python_version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
python_version: ${{ fromJSON(needs.python-versions.outputs.versions) }}

steps:
- name: Checkout
Expand Down
4 changes: 2 additions & 2 deletions ci/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ outputFiles:
- ci/build-pipeline.yaml

datasources:
runtimes:
url: ci/datasources/runtimes.yaml
python_versions:
url: python_versions.yaml
regions:
url: ci/datasources/regions.yaml
environments:
Expand Down
65 changes: 0 additions & 65 deletions ci/datasources/runtimes.yaml

This file was deleted.

20 changes: 12 additions & 8 deletions ci/input_files/build.yaml.tpl
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{{- $e2e_region := "us-west-2" -}}
{{- $runtimes := coll.Slice -}}
{{- range $py := (ds "python_versions").versions }}{{ range $arch := coll.Slice "amd64" "arm64" }}
{{- $runtimes = $runtimes | coll.Append (dict "name" (printf "python%s" ($py.version | strings.ReplaceAll "." "")) "python_version" $py.version "arch" $arch "image" $py.image) }}
{{- end }}{{ end -}}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is dope


stages:
- build
Expand Down Expand Up @@ -26,7 +30,7 @@ default:
- apt-get install nodejs -y
- npm install --global yarn

{{ range $runtime := (ds "runtimes").runtimes }}
{{ range $runtime := $runtimes }}

# TODO(astuyve) - figure out python build cache
.{{ $runtime.name }}-{{ $runtime.arch }}-cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
Expand Down Expand Up @@ -208,7 +212,7 @@ publish-pypi-package:
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
when: manual
needs: {{ range $runtime := (ds "runtimes").runtimes }}
needs: {{ range $runtime := $runtimes }}
- sign-layer ({{ $runtime.name }}-{{ $runtime.arch}})
{{- end }}
script:
Expand All @@ -219,11 +223,11 @@ layer bundle:
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
needs:
{{ range (ds "runtimes").runtimes }}
{{ range $runtimes }}
- build-layer ({{ .name }}-{{ .arch }})
{{ end }}
dependencies:
{{ range (ds "runtimes").runtimes }}
{{ range $runtimes }}
- build-layer ({{ .name }}-{{ .arch }})
{{ end }}
artifacts:
Expand All @@ -243,11 +247,11 @@ signed layer bundle:
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
needs:
{{ range (ds "runtimes").runtimes }}
{{ range $runtimes }}
- sign-layer ({{ .name }}-{{ .arch }})
{{ end }}
dependencies:
{{ range (ds "runtimes").runtimes }}
{{ range $runtimes }}
- sign-layer ({{ .name }}-{{ .arch }})
{{ end }}
artifacts:
Expand All @@ -272,13 +276,13 @@ e2e-test:
variables:
LANGUAGES_SUBSET: python
# These env vars are inherited from the dotenv reports of the publish-layer jobs
{{- range (ds "runtimes").runtimes }}
{{- range $runtimes }}
{{- if eq .arch "amd64" }}
{{- $version := print (.name | strings.Trim "python") }}
PYTHON_{{ $version }}_VERSION: $PYTHON_{{ $version }}_VERSION
{{- end }}
{{- end }}
needs: {{ range (ds "runtimes").runtimes }}
needs: {{ range $runtimes }}
{{- if eq .arch "amd64" }}
- "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
{{- end }}
Expand Down
76 changes: 4 additions & 72 deletions ci/publish_layers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,78 +10,10 @@
set -e

# Available runtimes: https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
AWS_CLI_PYTHON_VERSIONS=(
"python3.8"
"python3.8"
"python3.9"
"python3.9"
"python3.10"
"python3.10"
"python3.11"
"python3.11"
"python3.12"
"python3.12"
"python3.13"
"python3.13"
"python3.14"
"python3.14"
"python3.15"
"python3.15"
)
PYTHON_VERSIONS=(
"3.8-amd64"
"3.8-arm64"
"3.9-amd64"
"3.9-arm64"
"3.10-amd64"
"3.10-arm64"
"3.11-amd64"
"3.11-arm64"
"3.12-amd64"
"3.12-arm64"
"3.13-amd64"
"3.13-arm64"
"3.14-amd64"
"3.14-arm64"
"3.15-amd64"
"3.15-arm64"
)
LAYER_PATHS=(
".layers/datadog_lambda_py-amd64-3.8.zip"
".layers/datadog_lambda_py-arm64-3.8.zip"
".layers/datadog_lambda_py-amd64-3.9.zip"
".layers/datadog_lambda_py-arm64-3.9.zip"
".layers/datadog_lambda_py-amd64-3.10.zip"
".layers/datadog_lambda_py-arm64-3.10.zip"
".layers/datadog_lambda_py-amd64-3.11.zip"
".layers/datadog_lambda_py-arm64-3.11.zip"
".layers/datadog_lambda_py-amd64-3.12.zip"
".layers/datadog_lambda_py-arm64-3.12.zip"
".layers/datadog_lambda_py-amd64-3.13.zip"
".layers/datadog_lambda_py-arm64-3.13.zip"
".layers/datadog_lambda_py-amd64-3.14.zip"
".layers/datadog_lambda_py-arm64-3.14.zip"
".layers/datadog_lambda_py-amd64-3.15.zip"
".layers/datadog_lambda_py-arm64-3.15.zip"
)
LAYERS=(
"Datadog-Python38"
"Datadog-Python38-ARM"
"Datadog-Python39"
"Datadog-Python39-ARM"
"Datadog-Python310"
"Datadog-Python310-ARM"
"Datadog-Python311"
"Datadog-Python311-ARM"
"Datadog-Python312"
"Datadog-Python312-ARM"
"Datadog-Python313"
"Datadog-Python313-ARM"
"Datadog-Python314"
"Datadog-Python314-ARM"
"Datadog-Python315"
"Datadog-Python315-ARM"
)
source "$(dirname "${BASH_SOURCE[0]}")/../scripts/python_versions.sh"
AWS_CLI_PYTHON_VERSIONS=("${LAYER_RUNTIMES[@]}")
LAYERS=("${LAYER_NAMES[@]}")
PYTHON_VERSIONS=("${LAYER_KEYS[@]}")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the most beautiful thing I've seen all day.

STAGES=('prod', 'sandbox', 'staging', 'gov-staging', 'gov-prod')

printf "Starting script...\n\n"
Expand Down
31 changes: 31 additions & 0 deletions python_versions.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# Supported Python versions for the Lambda library.
#
# Layer builds, publish scripts, tests, and CI read this file. Add a version
# here and those pick it up. The README and the pyproject.toml classifiers are
# the lists still maintained by hand.
#
# version: CPython minor version.
# image: GitLab CI image tag (registry.ddbuild.io/images/mirror/python).
# preview: AWS has not published a SAM build image yet. Layer builds use
# public.ecr.aws/lambda/python:<version>-preview. Default false.
# system_tests: Set false to skip the GitHub system-tests layer build. That
# job installs a dd-trace-py main-branch wheel, which no longer
# publishes 3.8 wheels. Default true.
versions:
- version: "3.8"
image: "3.8"
system_tests: false
- version: "3.9"
image: "3.9"
- version: "3.10"
image: "3.10"
- version: "3.11"
image: "3.11.6"
- version: "3.12"
image: "3.12.0"
- version: "3.13"
image: "3.13.0"
- version: "3.14"
image: "3.14.0"
- version: "3.15"
image: "3.15.0"
38 changes: 2 additions & 36 deletions scripts/add_new_region.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,42 +12,8 @@ set -e

OLD_REGION='us-east-1'

LAYER_NAMES=(
"Datadog-Python38"
"Datadog-Python38-ARM"
"Datadog-Python39"
"Datadog-Python39-ARM"
"Datadog-Python310"
"Datadog-Python310-ARM"
"Datadog-Python311"
"Datadog-Python311-ARM"
"Datadog-Python312"
"Datadog-Python312-ARM"
"Datadog-Python313"
"Datadog-Python313-ARM"
"Datadog-Python314"
"Datadog-Python314-ARM"
"Datadog-Python315"
"Datadog-Python315-ARM"
)
PYTHON_VERSIONS_FOR_AWS_CLI=(
"python3.8"
"python3.8"
"python3.9"
"python3.9"
"python3.10"
"python3.10"
"python3.11"
"python3.11"
"python3.12"
"python3.12"
"python3.13"
"python3.13"
"python3.14"
"python3.14"
"python3.15"
"python3.15"
)
source "$(dirname "${BASH_SOURCE[0]}")/python_versions.sh"
PYTHON_VERSIONS_FOR_AWS_CLI=("${LAYER_RUNTIMES[@]}")
NEW_REGION=$1

publish_layer() {
Expand Down
Loading
Loading