Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions ci/get_secrets.sh
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,12 @@ _get_dd_api_key

printf "Assuming role...\n"

export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \
$(aws sts assume-role \
--role-arn "arn:aws:iam::$AWS_ACCOUNT:role/$ROLE_TO_ASSUME" \
AWS_CREDENTIALS=$(aws sts assume-role \
--role-arn "arn:aws:iam::$AWS_ACCOUNT:role/$ROLE_TO_ASSUME" \
--role-session-name "ci.datadog-lambda-python-$CI_JOB_ID-$CI_JOB_STAGE" \
--query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
--external-id $EXTERNAL_ID \
--output text))
--external-id "$EXTERNAL_ID" \
--output text)
read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<< "$AWS_CREDENTIALS"
unset AWS_CREDENTIALS
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN
33 changes: 23 additions & 10 deletions ci/input_files/build.yaml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ default:
# Retry when the runner fails to start
- runner_system_failure

# The Noble Python image uses managed system Python and does not include AWS CLI.
.install-python-tools: &install-python-tools
- apt-get update
- apt-get install -y --no-install-recommends python3-venv python-is-python3
- python3 -m venv /opt/ci-python-tools
- /opt/ci-python-tools/bin/pip install awscli
- export PATH="/opt/ci-python-tools/bin:$PATH"

# This is for serverless framework
.install-node: &install-node
- apt-get update
Expand All @@ -38,7 +46,7 @@ default:
build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: build
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1
artifacts:
expire_in: 1 day # Unsigned zips expire in 1 day
paths:
Expand All @@ -53,7 +61,7 @@ build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: test
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1
needs:
- build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
dependencies:
Expand Down Expand Up @@ -99,7 +107,7 @@ unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: test
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
rules:
- if: '$SKIP_E2E_TESTS == "true"'
when: never
Expand All @@ -112,19 +120,20 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
variables:
CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
before_script:
- *install-python-tools
- *install-node
- EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
- yarn global add serverless@^3.38.0 --prefix /usr/local
- yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
- cd integration_tests && yarn install && cd ..
- yarn --cwd tests/integration install
script:
- RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
retry: 2

sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: sign
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
when: manual
Expand All @@ -141,6 +150,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
paths:
- .layers/datadog_lambda_py-{{ $runtime.arch }}-{{ $runtime.python_version }}.zip
before_script:
- *install-python-tools
- apt-get update
- apt-get install -y uuid-runtime
{{ with $environment := (ds "environments").environments.prod }}
Expand All @@ -155,7 +165,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: publish
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
rules:
- if: '$SKIP_E2E_TESTS == "true"'
when: never
Expand Down Expand Up @@ -190,6 +200,8 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
- {{ .code }}
{{- end}}
before_script:
- *install-python-tools
Comment thread
ericdog marked this conversation as resolved.
Resolved
- export SKIP_PIP_INSTALL=true
- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
script:
- STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} DOTENV={{ $dotenv }} ./ci/publish_layers.sh
Expand All @@ -201,8 +213,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
publish-pypi-package:
stage: publish
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
before_script:
- *install-python-tools
- ./scripts/setup_python_env.sh
cache: []
rules:
Expand All @@ -217,7 +230,7 @@ publish-pypi-package:
layer bundle:
stage: build
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1
needs:
{{ range (ds "runtimes").runtimes }}
- build-layer ({{ .name }}-{{ .arch }})
Expand All @@ -238,7 +251,7 @@ layer bundle:

signed layer bundle:
stage: sign
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
tags: ["arch:amd64"]
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
Expand Down Expand Up @@ -286,7 +299,7 @@ e2e-test:

e2e-test-status:
stage: e2e
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3
tags: ["arch:amd64"]
timeout: 3h
rules:
Expand Down
3 changes: 1 addition & 2 deletions scripts/setup_python_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ if [ -z "${IS_LINT:-}" ] || [ "${IS_LINT}" = "0" ] || [ "${IS_LINT}" = "false" ]
spec_ddtrace_dep
fi

pip install virtualenv
virtualenv venv
python3 -m venv venv
source venv/bin/activate
pip install .[dev]
pip install poetry
Expand Down
Loading