From 9506b4b526b20418cdb15bc3139301960abb1196 Mon Sep 17 00:00:00 2001 From: Peter Abrahamsen Date: Tue, 29 Sep 2026 15:54:33 -0700 Subject: [PATCH] ci: build, smoke-test and deploy from GitHub Actions Every pull request builds the image and renders a synthetic MPEG-TS clip inside it as Lambda would run it: unprivileged, root-owned empty /tmp, the CPU and memory of the 1024 MB tier. Merging to main then runs `sam build && sam deploy` under the production environment, assuming the spectrogram-renderer-deploy role through GitHub's OIDC provider, so a deploy is a merge rather than a laptop. SAM's hello-world tests and event go; the sample event now names a real segment and writes nothing. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 52 +++++++++++++++++++++++++++ .gitignore | 4 ++- README.md | 34 ++++++++++-------- __init__.py | 0 events/event.json | 63 -------------------------------- events/spectrogram_job.json | 12 +++---- tests/__init__.py | 0 tests/smoke.py | 48 +++++++++++++++++++++++++ tests/unit/__init__.py | 0 tests/unit/test_handler.py | 72 ------------------------------------- 10 files changed, 128 insertions(+), 157 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 __init__.py delete mode 100644 events/event.json delete mode 100644 tests/__init__.py create mode 100644 tests/smoke.py delete mode 100644 tests/unit/__init__.py delete mode 100644 tests/unit/test_handler.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..60af4c9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: Build image and render a clip + runs-on: ubuntu-latest + steps: + - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Validate the SAM template + run: sam validate --lint + - name: Build the image + run: docker build -t spectrogram-renderer core + - name: Render a clip the way Lambda runs it + # Unprivileged, root-owned /tmp, and the CPU and memory of the 1024 MB tier. + run: | + docker run --rm --user 1000:1000 --tmpfs /tmp:uid=0,gid=0,mode=1777 \ + --cpus 0.58 --memory 1024m --entrypoint python \ + -v "$PWD/tests/smoke.py:/var/task/smoke.py" spectrogram-renderer smoke.py + + deploy: + name: Deploy to AWS + needs: test + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + environment: production + concurrency: deploy + permissions: + id-token: write + contents: read + steps: + - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: us-west-2 + - run: sam build + - run: sam deploy --no-confirm-changeset --no-fail-on-empty-changeset diff --git a/.gitignore b/.gitignore index 4808264..c576f65 100644 --- a/.gitignore +++ b/.gitignore @@ -241,4 +241,6 @@ $RECYCLE.BIN/ */build/* -# End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode \ No newline at end of file +# End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode +# SAM build output +.aws-sam/ diff --git a/README.md b/README.md index 165b865..00be66e 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# audio_viz +# spectrogram-renderer -A Lambda function that renders one spectrogram PNG from one audio segment. Orcasite invokes it directly by name from [`Orcasite.Radio.AwsClient`](../lib/orcasite/radio/aws_client.ex), once per `AudioImage`, passing the S3 location of the segment and where to put the image. It has no other entry point. +A Lambda function that renders one spectrogram PNG from one hydrophone audio segment. [Orcasite](https://github.com/orcasound/orcasite) invokes it by name from [`Orcasite.Radio.AwsClient`](https://github.com/orcasound/orcasite/blob/main/server/lib/orcasite/radio/aws_client.ex), once per `AudioImage`, passing the S3 location of the segment and where to put the image. It has no other entry point. - [`core/app.py`](core/app.py): the handler. Downloads the segment, decodes it with ffmpeg, renders with [`core/spectrogram_generator.py`](core/spectrogram_generator.py), uploads the PNG. - [`core/Dockerfile`](core/Dockerfile): the image, built on the AWS Lambda Python base with a static ffmpeg. -- [`template.yaml`](template.yaml): the function's memory, timeout and IAM policy, as a SAM template. Stack name and region are in [`samconfig.toml`](samconfig.toml). +- [`template.yaml`](template.yaml): the function's memory, timeout and IAM policy, as a SAM template. Stack name (`audio-viz`, kept from when this lived in orcasite) and region are in [`samconfig.toml`](samconfig.toml). ## Cold starts @@ -12,28 +12,32 @@ A new Lambda container starts with an empty `/tmp`, where numba, librosa and mat To check a deployed function, find `platform.report` records in its CloudWatch log (the template sets `LogFormat: JSON`). Cold invocations carry `initDurationMs`; their `durationMs + initDurationMs` should be within a couple of seconds of a warm invocation's `durationMs`. Lambda's `Duration` metric excludes init time, so it alone understates a cold start. -## Build, test and deploy +## Testing -Requires the [SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html), Docker, and an AWS profile with access to the Orcasound account. Merging to `main` does not deploy; someone runs this: +[`tests/smoke.py`](tests/smoke.py) renders a synthetic MPEG-TS clip inside the built image, run as an unprivileged user with a root-owned empty `/tmp` and the CPU and memory of the deployed tier, which is how Lambda runs it. It fails if the first render is slow enough to suggest the caches were not restored. [CI](.github/workflows/ci.yml) runs it on every pull request; locally: ```bash -cd server/audio_viz -sam build -AWS_PROFILE=orcasound sam deploy +docker build -t spectrogram-renderer core +docker run --rm --user 1000:1000 --tmpfs /tmp:uid=0,gid=0,mode=1777 --cpus 0.58 --memory 1024m \ + --entrypoint python -v "$PWD/tests/smoke.py:/var/task/smoke.py" spectrogram-renderer smoke.py ``` -`sam build` builds the image and `sam deploy` pushes it to ECR and updates the CloudFormation stack, showing the change set first. +That rehearses the application, not the sandbox: Lambda's Runtime Interface Emulator in the base image reproduces the invoke API, not the filesystem rules. Anything that touches `/tmp`'s own metadata or relies on image contents under `/tmp` will pass locally on a plain tmpfs and fail deployed, hence the `--user` and `--tmpfs` flags above. + +## Deploying + +Merging to `main` deploys: the [CI workflow](.github/workflows/ci.yml) runs the smoke test, then `sam build && sam deploy` under the `production` environment, assuming the `spectrogram-renderer-deploy` IAM role through GitHub's OIDC provider. That role can change only this stack, its ECR repository and its function role. -To run the built function locally against a real event: +To deploy from a machine instead, with an AWS profile for the Orcasound account and the [SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html): ```bash -AWS_PROFILE=orcasound sam local invoke AudioVizFunction --event events/spectrogram_job.json +sam build +AWS_PROFILE=orcasound sam deploy ``` -To tail its logs in AWS: +To invoke the deployed function on a real segment without writing anything (`image_key` null skips the upload): ```bash -sam logs -n AudioVizFunction --stack-name audio-viz --tail +aws lambda invoke --function-name "$(aws lambda list-functions --query "Functions[?contains(FunctionName,'AudioVizFunction')].FunctionName | [0]" --output text)" \ + --cli-binary-format raw-in-base64-out --payload "$(cat events/spectrogram_job.json)" /dev/stdout ``` - -`tests/` holds SAM's generated unit tests; they do not yet exercise the handler. diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/events/event.json b/events/event.json deleted file mode 100644 index a429ac5..0000000 --- a/events/event.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "body": "{\"message\": \"hello world\"}", - "resource": "/hello", - "path": "/hello", - "httpMethod": "GET", - "isBase64Encoded": false, - "queryStringParameters": { - "foo": "bar" - }, - "pathParameters": { - "proxy": "/path/to/resource" - }, - "stageVariables": { - "baz": "qux" - }, - "headers": { - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Accept-Encoding": "gzip, deflate, sdch", - "Accept-Language": "en-US,en;q=0.8", - "Cache-Control": "max-age=0", - "CloudFront-Forwarded-Proto": "https", - "CloudFront-Is-Desktop-Viewer": "true", - "CloudFront-Is-Mobile-Viewer": "false", - "CloudFront-Is-SmartTV-Viewer": "false", - "CloudFront-Is-Tablet-Viewer": "false", - "CloudFront-Viewer-Country": "US", - "Host": "1234567890.execute-api.us-east-1.amazonaws.com", - "Upgrade-Insecure-Requests": "1", - "User-Agent": "Custom User Agent String", - "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)", - "X-Amz-Cf-Id": "cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==", - "X-Forwarded-For": "127.0.0.1, 127.0.0.2", - "X-Forwarded-Port": "443", - "X-Forwarded-Proto": "https" - }, - "requestContext": { - "accountId": "123456789012", - "resourceId": "123456", - "stage": "prod", - "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", - "requestTime": "09/Apr/2015:12:34:56 +0000", - "requestTimeEpoch": 1428582896000, - "identity": { - "cognitoIdentityPoolId": null, - "accountId": null, - "cognitoIdentityId": null, - "caller": null, - "accessKey": null, - "sourceIp": "127.0.0.1", - "cognitoAuthenticationType": null, - "cognitoAuthenticationProvider": null, - "userArn": null, - "userAgent": "Custom User Agent String", - "user": null - }, - "path": "/prod/hello", - "resourcePath": "/hello", - "httpMethod": "POST", - "apiId": "1234567890", - "protocol": "HTTP/1.1" - } - } - \ No newline at end of file diff --git a/events/spectrogram_job.json b/events/spectrogram_job.json index d061e82..4363cf7 100644 --- a/events/spectrogram_job.json +++ b/events/spectrogram_job.json @@ -1,7 +1,7 @@ { - "id": "test_image", - "audio_bucket": "streaming-orcasound-net", - "audio_key": "rpi_sunset_bay/hls/1654758019/live004.ts", - "image_bucket": "dev-archive-orcasound-net", - "image_key": "spectrograms/live004_spect.png" -} \ No newline at end of file + "id": "spectrogram_job_example", + "audio_bucket": "audio-orcasound-net", + "audio_key": "rpi_orcasound_lab/hls/1790665213/live5238.ts", + "image_bucket": null, + "image_key": null +} diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/smoke.py b/tests/smoke.py new file mode 100644 index 0000000..d6cc678 --- /dev/null +++ b/tests/smoke.py @@ -0,0 +1,48 @@ +"""Renders one clip in a fresh container and reports how long each step took. + +Run inside the built image, the way CI does, to rehearse a Lambda cold start: +an unprivileged user, a root-owned empty /tmp, and the CPU and memory of the +deployed tier. The clip is AAC in MPEG-TS like a real segment. The render +skips S3 both ways, as in warm.py. + +Fails if the first render is slow enough to suggest the caches were not +restored, or if the two renders disagree about the image. +""" + +import subprocess +import sys +import time + +CLIP = "smoke.ts" +JOB = { + "id": CLIP, + "audio_bucket": "none", + "audio_key": "none", + "sample_rate": None, + "image_key": None, + "image_bucket": None, +} +# A cold render without the caches took over 30 s at this CPU share. +SLOW_FIRST_RENDER = 15.0 + +subprocess.run( + ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", + "anoisesrc=d=10:c=pink:r=48000", "-ac", "1", "-c:a", "aac", "-f", "mpegts", f"/tmp/{CLIP}"], + check=True, +) + +t0 = time.time() +import app # noqa: E402 (timed on purpose: this is where the caches are restored) + +t1 = time.time() +first = app.make_spectrogram(JOB) +t2 = time.time() +second = app.make_spectrogram(JOB) +t3 = time.time() + +print(f"import {t1 - t0:.1f}s | first render {t2 - t1:.1f}s | second render {t3 - t2:.1f}s | image {first['image_size']} bytes") + +if first["image_size"] != second["image_size"]: + sys.exit(f"renders differ: {first['image_size']} vs {second['image_size']} bytes") +if t2 - t1 > SLOW_FIRST_RENDER: + sys.exit(f"first render took {t2 - t1:.1f}s; were the caches restored?") diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/unit/test_handler.py b/tests/unit/test_handler.py deleted file mode 100644 index 28a4d15..0000000 --- a/tests/unit/test_handler.py +++ /dev/null @@ -1,72 +0,0 @@ -import json - -import pytest - -from core import app - - -@pytest.fixture() -def apigw_event(): - """ Generates API GW Event""" - - return { - "body": '{ "test": "body"}', - "resource": "/{proxy+}", - "requestContext": { - "resourceId": "123456", - "apiId": "1234567890", - "resourcePath": "/{proxy+}", - "httpMethod": "POST", - "requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef", - "accountId": "123456789012", - "identity": { - "apiKey": "", - "userArn": "", - "cognitoAuthenticationType": "", - "caller": "", - "userAgent": "Custom User Agent String", - "user": "", - "cognitoIdentityPoolId": "", - "cognitoIdentityId": "", - "cognitoAuthenticationProvider": "", - "sourceIp": "127.0.0.1", - "accountId": "", - }, - "stage": "prod", - }, - "queryStringParameters": {"foo": "bar"}, - "headers": { - "Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)", - "Accept-Language": "en-US,en;q=0.8", - "CloudFront-Is-Desktop-Viewer": "true", - "CloudFront-Is-SmartTV-Viewer": "false", - "CloudFront-Is-Mobile-Viewer": "false", - "X-Forwarded-For": "127.0.0.1, 127.0.0.2", - "CloudFront-Viewer-Country": "US", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Upgrade-Insecure-Requests": "1", - "X-Forwarded-Port": "443", - "Host": "1234567890.execute-api.us-east-1.amazonaws.com", - "X-Forwarded-Proto": "https", - "X-Amz-Cf-Id": "aaaaaaaaaae3VYQb9jd-nvCd-de396Uhbp027Y2JvkCPNLmGJHqlaA==", - "CloudFront-Is-Tablet-Viewer": "false", - "Cache-Control": "max-age=0", - "User-Agent": "Custom User Agent String", - "CloudFront-Forwarded-Proto": "https", - "Accept-Encoding": "gzip, deflate, sdch", - }, - "pathParameters": {"proxy": "/examplepath"}, - "httpMethod": "POST", - "stageVariables": {"baz": "qux"}, - "path": "/examplepath", - } - - -def test_lambda_handler(apigw_event, mocker): - - ret = app.lambda_handler(apigw_event, "") - data = json.loads(ret["body"]) - - assert ret["statusCode"] == 200 - assert "message" in ret["body"] - assert data["message"] == "hello world"