Repository navigation
280 lines (268 loc) · 12.5 KB
/
Copy pathtest.yml
File metadata and controls
280 lines (268 loc) · 12.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
# This workflow will install Python dependencies, run tests and lint
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Test
on:
push:
branches:
- stable
- dev
pull_request:
branches:
- stable
- dev
merge_group:
workflow_dispatch:
workflow_call:
inputs:
ref:
description: "Git ref to checkout"
required: false
type: string
base_image_version:
description: "Base image version to run the tests in (default: the target branch's pin in release.yml)"
required: false
type: string
permissions:
contents: read
# Keep urllib3-future from hijacking the urllib3 namespace (see pyproject.toml).
env:
URLLIB3_NO_OVERRIDE: "1"
concurrency:
group: test-${{ github.event_name }}-${{ github.event.pull_request.number || inputs.ref || github.ref }}
# Not merge_group: a cancelled run reports no check, hanging the queue entry until timeout.
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
jobs:
changes:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
outputs:
mode: ${{ steps.scope.outputs.mode }}
test_paths: ${{ steps.scope.outputs.test_paths }}
cov_paths: ${{ steps.scope.outputs.cov_paths }}
shards: ${{ steps.pytest.outputs.shards }}
base_image_version: ${{ steps.pytest.outputs.base_image_version }}
steps:
- name: Check out code from GitHub
uses: actions/checkout@v7
with:
ref: ${{ inputs.ref || github.ref }}
# On PRs, run only the tests affected by the changed files: a PR touching a
# single provider runs just that provider's tests. A change to shared code
# (incl. translations under music_assistant/), dependencies or the CI itself
# runs the full suite, as do pushes, merge groups, manual runs and
# reusable-workflow calls.
# PRs that only touch docs or other non-code files run nothing.
- name: Determine test scope
id: scope
env:
GH_TOKEN: ${{ github.token }}
run: |
if [ "${{ github.event_name }}" != "pull_request" ]; then
{
echo "mode=full"
echo "test_paths="
echo "cov_paths="
} >> "$GITHUB_OUTPUT"
exit 0
fi
# Emit both the new and (for renames) the previous path so a file moved
# between providers runs the source provider's tests too.
gh api --paginate \
"repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
--jq '.[] | .filename, (.previous_filename // empty)' | python3 scripts/ci_test_scope.py
# Tests run in the production base image (Dockerfile.base) pinned in release.yml for
# the branch this code ships from: stable's pin for stable, the nightly pin otherwise.
- name: Determine test container and shards
id: pytest
env:
MODE: ${{ steps.scope.outputs.mode }}
INPUT_VERSION: ${{ inputs.base_image_version }}
TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}
run: |
case "$TARGET_BRANCH" in
stable | refs/heads/stable) pin=BASE_IMAGE_VERSION_STABLE ;;
*) pin=BASE_IMAGE_VERSION_NIGHTLY ;;
esac
version="$INPUT_VERSION"
if [ -z "$version" ]; then
version=$(sed -n "s/^ *$pin: *\"\([^\"]*\)\".*/\1/p" .github/workflows/release.yml)
fi
if [ -z "$version" ]; then
echo "::error::Could not read $pin from .github/workflows/release.yml"
exit 1
fi
if [ "$MODE" = "full" ]; then shards="[1,2,3]"; else shards="[1]"; fi
{
echo "base_image_version=$version"
echo "shards=$shards"
} >> "$GITHUB_OUTPUT"
provider-scope:
# Advisory only (never fails the build): emit warning annotations when a PR that touches a
# provider also changes unrelated shared code, suggesting those changes move to their own PR.
if: ${{ github.event_name == 'pull_request' }}
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
steps:
- name: Check out code from GitHub
uses: actions/checkout@v7
with:
ref: ${{ inputs.ref || github.ref }}
- name: Warn on out-of-scope provider changes
env:
GH_TOKEN: ${{ github.token }}
run: |
gh api --paginate \
"repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
--jq '.[] | .filename, (.previous_filename // empty)' | python3 scripts/check_provider_scope.py
lint:
runs-on: ubuntu-latest
steps:
- name: Check out code from GitHub
uses: actions/checkout@v7
with:
ref: ${{ inputs.ref || github.ref }}
- name: Set up Python
uses: actions/setup-python@v7.1.0
with:
python-version-file: ".python-version"
check-latest: true
- name: Install uv
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
enable-cache: true
cache-dependency-glob: |
requirements_all.txt
pyproject.toml
.python-version
cache-suffix: lint
# keeps only the wheels uv built from source; pre-built wheels download faster than they restore
prune-cache: true
# only branch pushes save, PR and merge-queue runs restore their base branch's cache
save-cache: ${{ github.event_name == 'push' }}
- name: Install dependencies
# --index-strategy: allow PyPI packages when also using the PyTorch extra index
# https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes
run: uv pip install --system --index-strategy unsafe-best-match . .[test] -r requirements_all.txt
- name: Lint/test with pre-commit
run: SKIP=no-commit-to-branch pre-commit run --all-files
pytest:
name: pytest (${{ matrix.shard }}/${{ strategy.job-total }})
needs: changes
if: ${{ needs.changes.outputs.mode != 'skip' }}
runs-on: ubuntu-latest
# The production base image of the release this code ships in, so the tests run
# against the same ffmpeg, PyAV and Python version as that release.
container: ghcr.io/music-assistant/base:${{ needs.changes.outputs.base_image_version }}
# Backstop: a hung test process should fail in minutes, not sit at the 6h job cap.
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
shard: ${{ fromJSON(needs.changes.outputs.shards) }}
steps:
# git must be present before checkout, or it degrades to a REST tarball download;
# build-essential is needed to build aiolibdatachannel from its git source;
# curl, gpg and gpg-agent are needed by the codecov action
- name: Install build tools
run: apt-get update && apt-get install -y --no-install-recommends git build-essential curl gpg gpg-agent
- name: Check out code from GitHub
uses: actions/checkout@v7
with:
ref: ${{ inputs.ref || github.ref }}
- name: Install uv
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
enable-cache: true
cache-dependency-glob: |
requirements_all.txt
pyproject.toml
.python-version
cache-suffix: pytest
prune-cache: true
# the shards share one cache, so only the first one saves it
save-cache: ${{ github.event_name == 'push' && matrix.shard == 1 }}
- name: Install dependencies
# --index-strategy: allow PyPI packages when also using the PyTorch extra index
# https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes
run: uv pip install --system --index-strategy unsafe-best-match . .[test] -r requirements_all.txt
# Same as the production image: PyAV comes from the base image's wheel, built
# against its ffmpeg, and must match the version the requirements resolve to.
- name: Install PyAV from the base image
env:
BASE_IMAGE_VERSION: ${{ needs.changes.outputs.base_image_version }}
run: |
required=$(python3 -c "import importlib.metadata; print(importlib.metadata.version('av'))")
wheel=$(ls /usr/local/share/pyav-wheels/av-*.whl)
available=$(basename "$wheel" | cut -d- -f2)
if [ "$required" != "$available" ]; then
echo "::error::PyAV mismatch: requirements need $required, base image $BASE_IMAGE_VERSION ships $available. Rebuild the base image with PyAV $required."
exit 1
fi
uv pip install --system --force-reinstall --no-deps "$wheel"
- name: Pytest
# mode=partial runs only the changed providers' tests with coverage scoped to
# those provider packages; mode=full runs everything with full coverage.
# -n logical starts one worker per vCPU: -n auto counts physical cores (2 on the
# 4 vCPU runner), while most tests wait on I/O rather than the CPU.
# --dist loadfile keeps each test file on one xdist worker: snapshot updates
# never race on an .ambr file and module-scoped fixtures are reused.
# --max-worker-restart=0: with loadfile, replacing a worker that died on a native
# crash leaves the controller blocked on its event queue forever, so the job only
# ends at its 30m cap with no summary. Shutting down instead fails in seconds and
# names the crashing test; a crashed worker invalidates the run either way.
env:
MODE: ${{ needs.changes.outputs.mode }}
TEST_PATHS: ${{ needs.changes.outputs.test_paths }}
COV_PATHS: ${{ needs.changes.outputs.cov_paths }}
SHARD: ${{ matrix.shard }}
SHARDS: ${{ strategy.job-total }}
# aiolibdatachannel's _native.so doesn't link libstdc++ itself; the production
# image resolves it via its jemalloc LD_PRELOAD, here we preload it directly
LD_PRELOAD: libstdc++.so.6
run: |
if [ "$MODE" = "partial" ]; then
targets="$TEST_PATHS"
cov=""
for src in $COV_PATHS; do cov="$cov --cov=$src"; done
else
# each matrix job runs the test files whose path hashes to its shard
targets=$(python3 -c 'import pathlib, sys, zlib; i, n = map(int, sys.argv[1:]); print(*(p for p in sorted(pathlib.Path("tests").rglob("test_*.py")) if zlib.crc32(p.as_posix().encode()) % n == i - 1))' "$SHARD" "$SHARDS")
cov="--cov=music_assistant"
fi
# the container runs as root, which bypasses file permission bits and breaks
# tests exercising permission-denied paths; run pytest as an unprivileged user
useradd -m tester
chown -R tester:tester .
# -p numpy: pre-import numpy before pytest-cov starts coverage. Scoping
# --cov to a submodule (partial mode) makes coverage import the parent
# music_assistant package inside its sys_modules_saved() block, which then
# evicts numpy from sys.modules. Re-importing numpy reinitialises its C
# extension and creates a second _NoValue sentinel, breaking ndarray
# reductions (sum/mean) on py3.14. Loading numpy as an early plugin keeps
# it in the saved snapshot so it is never evicted.
setpriv --reuid=tester --regid=tester --init-groups \
env HOME=/home/tester \
pytest -p numpy -n logical --dist loadfile --max-worker-restart=0 --durations 10 $cov --cov-report=term-missing --cov-report=xml $targets
- name: Upload coverage to Codecov
if: ${{ !github.event.pull_request.head.repo.fork }}
uses: codecov/codecov-action@v7
with:
files: ./coverage.xml
flags: ${{ needs.changes.outputs.mode }}
fail_ci_if_error: false
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
# The branch ruleset requires a check named `test`: it passes when every pytest shard
# passed, or when there was nothing to test.
test:
needs: [changes, pytest]
if: ${{ !cancelled() }}
runs-on: ubuntu-latest
steps:
- name: Check test results
if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
run: exit 1