diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 6950325c945..ab90fcb567b 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -294,6 +294,81 @@ jobs: name: integration-tests-${{ matrix.arch }} - name: Extract Integration Tests Archive run: tar -xzvf integration-tests-${{ matrix.arch }}.tar.gz + - name: Resolve Latest Release Image + # The query fuzz tests compare the build under test against the latest *published* release. + # VERSION cannot answer "what is published" on its own: on a release branch it is bumped to + # the version being prepared (e.g. 1.22.0-rc.0) long before anything pushes that tag, and + # even on the GA tag push the v1.22.0 image is only pushed by `deploy`, which needs this job + # to pass first. The registry is the only source of truth, so ask it which GA tags exist and + # take the highest one that does not exceed VERSION. + # + # The <= bound (rather than simply "the highest published GA tag") only changes the result + # when a newer release already exists on quay than the branch being tested, e.g. preparing + # 1.21.2 on release-1.21 after v1.22.0 has shipped. + # + # Set the CORTEX_LATEST_RELEASE_IMAGE repository variable to bypass the lookup entirely. + if: matrix.tags == 'integration_query_fuzz' + env: + CORTEX_LATEST_RELEASE_IMAGE: ${{ vars.CORTEX_LATEST_RELEASE_IMAGE }} + run: | + if [ -n "${CORTEX_LATEST_RELEASE_IMAGE:-}" ]; then + echo "Using the CORTEX_LATEST_RELEASE_IMAGE override: ${CORTEX_LATEST_RELEASE_IMAGE}" + echo "CORTEX_LATEST_RELEASE_IMAGE=${CORTEX_LATEST_RELEASE_IMAGE}" >> "$GITHUB_ENV" + exit 0 + fi + + version=$(tr -d '[:space:]' < testdata/VERSION) + base=${version%%-*} + if ! printf '%s' "$base" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "ERROR: VERSION '${version}' does not begin with a major.minor.patch version." >&2 + exit 1 + fi + + # List the GA tags published to quay.io. filter_tag_name keeps the release tags and drops + # the per-commit master-* ones; the API pages at 100 tags, so follow has_additional. + tags_file=$(mktemp) + page=1 + while [ "$page" -le 20 ]; do + body="" + for attempt in 1 2 3; do + if body=$(curl -sSf --max-time 30 \ + "https://quay.io/api/v1/repository/cortexproject/cortex/tag/?onlyActiveTags=true&limit=100&page=${page}&filter_tag_name=like:v"); then + break + fi + echo "WARNING: listing quay.io tags page ${page} failed (attempt ${attempt}/3); retrying..." >&2 + body="" + sleep $((attempt * 5)) + done + if [ -z "$body" ]; then + echo "ERROR: unable to list the published tags from quay.io." >&2 + exit 1 + fi + printf '%s' "$body" | jq -r '.tags[].name' >> "$tags_file" + [ "$(printf '%s' "$body" | jq -r '.has_additional')" = "true" ] || break + page=$((page + 1)) + done + + published=$(grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' "$tags_file" | sed 's/^v//' | sort -u -V) + if [ -z "$published" ]; then + echo "ERROR: quay.io reported no published GA release tags." >&2 + exit 1 + fi + + if printf '%s\n' "$published" | grep -qxF "$base"; then + # VERSION itself names a published release, which is the steady state on master. + resolved="$base" + else + # Splice the (unpublished) base into the sorted list and take the entry just below it. + resolved=$(printf '%s\n%s\n' "$published" "$base" | sort -V | + awk -v base="$base" '$0 == base { exit } { previous = $0 } END { print previous }') + fi + if [ -z "$resolved" ]; then + echo "ERROR: quay.io has no published GA release at or below ${base}." >&2 + exit 1 + fi + + echo "VERSION is ${version}; the latest release published at or below ${base} is v${resolved}." + echo "CORTEX_LATEST_RELEASE_IMAGE=quay.io/cortexproject/cortex:v${resolved}" >> "$GITHUB_ENV" - name: Preload Images # We download docker images used by integration tests so that all images are available # locally and the download time doesn't account in the test execution time, which is subject @@ -329,7 +404,7 @@ jobs: retry docker pull quay.io/cortexproject/cortex:v1.21.0 retry docker pull quay.io/cortexproject/cortex:v1.21.1 elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then - retry docker pull quay.io/cortexproject/cortex:v$(cat testdata/VERSION) + retry docker pull "$CORTEX_LATEST_RELEASE_IMAGE" retry docker pull quay.io/prometheus/prometheus:v3.9.1 elif [ "$TEST_TAGS" = "integration_configs_db" ]; then retry docker pull postgres:9.6.16 diff --git a/integration/util.go b/integration/util.go index 0ec7721838c..89e4e248f10 100644 --- a/integration/util.go +++ b/integration/util.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/pkg/errors" @@ -36,20 +37,102 @@ func getCortexProjectDir() string { return os.Getenv("GOPATH") + "/src/github.com/cortexproject/cortex" } -// getLatestReleaseImage returns the Cortex image reference for the latest release, -// derived from the VERSION file at the project root. +// getLatestReleaseImage returns the Cortex image reference for the latest published +// release. +// +// CORTEX_LATEST_RELEASE_IMAGE short-circuits the resolution. CI always sets it: the +// integration workflow asks quay.io which GA tags actually exist and picks the highest one +// that does not exceed VERSION, because the registry is the only source of truth for what +// is published (see .github/workflows/test-build-deploy.yml). +// +// Without it — a local run — fall back to deriving the version from the VERSION file at the +// project root, which needs no network but cannot see what the registry holds. func getLatestReleaseImage() (string, error) { + if image := os.Getenv("CORTEX_LATEST_RELEASE_IMAGE"); image != "" { + return image, nil + } + content, err := os.ReadFile(filepath.Join(getCortexProjectDir(), "VERSION")) if err != nil { return "", errors.Wrap(err, "unable to read VERSION file") } - version := strings.TrimSpace(string(content)) + version, err := latestReleaseVersion(strings.TrimSpace(string(content))) + if err != nil { + return "", err + } + + return fmt.Sprintf("quay.io/cortexproject/cortex:v%s", version), nil +} + +// latestReleaseVersion maps the contents of the VERSION file to a version that has very +// likely been published to the container registries. It is the offline fallback for +// getLatestReleaseImage; CI resolves against the registry instead. +// +// VERSION does not always name a published release. On a release branch it is bumped to +// the version being prepared (e.g. "1.22.0-rc.0") long before the deploy job publishes +// that tag, and the integration job runs before deploy. So a pre-release version resolves +// to the release preceding it, which is always already published by then: +// +// 1.21.1 -> 1.21.1 (VERSION on master is the last GA, whose image exists) +// 1.22.0-rc.0 -> 1.21.0 (the previous minor always shipped a .0) +// 1.22.2-rc.1 -> 1.22.1 (the preceding patch of the same minor) +// +// A GA VERSION is assumed published, which holds everywhere except the GA tag build itself +// — there v1.22.0 is only pushed by deploy, after this runs. That case is why CI consults +// the registry rather than relying on this. +func latestReleaseVersion(version string) (string, error) { if version == "" { return "", errors.New("VERSION file is empty") } - return fmt.Sprintf("quay.io/cortexproject/cortex:v%s", version), nil + // Anything after the first "-" is a pre-release identifier (e.g. "-rc.0"). + base, preRelease, isPreRelease := strings.Cut(version, "-") + if !isPreRelease { + return version, nil + } + + major, minor, patch, err := parseVersion(base) + if err != nil { + return "", errors.Wrapf(err, "unable to resolve the release preceding pre-release version %q", version) + } + + switch { + case patch > 0: + // A patch pre-release: the preceding patch of the same minor is published. + patch-- + case minor > 0: + // A minor pre-release: the previous minor's initial release is published. Using + // .0 rather than its latest patch keeps this derivable from VERSION alone. + minor-- + patch = 0 + default: + // A major pre-release (e.g. "2.0.0-rc.0"). The last release of the previous major + // is not derivable from VERSION, so the maintainer has to say which one it is. + return "", errors.Errorf("cannot resolve the release preceding major pre-release version %q (base %q, pre-release %q):"+ + " set CORTEX_LATEST_RELEASE_IMAGE to the latest published release image", version, base, preRelease) + } + + return fmt.Sprintf("%d.%d.%d", major, minor, patch), nil +} + +func parseVersion(version string) (major, minor, patch int, err error) { + parts := strings.Split(version, ".") + if len(parts) != 3 { + return 0, 0, 0, errors.Errorf("expected a major.minor.patch version, got %q", version) + } + + out := make([]int, len(parts)) + for i, part := range parts { + if out[i], err = strconv.Atoi(part); err != nil { + return 0, 0, 0, errors.Wrapf(err, "invalid version %q", version) + } + if out[i] < 0 { + return 0, 0, 0, errors.Errorf("invalid version %q", version) + } + } + + return out[0], out[1], out[2], nil } func writeFileToSharedDir(s *e2e.Scenario, dst string, content []byte) error { diff --git a/integration/util_test.go b/integration/util_test.go new file mode 100644 index 00000000000..8b84d988562 --- /dev/null +++ b/integration/util_test.go @@ -0,0 +1,94 @@ +//go:build integration + +package integration + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLatestReleaseVersion(t *testing.T) { + tests := map[string]struct { + version string + expected string + expectedErr bool + }{ + "a GA version is already published": { + version: "1.21.1", + expected: "1.21.1", + }, + "a GA version with a zero patch is already published": { + version: "1.21.0", + expected: "1.21.0", + }, + "a minor release candidate falls back to the previous minor": { + version: "1.22.0-rc.0", + expected: "1.21.0", + }, + "a later minor release candidate falls back to the same previous minor": { + version: "1.22.0-rc.3", + expected: "1.21.0", + }, + "a patch release candidate falls back to the preceding patch": { + version: "1.22.1-rc.0", + expected: "1.22.0", + }, + "a later patch release candidate falls back to the preceding patch": { + version: "1.22.3-rc.1", + expected: "1.22.2", + }, + "a major release candidate cannot be resolved": { + version: "2.0.0-rc.0", + expectedErr: true, + }, + "an empty VERSION is rejected": { + version: "", + expectedErr: true, + }, + "a malformed pre-release base is rejected": { + version: "1.22-rc.0", + expectedErr: true, + }, + "a non-numeric pre-release base is rejected": { + version: "1.x.0-rc.0", + expectedErr: true, + }, + } + + for name, testData := range tests { + t.Run(name, func(t *testing.T) { + actual, err := latestReleaseVersion(testData.version) + if testData.expectedErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, testData.expected, actual) + }) + } +} + +func TestGetLatestReleaseImage(t *testing.T) { + // Point getCortexProjectDir() at a scratch checkout so we can exercise the VERSION file + // contents a release branch would actually have. + dir := t.TempDir() + t.Setenv("CORTEX_CHECKOUT_DIR", dir) + require.NoError(t, os.WriteFile(filepath.Join(dir, "VERSION"), []byte("1.22.0-rc.0\n"), 0o600)) + + image, err := getLatestReleaseImage() + require.NoError(t, err) + assert.Equal(t, "quay.io/cortexproject/cortex:v1.21.0", image) +} + +func TestGetLatestReleaseImage_HonorsOverride(t *testing.T) { + t.Setenv("CORTEX_LATEST_RELEASE_IMAGE", "quay.io/cortexproject/cortex:v1.20.1") + + image, err := getLatestReleaseImage() + require.NoError(t, err) + assert.Equal(t, "quay.io/cortexproject/cortex:v1.20.1", image) +}