Resolve the latest release image to a published version

.github/workflows/test-build-deploy.yml

@@ -294,6 +294,81 @@ jobs:
           name: integration-tests-${{ matrix.arch }}
       - name: Extract Integration Tests Archive
         run: tar -xzvf integration-tests-${{ matrix.arch }}.tar.gz
+      - name: Resolve Latest Release Image
+        # The query fuzz tests compare the build under test against the latest *published* release.
+        # VERSION cannot answer "what is published" on its own: on a release branch it is bumped to
+        # the version being prepared (e.g. 1.22.0-rc.0) long before anything pushes that tag, and
+        # even on the GA tag push the v1.22.0 image is only pushed by `deploy`, which needs this job
+        # to pass first. The registry is the only source of truth, so ask it which GA tags exist and
+        # take the highest one that does not exceed VERSION.
+        #
+        # The <= bound (rather than simply "the highest published GA tag") only changes the result
+        # when a newer release already exists on quay than the branch being tested, e.g. preparing
+        # 1.21.2 on release-1.21 after v1.22.0 has shipped.
+        #
+        # Set the CORTEX_LATEST_RELEASE_IMAGE repository variable to bypass the lookup entirely.
+        if: matrix.tags == 'integration_query_fuzz'
+        env:
+          CORTEX_LATEST_RELEASE_IMAGE: ${{ vars.CORTEX_LATEST_RELEASE_IMAGE }}
+        run: |
+          if [ -n "${CORTEX_LATEST_RELEASE_IMAGE:-}" ]; then
+            echo "Using the CORTEX_LATEST_RELEASE_IMAGE override: ${CORTEX_LATEST_RELEASE_IMAGE}"
+            echo "CORTEX_LATEST_RELEASE_IMAGE=${CORTEX_LATEST_RELEASE_IMAGE}" >> "$GITHUB_ENV"
+            exit 0
+          fi
+
+          version=$(tr -d '[:space:]' < testdata/VERSION)
+          base=${version%%-*}
+          if ! printf '%s' "$base" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
+            echo "ERROR: VERSION '${version}' does not begin with a major.minor.patch version." >&2
+            exit 1
+          fi
+
+          # List the GA tags published to quay.io. filter_tag_name keeps the release tags and drops
+          # the per-commit master-* ones; the API pages at 100 tags, so follow has_additional.
+          tags_file=$(mktemp)
+          page=1
+          while [ "$page" -le 20 ]; do
+            body=""
+            for attempt in 1 2 3; do
+              if body=$(curl -sSf --max-time 30 \
+                "https://quay.io/api/v1/repository/cortexproject/cortex/tag/?onlyActiveTags=true&limit=100&page=${page}&filter_tag_name=like:v"); then
+                break
+              fi
+              echo "WARNING: listing quay.io tags page ${page} failed (attempt ${attempt}/3); retrying..." >&2
+              body=""
+              sleep $((attempt * 5))
+            done
+            if [ -z "$body" ]; then
+              echo "ERROR: unable to list the published tags from quay.io." >&2
+              exit 1
+            fi
+            printf '%s' "$body" | jq -r '.tags[].name' >> "$tags_file"
+            [ "$(printf '%s' "$body" | jq -r '.has_additional')" = "true" ] || break
+            page=$((page + 1))
+          done
+
+          published=$(grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' "$tags_file" | sed 's/^v//' | sort -u -V)
+          if [ -z "$published" ]; then
+            echo "ERROR: quay.io reported no published GA release tags." >&2
+            exit 1
+          fi
+
+          if printf '%s\n' "$published" | grep -qxF "$base"; then
+            # VERSION itself names a published release, which is the steady state on master.
+            resolved="$base"
+          else
+            # Splice the (unpublished) base into the sorted list and take the entry just below it.
+            resolved=$(printf '%s\n%s\n' "$published" "$base" | sort -V |
+              awk -v base="$base" '$0 == base { exit } { previous = $0 } END { print previous }')
+          fi
+          if [ -z "$resolved" ]; then
+            echo "ERROR: quay.io has no published GA release at or below ${base}." >&2
+            exit 1
+          fi
+
+          echo "VERSION is ${version}; the latest release published at or below ${base} is v${resolved}."
+          echo "CORTEX_LATEST_RELEASE_IMAGE=quay.io/cortexproject/cortex:v${resolved}" >> "$GITHUB_ENV"
       - name: Preload Images
         # We download docker images used by integration tests so that all images are available
         # locally and the download time doesn't account in the test execution time, which is subject
@@ -329,7 +404,7 @@ jobs:
             retry docker pull quay.io/cortexproject/cortex:v1.21.0
             retry docker pull quay.io/cortexproject/cortex:v1.21.1
           elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then
-            retry docker pull quay.io/cortexproject/cortex:v$(cat testdata/VERSION)
+            retry docker pull "$CORTEX_LATEST_RELEASE_IMAGE"
             retry docker pull quay.io/prometheus/prometheus:v3.9.1
           elif [ "$TEST_TAGS" = "integration_configs_db" ]; then
             retry docker pull postgres:9.6.16

integration/util.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -36,20 +37,102 @@ func getCortexProjectDir() string {
 	return os.Getenv("GOPATH") + "/src/github.com/cortexproject/cortex"
 }
 
-// getLatestReleaseImage returns the Cortex image reference for the latest release,
-// derived from the VERSION file at the project root.
+// getLatestReleaseImage returns the Cortex image reference for the latest published
+// release.
+//
+// CORTEX_LATEST_RELEASE_IMAGE short-circuits the resolution. CI always sets it: the
+// integration workflow asks quay.io which GA tags actually exist and picks the highest one
+// that does not exceed VERSION, because the registry is the only source of truth for what
+// is published (see .github/workflows/test-build-deploy.yml).
+//
+// Without it — a local run — fall back to deriving the version from the VERSION file at the
+// project root, which needs no network but cannot see what the registry holds.
 func getLatestReleaseImage() (string, error) {
+	if image := os.Getenv("CORTEX_LATEST_RELEASE_IMAGE"); image != "" {
+		return image, nil
+	}
+
 	content, err := os.ReadFile(filepath.Join(getCortexProjectDir(), "VERSION"))
 	if err != nil {
 		return "", errors.Wrap(err, "unable to read VERSION file")
 	}
 
-	version := strings.TrimSpace(string(content))
+	version, err := latestReleaseVersion(strings.TrimSpace(string(content)))
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("quay.io/cortexproject/cortex:v%s", version), nil
+}
+
+// latestReleaseVersion maps the contents of the VERSION file to a version that has very
+// likely been published to the container registries. It is the offline fallback for
+// getLatestReleaseImage; CI resolves against the registry instead.
+//
+// VERSION does not always name a published release. On a release branch it is bumped to
+// the version being prepared (e.g. "1.22.0-rc.0") long before the deploy job publishes
+// that tag, and the integration job runs before deploy. So a pre-release version resolves
+// to the release preceding it, which is always already published by then:
+//
+//	1.21.1      -> 1.21.1  (VERSION on master is the last GA, whose image exists)
+//	1.22.0-rc.0 -> 1.21.0  (the previous minor always shipped a .0)
+//	1.22.2-rc.1 -> 1.22.1  (the preceding patch of the same minor)
+//
+// A GA VERSION is assumed published, which holds everywhere except the GA tag build itself
+// — there v1.22.0 is only pushed by deploy, after this runs. That case is why CI consults
+// the registry rather than relying on this.
+func latestReleaseVersion(version string) (string, error) {
 	if version == "" {
 		return "", errors.New("VERSION file is empty")
 	}
 
-	return fmt.Sprintf("quay.io/cortexproject/cortex:v%s", version), nil
+	// Anything after the first "-" is a pre-release identifier (e.g. "-rc.0").
+	base, preRelease, isPreRelease := strings.Cut(version, "-")
+	if !isPreRelease {
+		return version, nil
+	}
+
+	major, minor, patch, err := parseVersion(base)
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to resolve the release preceding pre-release version %q", version)
+	}
+
+	switch {
+	case patch > 0:
+		// A patch pre-release: the preceding patch of the same minor is published.
+		patch--
+	case minor > 0:
+		// A minor pre-release: the previous minor's initial release is published. Using
+		// .0 rather than its latest patch keeps this derivable from VERSION alone.
+		minor--
+		patch = 0
+	default:
+		// A major pre-release (e.g. "2.0.0-rc.0"). The last release of the previous major
+		// is not derivable from VERSION, so the maintainer has to say which one it is.
+		return "", errors.Errorf("cannot resolve the release preceding major pre-release version %q (base %q, pre-release %q):"+
+			" set CORTEX_LATEST_RELEASE_IMAGE to the latest published release image", version, base, preRelease)
+	}
+
+	return fmt.Sprintf("%d.%d.%d", major, minor, patch), nil
+}
+
+func parseVersion(version string) (major, minor, patch int, err error) {
+	parts := strings.Split(version, ".")
+	if len(parts) != 3 {
+		return 0, 0, 0, errors.Errorf("expected a major.minor.patch version, got %q", version)
+	}
+
+	out := make([]int, len(parts))
+	for i, part := range parts {
+		if out[i], err = strconv.Atoi(part); err != nil {
+			return 0, 0, 0, errors.Wrapf(err, "invalid version %q", version)
+		}
+		if out[i] < 0 {
+			return 0, 0, 0, errors.Errorf("invalid version %q", version)
+		}
+	}
+
+	return out[0], out[1], out[2], nil
 }
 
 func writeFileToSharedDir(s *e2e.Scenario, dst string, content []byte) error {

integration/util_test.go

@@ -0,0 +1,94 @@
+//go:build integration
+
+package integration
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLatestReleaseVersion(t *testing.T) {
+	tests := map[string]struct {
+		version     string
+		expected    string
+		expectedErr bool
+	}{
+		"a GA version is already published": {
+			version:  "1.21.1",
+			expected: "1.21.1",
+		},
+		"a GA version with a zero patch is already published": {
+			version:  "1.21.0",
+			expected: "1.21.0",
+		},
+		"a minor release candidate falls back to the previous minor": {
+			version:  "1.22.0-rc.0",
+			expected: "1.21.0",
+		},
+		"a later minor release candidate falls back to the same previous minor": {
+			version:  "1.22.0-rc.3",
+			expected: "1.21.0",
+		},
+		"a patch release candidate falls back to the preceding patch": {
+			version:  "1.22.1-rc.0",
+			expected: "1.22.0",
+		},
+		"a later patch release candidate falls back to the preceding patch": {
+			version:  "1.22.3-rc.1",
+			expected: "1.22.2",
+		},
+		"a major release candidate cannot be resolved": {
+			version:     "2.0.0-rc.0",
+			expectedErr: true,
+		},
+		"an empty VERSION is rejected": {
+			version:     "",
+			expectedErr: true,
+		},
+		"a malformed pre-release base is rejected": {
+			version:     "1.22-rc.0",
+			expectedErr: true,
+		},
+		"a non-numeric pre-release base is rejected": {
+			version:     "1.x.0-rc.0",
+			expectedErr: true,
+		},
+	}
+
+	for name, testData := range tests {
+		t.Run(name, func(t *testing.T) {
+			actual, err := latestReleaseVersion(testData.version)
+			if testData.expectedErr {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			assert.Equal(t, testData.expected, actual)
+		})
+	}
+}
+
+func TestGetLatestReleaseImage(t *testing.T) {
+	// Point getCortexProjectDir() at a scratch checkout so we can exercise the VERSION file
+	// contents a release branch would actually have.
+	dir := t.TempDir()
+	t.Setenv("CORTEX_CHECKOUT_DIR", dir)
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "VERSION"), []byte("1.22.0-rc.0\n"), 0o600))
+
+	image, err := getLatestReleaseImage()
+	require.NoError(t, err)
+	assert.Equal(t, "quay.io/cortexproject/cortex:v1.21.0", image)
+}
+
+func TestGetLatestReleaseImage_HonorsOverride(t *testing.T) {
+	t.Setenv("CORTEX_LATEST_RELEASE_IMAGE", "quay.io/cortexproject/cortex:v1.20.1")
+
+	image, err := getLatestReleaseImage()
+	require.NoError(t, err)
+	assert.Equal(t, "quay.io/cortexproject/cortex:v1.20.1", image)
+}