From 7158a3178b363667d5d58e8111c7b10eb66371c9 Mon Sep 17 00:00:00 2001 From: Sweepy Date: Sun, 15 Nov 2026 23:01:37 +0000 Subject: [PATCH 1/2] fix(lookup): date every proposable release and admit prereleases only on purpose - Go, Maven and NuGet date and flag every listed version, not the newest few: Go `.info` and Maven POM dates are asked once per version (bounded fan-out) and kept as `release-dates` state, so a warm lookup asks only about versions it has not seen, wherever they sort; NuGet reads every registration page. A failed date request, or a NuGet version the registration does not describe yet, marks the version unchecked, and policy holds an unchecked release under a positive wait instead of reading it as undated. - A `permitted` filter admits prereleases only when it asks for them: a regex that matches the prerelease part, or a range that names a prerelease. Negated and merely narrowing filters no longer do, which stops Maven `[1,2)` or PyPI `<3` letting betas through. - mise's `prerelease: true` is carried into the catalog and treated like a prerelease version, matching `github-releases`. - A `github-tags` lookup through mise-versions falls back to GitHub's own tag catalog when the mirror leaves a tag undated, so annotated tags are dated by their tagger as the docs promise. --- crates/config/src/matching.rs | 35 +++++ crates/datasource/src/catalog.rs | 21 ++- crates/datasource/src/compact.rs | 6 +- crates/datasource/src/dates.rs | 101 +++++++++++++ crates/datasource/src/forge.rs | 4 +- crates/datasource/src/github.rs | 8 +- crates/datasource/src/helm.rs | 2 +- crates/datasource/src/images.rs | 4 +- crates/datasource/src/lib.rs | 81 +++------- crates/datasource/src/maven.rs | 60 +++++--- crates/datasource/src/mise.rs | 3 + crates/datasource/src/nuget.rs | 112 +++++++++----- crates/datasource/src/packagist.rs | 2 +- crates/datasource/src/registries.rs | 6 +- crates/datasource/src/rubygems.rs | 2 +- crates/datasource/src/versions_host.rs | 11 +- crates/datasource/tests/compact.rs | 2 +- crates/datasource/tests/go_proxy.rs | 129 ++++++++++++---- crates/datasource/tests/mise_lookup.rs | 25 +++- crates/datasource/tests/registry_protocol.rs | 6 +- crates/datasource/tests/sources.rs | 141 +++++++++++++++++- crates/datasource/tests/versions_host.rs | 94 ++++++++++++ crates/engine/src/catalogs.rs | 4 +- crates/engine/src/lib.rs | 8 +- crates/lookup/src/lib.rs | 99 ++++++++++-- crates/lookup/tests/advisories.rs | 16 ++ crates/lookup/tests/corpus.rs | 6 + crates/lookup/tests/coverage.rs | 2 + crates/lookup/tests/facts.rs | 2 + crates/lookup/tests/pinned.rs | 2 + crates/lookup/tests/policy.rs | 133 +++++++++++++++++ .../lockfiles/tests/real_sparse_registry.rs | 2 + docs/src/content/docs/guide/configuration.md | 2 +- docs/src/content/docs/guide/lookup-policy.md | 24 ++- docs/src/content/docs/guide/sources.md | 15 +- docs/src/content/docs/internals/cache.md | 7 + tests/derived-versions.tsv | 36 ++--- tests/state-exempt.tsv | 1 + tests/state-versions.tsv | 18 +-- 39 files changed, 1005 insertions(+), 227 deletions(-) create mode 100644 crates/datasource/src/dates.rs diff --git a/crates/config/src/matching.rs b/crates/config/src/matching.rs index fa98f70b..2ac314cb 100644 --- a/crates/config/src/matching.rs +++ b/crates/config/src/matching.rs @@ -129,4 +129,39 @@ impl Filter { }; Ok(yes != self.negated) } + /// Whether the filter deliberately admits the prerelease `text`, whose + /// version without its prerelease part is `release`. A negated filter + /// only excludes, so it never asks. A `regex:` filter asks when it + /// matches the prerelease but not its release, so the match comes from + /// the prerelease part (`regex:-rc\.`, `regex:nightly`). A `semver:` + /// range asks when it names a prerelease version itself + /// (`semver:>=3.0.0-rc.1`, `semver:[1.0-beta1,2)`) and matches `text`. + pub fn asks_for_prerelease( + &self, + scheme: Ecosystem, + text: &str, + release: &str, + ) -> Result { + if self.negated { + return Ok(false); + } + match &self.kind { + FilterKind::Regex(regex) => Ok(regex.is_match(text) && !regex.is_match(release)), + FilterKind::Semver(range) => { + Ok(names_prerelease(scheme, range) && self.matches(scheme, text)?) + } + } + } +} +/// Whether a range's text names a prerelease version in `scheme`'s syntax. +fn names_prerelease(scheme: Ecosystem, range: &str) -> bool { + use patchula_versioning::Versioning; + range + .split(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | '+'))) + .filter(|token| !token.is_empty()) + .any(|token| { + scheme + .parse(token) + .is_ok_and(|version| !version.0.pre.is_empty()) + }) } diff --git a/crates/datasource/src/catalog.rs b/crates/datasource/src/catalog.rs index d365e8a0..b357c1d3 100644 --- a/crates/datasource/src/catalog.rs +++ b/crates/datasource/src/catalog.rs @@ -19,10 +19,20 @@ pub struct Version { pub digest: Option, pub deprecated: bool, pub yanked: bool, + /// The source marks this release a prerelease, whatever its version + /// syntax says (a GitHub release flagged as one, listed through mise). + #[serde(default)] + pub prerelease: bool, + /// The source was not asked for this release's date and flags, or the + /// request failed: `published_at`, `deprecated` and `yanked` say + /// nothing. Policy holds such a release instead of reading it as + /// undated, so a missing answer never skips the release-age wait. + #[serde(default)] + pub unchecked: bool, } impl Version { /// A listed version with no date or digest, neither deprecated nor - /// yanked. + /// yanked: the source has been asked and has nothing more to say. pub fn new(version: String) -> Self { Self { version, @@ -30,6 +40,15 @@ impl Version { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, + } + } + /// A listed version whose date and flags the source was not asked for. + pub fn unchecked(version: String) -> Self { + Self { + unchecked: true, + ..Self::new(version) } } } diff --git a/crates/datasource/src/compact.rs b/crates/datasource/src/compact.rs index eb54d9c1..a4b0f033 100644 --- a/crates/datasource/src/compact.rs +++ b/crates/datasource/src/compact.rs @@ -59,7 +59,7 @@ pub(super) fn npm(body: &mut dyn std::io::Read) -> Result, CacheError> { sort(&mut catalog.versions, Ecosystem::Npm); encode(catalog) } -pub(crate) const NPM_CATALOG: Derived = Derived::new("npm-catalog-v2", 1, npm); +pub(crate) const NPM_CATALOG: Derived = Derived::new("npm-catalog-v2", 2, npm); #[derive(Deserialize)] struct CrateVersion { @@ -85,7 +85,7 @@ pub(super) fn crates(body: &mut dyn std::io::Read) -> Result, CacheError catalog.versions.reverse(); // Sparse index is append-only publication order. encode(catalog) } -pub(crate) const CRATES_CATALOG: Derived = Derived::new("crates-catalog-v2", 1, crates); +pub(crate) const CRATES_CATALOG: Derived = Derived::new("crates-catalog-v2", 2, crates); #[derive(Deserialize)] struct Pypi { @@ -117,7 +117,7 @@ pub(super) fn pypi(body: &mut dyn std::io::Read) -> Result, CacheError> sort(&mut catalog.versions, Ecosystem::Pep440); encode(catalog) } -pub(crate) const PYPI_CATALOG: Derived = Derived::new("pypi-catalog-v2", 1, pypi); +pub(crate) const PYPI_CATALOG: Derived = Derived::new("pypi-catalog-v2", 2, pypi); #[cfg(test)] mod tests { diff --git a/crates/datasource/src/dates.rs b/crates/datasource/src/dates.rs new file mode 100644 index 00000000..2b73d98d --- /dev/null +++ b/crates/datasource/src/dates.rs @@ -0,0 +1,101 @@ +//! Per-version release dates for sources whose listing carries none (Go's +//! `.info`, a Maven POM's `Last-Modified`). Every listed version is asked +//! about once and the answer kept, so a warm lookup asks only about versions +//! it has not seen, wherever they sort: a patch release on an older line is +//! dated like a new major. A version whose request failed is marked +//! unchecked and asked about again by the next lookup. +use super::*; +use futures_util::{StreamExt, stream}; + +/// Per-version date requests a lookup has in flight at once. +const FAN_OUT: usize = 8; + +/// What the source said about one version. +#[derive(Clone, Serialize, Deserialize)] +pub(crate) enum Dated { + /// The version exists; its publication time, when the source gives one. + At(Option), + /// The source lists the version but cannot serve it: not installable. + Missing, +} +/// The answers kept for one package, by version. +#[derive(Default, Serialize, Deserialize)] +struct Dates(BTreeMap); +#[derive(Serialize)] +pub(crate) struct DatesScope { + pub(crate) source: Source, + pub(crate) package: String, + /// The operator registry entry's scope when one answered, so a date read + /// with a credential is only reused under the same one. + pub(crate) credential: Option, +} +impl State for Dates { + const NAMESPACE: &'static str = "release-dates"; + const VERSION: u32 = 1; + type Scope = DatesScope; +} + +impl Client { + /// Date every version in `versions` that has no date yet, asking `ask` + /// only about versions the store has no answer for. `ask` returns + /// `None` when the request failed: that version is marked unchecked and + /// nothing is stored for it. Missing versions are dropped. + pub(crate) async fn date_versions( + &self, + scope: &DatesScope, + versions: &mut Vec, + ask: F, + ) -> Result<()> + where + F: Fn(String) -> Fut, + Fut: std::future::Future>, + { + let mut dates = self.cache.state::(scope).await?.unwrap_or_default(); + let wanted: Vec = versions + .iter() + .filter(|v| v.published_at.is_none() && !dates.0.contains_key(&v.version)) + .map(|v| v.version.clone()) + .collect(); + let answers: Vec<(String, Option)> = stream::iter(wanted) + .map(|version| { + let answer = ask(version.clone()); + async move { (version, answer.await) } + }) + .buffer_unordered(FAN_OUT) + .collect() + .await; + let mut failed = std::collections::HashSet::new(); + let mut learned = false; + for (version, answer) in answers { + match answer { + Some(dated) => { + dates.0.insert(version, dated); + learned = true; + } + None => { + failed.insert(version); + } + } + } + versions.retain_mut(|version| { + if failed.contains(&version.version) { + version.unchecked = true; + return true; + } + match dates.0.get(&version.version) { + Some(Dated::Missing) => false, + Some(Dated::At(time)) => { + if version.published_at.is_none() { + version.published_at.clone_from(time); + } + true + } + None => true, + } + }); + if learned { + self.cache.set_state(scope, &dates).await?; + } + Ok(()) + } +} diff --git a/crates/datasource/src/forge.rs b/crates/datasource/src/forge.rs index e105972d..b392b0d9 100644 --- a/crates/datasource/src/forge.rs +++ b/crates/datasource/src/forge.rs @@ -35,9 +35,9 @@ fn tags( serde_json::to_vec(&versions).map_err(|_| patchula_http::CacheError::Document) } pub(crate) const FORGE_GITLAB_TAGS: patchula_http::Derived = - patchula_http::Derived::new("forge-gitlab-tags", 1, gitlab); + patchula_http::Derived::new("forge-gitlab-tags", 2, gitlab); pub(crate) const FORGE_GITEA_TAGS: patchula_http::Derived = - patchula_http::Derived::new("forge-gitea-tags", 1, gitea); + patchula_http::Derived::new("forge-gitea-tags", 2, gitea); fn forge(package: &str) -> Result { Repository::from_url(&format!("https://{package}")) diff --git a/crates/datasource/src/github.rs b/crates/datasource/src/github.rs index 1d3ec549..1bab51c2 100644 --- a/crates/datasource/src/github.rs +++ b/crates/datasource/src/github.rs @@ -55,7 +55,7 @@ fn compact_release_head( >(body)?) } pub(crate) const GITHUB_RELEASE_HEAD: patchula_http::Derived = - patchula_http::Derived::new("github-release-head", 5, compact_release_head); + patchula_http::Derived::new("github-release-head", 6, compact_release_head); /// The newest release's identity on a release source; a change refetches /// history. `None` records a source with no releases. @@ -68,7 +68,7 @@ pub(crate) struct SourceHeadScope { } impl State for SourceHead { const NAMESPACE: &'static str = "source-head"; - const VERSION: u32 = 5; + const VERSION: u32 = 6; // Code-versioned: the stored fingerprint holds the newest release's // fields as this build picks them, an entry built by code that picked // differently must not compare equal. @@ -85,7 +85,7 @@ pub(crate) struct TagCatalogScope { } impl State for TagCatalog { const NAMESPACE: &'static str = "tag-catalog"; - const VERSION: u32 = 5; + const VERSION: u32 = 6; // Code-versioned: the fingerprint in the key names the refs, not the // catalog this build's peel-and-map code derived from them. const CODE_VERSIONED: bool = true; @@ -101,7 +101,7 @@ pub(crate) struct TagObjectScope { } impl State for TagObject { const NAMESPACE: &'static str = "tag-object"; - const VERSION: u32 = 5; + const VERSION: u32 = 6; // Code-versioned: the stored version is this build's decode of the // object the scope's sha names. const CODE_VERSIONED: bool = true; diff --git a/crates/datasource/src/helm.rs b/crates/datasource/src/helm.rs index dcbb306b..0a8168ae 100644 --- a/crates/datasource/src/helm.rs +++ b/crates/datasource/src/helm.rs @@ -1036,7 +1036,7 @@ fn helm_walk(body: &mut dyn Read) -> std::result::Result, CacheError> { .collect(); serde_json::to_vec(&charts).map_err(|_| CacheError::Document) } -pub(crate) const HELM_INDEX: Derived = Derived::new("helm-index", 3, helm); +pub(crate) const HELM_INDEX: Derived = Derived::new("helm-index", 4, helm); #[cfg(test)] mod tests { diff --git a/crates/datasource/src/images.rs b/crates/datasource/src/images.rs index 8f114a3d..bcb8bb2e 100644 --- a/crates/datasource/src/images.rs +++ b/crates/datasource/src/images.rs @@ -133,7 +133,7 @@ fn compact_platform( .map_err(|_| Document) } pub(crate) const IMAGE_PLATFORM: patchula_http::Derived = - patchula_http::Derived::new("image-platform", 3, compact_platform); + patchula_http::Derived::new("image-platform", 4, compact_platform); fn platform_name(os: &str, architecture: &str, variant: Option<&str>) -> String { match variant { @@ -304,7 +304,7 @@ fn compact_sbom( serde_json::to_vec(&compact).map_err(|_| Document) } pub(crate) const SBOM_OS: patchula_http::Derived = - patchula_http::Derived::new("sbom-os", 3, compact_sbom); + patchula_http::Derived::new("sbom-os", 4, compact_sbom); /// OSV's ecosystem name for an OS release, when OSV covers it. fn ecosystem(kind: &str, namespace: &str, distro: Option<&(String, String)>) -> Option { diff --git a/crates/datasource/src/lib.rs b/crates/datasource/src/lib.rs index 32707bc5..11088e43 100644 --- a/crates/datasource/src/lib.rs +++ b/crates/datasource/src/lib.rs @@ -6,6 +6,7 @@ mod catalog; pub use catalog::{Catalog, Version}; pub(crate) use catalog::{sha256_hex, sort}; mod compact; +mod dates; #[cfg(test)] mod derive_parity; mod docker; @@ -144,8 +145,6 @@ impl std::fmt::Display for Failure { f.write_str(&self.message) } } -/// Go releases dated through `.info` on a cold lookup, newest first. -const GO_DATED_RELEASES: usize = 20; /// A stored listing or resolution is reused for this long. The tools keep /// their own version lists for about as long, and a release-age wait is /// evaluated against the dates in the listing, so a stored one never hides @@ -437,48 +436,32 @@ impl Client { latest_version }; sort(&mut versions, Ecosystem::Go); - let known = self.cache.releases("go", package).await?; - let known: std::collections::HashMap<_, _> = known - .releases - .into_iter() - .map(|v| (v.version, v.published_at)) - .collect(); - // Fetch .info only for new releases; retain publication timestamps for - // older ones. Never refetch full per-version history on a warm lookup. - // A cold lookup dates only the newest releases: older ones are outside - // any release-age wait, and a module can list hundreds of versions. - // The proxy lists some versions it cannot describe (404); those are - // not installable releases and are dropped. - let mut reached_known = false; - let mut fetched = 0; - let mut missing = std::collections::HashSet::new(); - for version in &mut versions { - if let Some(time) = known.get(&version.version) { - version.published_at.clone_from(time); - reached_known = true; - } else if !reached_known && fetched < GO_DATED_RELEASES { - fetched += 1; - let response = self - .http - .get( - &format!("{prefix}/{}.info", version.version), - "application/json", - ) - .await; - let response = match response { - Ok(response) => response, + // Every listed version is dated through `.info`, once: the answer is + // kept, so a warm lookup asks only about versions it has not seen, + // including a patch release on an older line. The proxy lists some + // versions it cannot describe (404); those are not installable + // releases and are dropped. + let scope = dates::DatesScope { + source: Source::Go, + package: package.into(), + credential: None, + }; + self.date_versions(&scope, &mut versions, |version| { + let url = format!("{prefix}/{version}.info"); + async move { + match self.http.get(&url, "application/json").await { + Ok(response) => { + let info: Value = serde_json::from_slice(&response.body).ok()?; + Some(dates::Dated::At(info["Time"].as_str().map(str::to_owned))) + } Err(patchula_http::CacheError::Status(404 | 410)) => { - missing.insert(version.version.clone()); - continue; + Some(dates::Dated::Missing) } - Err(error) => return Err(error.into()), - }; - let info: Value = serde_json::from_slice(&response.body)?; - version.published_at = info["Time"].as_str().map(str::to_owned); + Err(_) => None, + } } - } - versions.retain(|v| !missing.contains(&v.version)); - self.merge("go", package, &versions).await?; + }) + .await?; Ok(Catalog { versions, dist_tags: BTreeMap::from([("latest".into(), latest_version)]), @@ -509,22 +492,6 @@ impl Client { }; Ok(serde_json::from_slice(&response.body)?) } - async fn merge(&self, source: &str, package: &str, versions: &[Version]) -> Result<()> { - let head = self.cache.release_head(source, package).await?; - let mut page = Vec::new(); - for v in versions { - page.push(Release { - version: v.version.clone(), - published_at: v.published_at.clone(), - digest: v.digest.clone(), - }); - if head.as_deref() == Some(&v.version) { - break; - } - } - self.cache.merge_releases(source, package, &page).await?; - Ok(()) - } } fn runners(package: &str) -> Result { use std::sync::LazyLock; diff --git a/crates/datasource/src/maven.rs b/crates/datasource/src/maven.rs index d601a626..3cabc89d 100644 --- a/crates/datasource/src/maven.rs +++ b/crates/datasource/src/maven.rs @@ -1,12 +1,10 @@ //! Maven repositories: `maven-metadata.xml` lists versions, and each POM's -//! `Last-Modified` dates the newest releases for the release-age wait. +//! `Last-Modified` dates every one of them for the release-age wait. use super::*; use patchula_config::split_package; const CENTRAL: &str = "https://repo.maven.apache.org/maven2"; const GOOGLE: &str = "https://dl.google.com/dl/android/maven2"; -/// Only recent releases can still be inside a release-age wait. -const DATED: usize = 10; /// `group:artifact`, optionally prefixed by `repository-url|`. Android groups /// default to Google's Maven repository, everything else to Maven Central. @@ -61,7 +59,7 @@ fn versions( serde_json::to_vec(&versions).map_err(|_| patchula_http::CacheError::Document) } pub(crate) const MAVEN_METADATA: patchula_http::Derived = - patchula_http::Derived::new("maven-metadata", 1, versions); + patchula_http::Derived::new("maven-metadata", 2, versions); /// The directory holding an artifact's `maven-metadata.xml`. fn base(package: &str) -> Result { @@ -87,27 +85,43 @@ impl Client { let listed: Vec = serde_json::from_slice(&response.body)?; let mut versions: Vec = listed.into_iter().map(Version::new).collect(); sort(&mut versions, Ecosystem::Maven); - let dates = futures_util::future::join_all(versions.iter().take(DATED).map(|v| { - let url = format!("{base}/{0}/{artifact}-{0}.pom", v.version); + // Every version is dated once, by its POM, and the answer kept: a + // new release on an older line is dated like a new major. A version + // without a POM is not resolvable and is dropped; a failed request + // leaves the version unchecked, which policy holds. + let scope = dates::DatesScope { + source: Source::Maven, + package: base.clone(), + credential: self + .http + .credential(&format!("{base}/")) + .map(|entry| entry.scope().to_owned()), + }; + self.date_versions(&scope, &mut versions, |version| { + let url = format!("{base}/{version}/{artifact}-{version}.pom"); async move { - let response = self - .http - .registry_head(&url, "application/xml", None) - .await - .ok()?; - let modified = response.headers.get("last-modified")?.to_str().ok()?; - chrono::DateTime::parse_from_rfc2822(modified) - .ok() - .map(|t| { - t.to_utc() - .to_rfc3339_opts(chrono::SecondsFormat::Secs, true) - }) + match self.http.registry_head(&url, "application/xml", None).await { + Ok(response) => Some(dates::Dated::At( + response + .headers + .get("last-modified") + .and_then(|modified| modified.to_str().ok()) + .and_then(|modified| { + chrono::DateTime::parse_from_rfc2822(modified).ok() + }) + .map(|t| { + t.to_utc() + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + }), + )), + Err(patchula_http::CacheError::Status(404 | 410)) => { + Some(dates::Dated::Missing) + } + Err(_) => None, + } } - })) - .await; - for (version, date) in versions.iter_mut().zip(dates) { - version.published_at = date; - } + }) + .await?; Ok(Catalog { versions, dist_tags: BTreeMap::new(), diff --git a/crates/datasource/src/mise.rs b/crates/datasource/src/mise.rs index 0c1ff255..34501425 100644 --- a/crates/datasource/src/mise.rs +++ b/crates/datasource/src/mise.rs @@ -265,6 +265,9 @@ impl Client { .into_iter() .map(|release| Version { published_at: release.created_at.as_deref().map(date), + // A GitHub-backed tool's release flagged as a prerelease is + // one, even when its version looks stable. + prerelease: release.prerelease == Some(true), ..Version::new(release.version) }) .collect(); diff --git a/crates/datasource/src/nuget.rs b/crates/datasource/src/nuget.rs index 4faa9858..feddb1cb 100644 --- a/crates/datasource/src/nuget.rs +++ b/crates/datasource/src/nuget.rs @@ -1,6 +1,11 @@ -//! nuget.org: the flat container lists versions; the newest registration -//! page dates them and marks unlisted (yanked) and deprecated releases. +//! nuget.org: the flat container lists versions; the registration pages, +//! every one of them, date them and mark unlisted (yanked) and deprecated +//! releases. use super::*; +use futures_util::{StreamExt, stream}; + +/// Registration pages read at once. +const PAGE_FAN_OUT: usize = 4; #[derive(Serialize, Deserialize)] struct Entry { @@ -22,7 +27,7 @@ fn listing( serde_json::to_vec(&index.versions).map_err(|_| patchula_http::CacheError::Document) } pub(crate) const NUGET_FLAT: patchula_http::Derived = - patchula_http::Derived::new("nuget-flat", 1, listing); + patchula_http::Derived::new("nuget-flat", 2, listing); fn entries(items: &serde_json::Value) -> Vec { items .as_array() @@ -43,24 +48,28 @@ fn entries(items: &serde_json::Value) -> Vec { }) .collect() } -/// The registration index: the newest page's entries, inline or as a link. +/// The registration index: every page, its entries inline or as a link. fn registration( body: &mut dyn std::io::Read, ) -> std::result::Result, patchula_http::CacheError> { let index: serde_json::Value = patchula_http::conditional::json_document(body) .map_err(|_| patchula_http::CacheError::Document)?; - let newest = index["items"].as_array().and_then(|pages| pages.last()); - let compact = match newest { - Some(page) if page["items"].is_array() => { - serde_json::json!({"entries": entries(&page["items"])}) - } - Some(page) => serde_json::json!({"page": page["@id"]}), - None => serde_json::json!({"entries": []}), - }; - serde_json::to_vec(&compact).map_err(|_| patchula_http::CacheError::Document) + let pages: Vec = index["items"] + .as_array() + .ok_or(patchula_http::CacheError::Document)? + .iter() + .map(|page| { + if page["items"].is_array() { + serde_json::json!({"entries": entries(&page["items"])}) + } else { + serde_json::json!({"page": page["@id"]}) + } + }) + .collect(); + serde_json::to_vec(&pages).map_err(|_| patchula_http::CacheError::Document) } pub(crate) const NUGET_REGISTRATION: patchula_http::Derived = - patchula_http::Derived::new("nuget-registration", 1, registration); + patchula_http::Derived::new("nuget-registration", 2, registration); fn page(body: &mut dyn std::io::Read) -> std::result::Result, patchula_http::CacheError> { let page: serde_json::Value = patchula_http::conditional::json_document(body) .map_err(|_| patchula_http::CacheError::Document)?; @@ -68,7 +77,17 @@ fn page(body: &mut dyn std::io::Read) -> std::result::Result, patchula_h .map_err(|_| patchula_http::CacheError::Document) } pub(crate) const NUGET_REGISTRATION_PAGE: patchula_http::Derived = - patchula_http::Derived::new("nuget-registration-page", 1, page); + patchula_http::Derived::new("nuget-registration-page", 2, page); + +/// How a version is matched between the flat container and the +/// registration: case-insensitive and without build metadata, which the +/// flat container leaves out. +fn identity(version: &str) -> String { + version + .split_once('+') + .map_or(version, |(version, _)| version) + .to_ascii_lowercase() +} impl Client { pub(crate) async fn nuget(&self, package: &str) -> Result { @@ -89,12 +108,12 @@ impl Client { .await?; let listed: Vec = serde_json::from_slice(&response.body)?; #[derive(Deserialize)] - struct Metadata { + struct Page { #[serde(default)] entries: Vec, page: Option, } - let mut metadata: Metadata = serde_json::from_slice( + let pages: Vec = serde_json::from_slice( &self .http .get_derived( @@ -105,34 +124,51 @@ impl Client { .await? .body, )?; - if let Some(url) = metadata - .page - .take() - .filter(|u| u.starts_with("https://api.nuget.org/")) - { - metadata.entries = serde_json::from_slice::( - &self - .http - .get_derived(&url, "application/json", &NUGET_REGISTRATION_PAGE) - .await? - .body, - )? - .entries; + // Every page is read, not just the newest: a page holds 64 versions + // counted from the oldest, so an older line's new patch release, or + // an unlisted one, can sit on any of them. A page that fails fails + // the lookup rather than leave its versions looking listed. + let pages: Vec>> = stream::iter(pages) + .map(|page| async move { + let Some(url) = page.page else { + return Ok(page.entries); + }; + if !url.starts_with("https://api.nuget.org/") { + return Err(Error::Invalid); + } + Ok(serde_json::from_slice::( + &self + .http + .get_derived(&url, "application/json", &NUGET_REGISTRATION_PAGE) + .await? + .body, + )? + .entries) + }) + .buffered(PAGE_FAN_OUT) + .collect() + .await; + let mut metadata = Vec::new(); + for page in pages { + metadata.extend(page?); } let known: BTreeMap = metadata - .entries .into_iter() - .map(|e| (e.version.to_ascii_lowercase(), e)) + .map(|e| (identity(&e.version), e)) .collect(); let mut versions: Vec = listed .into_iter() .map(|version| { - let entry = known.get(&version.to_ascii_lowercase()); - Version { - published_at: entry.and_then(|e| e.published.clone()), - deprecated: entry.is_some_and(|e| e.deprecated), - yanked: entry.is_some_and(|e| !e.listed), - ..Version::new(version) + // A version the registration does not describe yet (it + // trails the flat container) is unchecked, which policy holds. + match known.get(&identity(&version)) { + Some(entry) => Version { + published_at: entry.published.clone(), + deprecated: entry.deprecated, + yanked: !entry.listed, + ..Version::new(version) + }, + None => Version::unchecked(version), } }) .collect(); diff --git a/crates/datasource/src/packagist.rs b/crates/datasource/src/packagist.rs index 809700f4..96bdeeb6 100644 --- a/crates/datasource/src/packagist.rs +++ b/crates/datasource/src/packagist.rs @@ -25,7 +25,7 @@ fn versions( serde_json::to_vec(&versions).map_err(|_| patchula_http::CacheError::Document) } pub(crate) const PACKAGIST_P2: patchula_http::Derived = - patchula_http::Derived::new("packagist-p2", 1, versions); + patchula_http::Derived::new("packagist-p2", 2, versions); impl Client { pub(crate) async fn packagist(&self, package: &str) -> Result { diff --git a/crates/datasource/src/registries.rs b/crates/datasource/src/registries.rs index 34632337..78c8e28b 100644 --- a/crates/datasource/src/registries.rs +++ b/crates/datasource/src/registries.rs @@ -38,7 +38,7 @@ fn terraform(body: &mut dyn std::io::Read) -> std::result::Result, Cache encode(&catalog) } pub(crate) const TERRAFORM_CATALOG: Derived = - Derived::new("terraform-catalog", 3, terraform); + Derived::new("terraform-catalog", 4, terraform); /// One chart from a Helm repository index: its versions, and what release /// notes read from each version's metadata. @@ -436,13 +436,13 @@ fn simple_any(body: &mut dyn std::io::Read) -> std::result::Result, Cach serde_json::to_vec(&catalogs).map_err(|_| CacheError::Document) } pub(crate) const PYPI_INDEX_CATALOG: Derived = - Derived::new("pypi-index-catalog", 3, simple_any); + Derived::new("pypi-index-catalog", 4, simple_any); fn simple(body: &mut dyn std::io::Read) -> std::result::Result, CacheError> { encode(&catalog_of(&simple_files(body)?)) } pub(crate) const PYPI_SIMPLE_CATALOG: Derived = - Derived::new("pypi-simple-catalog", 3, simple); + Derived::new("pypi-simple-catalog", 4, simple); #[derive(Deserialize)] struct SimpleFile { diff --git a/crates/datasource/src/rubygems.rs b/crates/datasource/src/rubygems.rs index 8ab63a29..1e22c651 100644 --- a/crates/datasource/src/rubygems.rs +++ b/crates/datasource/src/rubygems.rs @@ -26,7 +26,7 @@ fn versions( .map_err(|_| patchula_http::CacheError::Document) } pub(crate) const RUBYGEMS_VERSIONS: patchula_http::Derived = - patchula_http::Derived::new("rubygems-versions", 1, versions); + patchula_http::Derived::new("rubygems-versions", 2, versions); impl Client { pub(crate) async fn rubygems(&self, package: &str) -> Result { diff --git a/crates/datasource/src/versions_host.rs b/crates/datasource/src/versions_host.rs index f47f1ef2..25b7b42f 100644 --- a/crates/datasource/src/versions_host.rs +++ b/crates/datasource/src/versions_host.rs @@ -69,7 +69,16 @@ impl Client { )); } // Tags carry no reliable dates of their own; the releases - // document dates them. + // document dates them. A tag it leaves undated may be an + // annotated tag, which GitHub dates by its tagger: the mirror + // cannot tell, so GitHub's own tag catalog answers instead. + if tags + .tags + .iter() + .any(|tag| dates.get(&tag.tag).cloned().flatten().is_none()) + { + return Ok(None); + } tags.tags .iter() .map(|tag| { diff --git a/crates/datasource/tests/compact.rs b/crates/datasource/tests/compact.rs index dfba900a..71a0c53a 100644 --- a/crates/datasource/tests/compact.rs +++ b/crates/datasource/tests/compact.rs @@ -120,7 +120,7 @@ async fn fresh_reopen_304_changed_200_and_invalid_200_preserve_catalog_semantics BTreeMap::from([("accept".into(), "application/json".into())]), ) .unwrap() - .representation(&patchula_cache::representation("npm-catalog-v2", 1)); + .representation(&patchula_cache::representation("npm-catalog-v2", 2)); let mut entry = cache.http_get(&key).await.unwrap().unwrap(); assert!( entry.body.len() < 500, diff --git a/crates/datasource/tests/go_proxy.rs b/crates/datasource/tests/go_proxy.rs index 4ef76cbb..f82e7311 100644 --- a/crates/datasource/tests/go_proxy.rs +++ b/crates/datasource/tests/go_proxy.rs @@ -1,28 +1,48 @@ //! Independent Go proxy contract fixtures; no network client is constructed. use patchula_cache::Cache; -use patchula_datasource::Client; +use patchula_datasource::{Catalog, Client}; use patchula_http::{Exchange, Response, Transport}; use patchula_words::Source; use std::sync::{Arc, Mutex}; -/// A module listing thirty releases, one of which the proxy cannot describe. +/// A module proxy whose listing, undescribable (404) and failing (500) +/// versions a test sets, recording every `.info` request. struct Proxy { + list: Mutex>, + missing: Vec, + failing: Mutex>, infos: Mutex>, } +impl Proxy { + fn new(list: Vec, missing: &[&str]) -> Arc { + Arc::new(Self { + list: Mutex::new(list), + missing: missing.iter().map(|v| (*v).to_owned()).collect(), + failing: Mutex::new(Vec::new()), + infos: Mutex::new(Vec::new()), + }) + } + fn infos(&self) -> Vec { + let mut infos = std::mem::take(&mut *self.infos.lock().unwrap()); + infos.sort(); + infos + } +} impl Transport for Proxy { fn send(&self, request: reqwest::Request) -> Exchange<'_> { Box::pin(async move { let path = request.url().path().to_string(); let (status, body) = if path.ends_with("/@v/list") { - let list: Vec = (0..30).map(|n| format!("v1.{n}.0")).collect(); - (200, list.join("\n")) + (200, self.list.lock().unwrap().join("\n")) } else if let Some(version) = path .strip_prefix("/example.com/mod/@v/") .and_then(|p| p.strip_suffix(".info")) { self.infos.lock().unwrap().push(version.to_string()); - if version == "v1.28.0" { + if self.missing.iter().any(|v| v == version) { (404, "not found: unknown revision".to_string()) + } else if self.failing.lock().unwrap().iter().any(|v| v == version) { + (500, "unavailable".to_string()) } else { ( 200, @@ -34,20 +54,16 @@ impl Transport for Proxy { }; Ok(Response { status: reqwest::StatusCode::from_u16(status).unwrap(), - headers: Default::default(), + headers: [(reqwest::header::CACHE_CONTROL, "no-store".parse().unwrap())] + .into_iter() + .collect(), body: body.into_bytes(), stale: None, }) }) } } - -#[tokio::test] -async fn undescribed_versions_are_dropped_and_cold_lookups_date_only_recent_releases() { - let proxy = Arc::new(Proxy { - infos: Mutex::new(Vec::new()), - }); - let cache = Cache::open(":memory:", Default::default()).await.unwrap(); +async fn lookup(proxy: &Arc, cache: &Cache) -> Catalog { let github = patchula_github::Client::with_transport( "offline", cache.clone(), @@ -55,19 +71,35 @@ async fn undescribed_versions_are_dropped_and_cold_lookups_date_only_recent_rele proxy.clone(), ) .unwrap(); - let client = Client::new( + Client::new( github, patchula_datasource::cached(cache.clone(), proxy.clone(), 4).unwrap(), - cache, + cache.clone(), ) - .with_versions_host(false); - let catalog = client - .lookup(&patchula_config::PackageId::public( - Source::Go, - "example.com/mod", - )) - .await - .unwrap(); + .with_versions_host(false) + .lookup(&patchula_config::PackageId::public( + Source::Go, + "example.com/mod", + )) + .await + .unwrap() +} +fn version<'a>(catalog: &'a Catalog, version: &str) -> &'a patchula_datasource::Version { + catalog + .versions + .iter() + .find(|v| v.version == version) + .unwrap_or_else(|| panic!("{version} is listed")) +} + +/// Every listed version is dated once, however many there are; versions +/// the proxy cannot describe are dropped. +#[tokio::test] +async fn every_version_is_dated_once_and_undescribed_versions_are_dropped() { + let list: Vec = (0..30).map(|n| format!("v1.{n}.0")).collect(); + let proxy = Proxy::new(list, &["v1.28.0"]); + let cache = Cache::open(":memory:", Default::default()).await.unwrap(); + let catalog = lookup(&proxy, &cache).await; let versions: Vec<_> = catalog .versions .iter() @@ -76,8 +108,51 @@ async fn undescribed_versions_are_dropped_and_cold_lookups_date_only_recent_rele assert_eq!(versions.len(), 29, "{versions:?}"); assert!(!versions.contains(&"v1.28.0")); assert_eq!(versions[0], "v1.29.0"); - let infos = proxy.infos.lock().unwrap(); - assert_eq!(infos.len(), 20, "{infos:?}"); - assert!(catalog.versions[0].published_at.is_some()); - assert!(catalog.versions.last().unwrap().published_at.is_none()); + assert_eq!(proxy.infos().len(), 30); + assert!( + catalog + .versions + .iter() + .all(|v| v.published_at.is_some() && !v.unchecked) + ); + // A warm lookup asks about nothing it has already dated, not even the + // version the proxy could not describe. + lookup(&proxy, &cache).await; + assert_eq!(proxy.infos(), Vec::::new()); +} + +/// A new patch release on an older line is dated, though newer versions +/// were dated before it appeared. +#[tokio::test] +async fn a_new_release_below_the_newest_known_is_dated() { + let proxy = Proxy::new(vec!["v0.29.3".into(), "v0.30.0".into()], &[]); + let cache = Cache::open(":memory:", Default::default()).await.unwrap(); + lookup(&proxy, &cache).await; + assert_eq!(proxy.infos(), ["v0.29.3", "v0.30.0"]); + proxy.list.lock().unwrap().push("v0.29.4".into()); + let catalog = lookup(&proxy, &cache).await; + assert_eq!(proxy.infos(), ["v0.29.4"]); + assert_eq!( + version(&catalog, "v0.29.4").published_at.as_deref(), + Some("2026-01-01T00:00:00Z") + ); +} + +/// A version whose `.info` request fails is unchecked, not undated, and is +/// asked about again by the next lookup. +#[tokio::test] +async fn a_failed_date_request_leaves_the_version_unchecked_and_is_retried() { + let proxy = Proxy::new(vec!["v1.0.0".into(), "v1.0.1".into()], &[]); + proxy.failing.lock().unwrap().push("v1.0.1".into()); + let cache = Cache::open(":memory:", Default::default()).await.unwrap(); + let catalog = lookup(&proxy, &cache).await; + let failed = version(&catalog, "v1.0.1"); + assert!(failed.unchecked && failed.published_at.is_none()); + assert!(!version(&catalog, "v1.0.0").unchecked); + proxy.infos(); + proxy.failing.lock().unwrap().clear(); + let catalog = lookup(&proxy, &cache).await; + assert_eq!(proxy.infos(), ["v1.0.1"]); + let dated = version(&catalog, "v1.0.1"); + assert!(!dated.unchecked && dated.published_at.is_some()); } diff --git a/crates/datasource/tests/mise_lookup.rs b/crates/datasource/tests/mise_lookup.rs index 2ddbd351..c298ed58 100644 --- a/crates/datasource/tests/mise_lookup.rs +++ b/crates/datasource/tests/mise_lookup.rs @@ -119,16 +119,35 @@ async fn prereleases_stay_in_the_catalog_for_the_policy_to_decide() { .unwrap(); for version in ["3.0.0-alpha.0", "3.0.0-alpha.11", "1.11.0-rc.1"] { assert!( - prettier.versions.iter().any(|v| v.version == version), + prettier + .versions + .iter() + .any(|v| v.version == version && v.prerelease), "{version}" ); } - // GitHub-backed tools mark them too, with versions that look stable. + assert!( + prettier + .versions + .iter() + .any(|v| v.version == "3.0.0" && !v.prerelease) + ); + // GitHub-backed tools mark them too, and the flag is carried even where + // the version would look stable, so policy treats it as a prerelease. let jq = source .lookup(&patchula_config::PackageId::public(Source::Mise, "jq")) .await .unwrap(); - assert!(jq.versions.iter().any(|v| v.version == "1.7rc2")); + assert!( + jq.versions + .iter() + .any(|v| v.version == "1.7rc2" && v.prerelease) + ); + assert!( + jq.versions + .iter() + .any(|v| v.version == "1.7.1" && !v.prerelease) + ); } #[tokio::test] diff --git a/crates/datasource/tests/registry_protocol.rs b/crates/datasource/tests/registry_protocol.rs index 0134d5df..4a450215 100644 --- a/crates/datasource/tests/registry_protocol.rs +++ b/crates/datasource/tests/registry_protocol.rs @@ -496,11 +496,9 @@ async fn go_skips_latest_for_tagged_versions_and_preserves_pseudo_version_fallba ), )); } + // Every listed version is dated through `.info`; @latest's + // pseudo-version already carries its time. let mut versions: Vec<_> = list.lines().collect(); - if let Some(latest) = latest { - versions.push(latest); - } - // The source sorts descending and hydrates every new tagged version. versions.sort_by(|a, b| b.cmp(a)); for version in versions { reads.push_back(( diff --git a/crates/datasource/tests/sources.rs b/crates/datasource/tests/sources.rs index f5142d4a..f4944cc5 100644 --- a/crates/datasource/tests/sources.rs +++ b/crates/datasource/tests/sources.rs @@ -114,8 +114,10 @@ async fn nuget_versions_take_dates_and_flags_from_the_registration() { assert!(!at("13.0.2").yanked && !at("13.0.2").deprecated); // Unlisted is yanked; a deprecation object marks deprecated. assert!(at("13.0.1").yanked && at("13.0.1").deprecated); - // A version the registration does not know lists without metadata. - assert!(at("12.0.3").published_at.is_none() && !at("12.0.3").yanked); + // A version the registration does not describe yet is unchecked, which + // policy holds, rather than listed and undated. + assert!(at("12.0.3").unchecked && at("12.0.3").published_at.is_none()); + assert!(!at("13.0.2").unchecked); assert!(steps.0.lock().unwrap().is_empty()); } @@ -153,7 +155,8 @@ async fn nuget_registration_follows_only_nugets_own_pages() { ), ( "https://api.nuget.org/v3/registration5-gz-semver2/offsite/index.json".into(), - // A page link off nuget.org is never requested. + // A page link off nuget.org is never requested, and fails the + // lookup rather than leave its versions looking listed. response( 200, r#"{"items":[{"@id":"https://evil.example.com/page.json","count":1}]}"#, @@ -173,15 +176,67 @@ async fn nuget_registration_follows_only_nugets_own_pages() { Source::Nuget, "offsite", )) + .await; + assert!(offsite.is_err()); + assert!(steps.0.lock().unwrap().is_empty()); +} + +/// Every registration page is read: an older page holds an unlisted patch +/// release on an older line, which must not look listed and undated. +#[tokio::test] +async fn nuget_reads_every_registration_page() { + let older = "https://api.nuget.org/v3/registration5-gz-semver2/lib/page-6.json"; + let steps = Arc::new(Steps(Mutex::new(VecDeque::from([ + ( + "https://api.nuget.org/v3-flatcontainer/lib/index.json".into(), + response(200, r#"{"versions":["6.0.1","6.0.5","7.0.0"]}"#, &[]), + ), + ( + "https://api.nuget.org/v3/registration5-gz-semver2/lib/index.json".into(), + response( + 200, + &format!( + r#"{{"items":[ + {{"@id":"{older}","count":2,"lower":"6.0.1","upper":"6.0.5"}}, + {{"count":1,"lower":"7.0.0","upper":"7.0.0","items":[ + {{"catalogEntry":{{"version":"7.0.0","published":"2024-06-01T00:00:00Z","listed":true}}}} + ]}} + ]}}"# + ), + &[], + ), + ), + ( + older.into(), + response( + 200, + r#"{"items":[ + {"catalogEntry":{"version":"6.0.1","published":"2023-01-01T00:00:00Z","listed":true}}, + {"catalogEntry":{"version":"6.0.5+build.7","published":"2024-07-01T00:00:00Z","listed":false}} + ]}"#, + &[], + ), + ), + ])))); + let dir = tempfile::tempdir().unwrap(); + let catalog = client(steps.clone(), dir.path()) + .await + .lookup(&patchula_config::PackageId::public(Source::Nuget, "lib")) .await .unwrap(); - assert_eq!(offsite.versions.len(), 1); - assert!(offsite.versions[0].published_at.is_none()); + let patch = catalog + .versions + .iter() + .find(|v| v.version == "6.0.5") + .unwrap(); + assert!(patch.yanked && !patch.unchecked); + assert_eq!(patch.published_at.as_deref(), Some("2024-07-01T00:00:00Z")); + assert!(catalog.versions.iter().all(|v| !v.unchecked)); assert!(steps.0.lock().unwrap().is_empty()); } #[tokio::test] -async fn maven_reads_versions_and_dates_the_newest_from_poms() { +async fn maven_reads_versions_and_dates_each_from_its_pom() { let steps = Arc::new(Steps(Mutex::new(VecDeque::from([ ( "https://maven.acme.dev/com/acme/lib/maven-metadata.xml".into(), @@ -256,6 +311,80 @@ async fn maven_reads_versions_and_dates_the_newest_from_poms() { .unwrap(); assert_eq!(central.versions[0].version, "3.0.0"); assert!(central.versions[0].published_at.is_none()); + assert!(!central.versions[0].unchecked); + assert!(steps.0.lock().unwrap().is_empty()); +} + +/// Every Maven version is dated once, not just the newest ten; a later +/// lookup asks only about versions it has not dated, wherever they sort. +/// A failed POM request leaves its version unchecked and is asked again; a +/// version without a POM is dropped. +#[tokio::test] +async fn maven_dates_every_version_once_and_retries_failures() { + let base = "https://repo.maven.apache.org/maven2/com/example/lib"; + let metadata = |versions: &[&str]| { + let listed: String = versions + .iter() + .map(|v| format!("{v}")) + .collect(); + response( + 200, + &format!("{listed}"), + &[("cache-control", "no-store")], + ) + }; + let pom = |version: &str| format!("{base}/{version}/lib-{version}.pom"); + let dated = || { + response( + 200, + "", + &[ + ("last-modified", "Wed, 01 Jan 2025 00:00:00 GMT"), + ("cache-control", "no-store"), + ], + ) + }; + let mut first: Vec = (0..12).map(|n| format!("3.{n}.0")).collect(); + first.push("3.3.1".into()); + let first: Vec<&str> = first.iter().map(String::as_str).collect(); + let mut steps = VecDeque::from([(format!("{base}/maven-metadata.xml"), metadata(&first))]); + for version in &first { + if *version == "3.3.1" { + steps.push_back((pom(version), response(503, "", &[]))); + } else { + steps.push_back((pom(version), dated())); + } + } + let mut second = first.clone(); + second.push("3.3.2"); + second.push("3.3.3"); + steps.push_back((format!("{base}/maven-metadata.xml"), metadata(&second))); + steps.push_back((pom("3.3.1"), dated())); + steps.push_back((pom("3.3.2"), dated())); + steps.push_back((pom("3.3.3"), response(404, "", &[]))); + let steps = Arc::new(Steps(Mutex::new(steps))); + let dir = tempfile::tempdir().unwrap(); + let sources = client(steps.clone(), dir.path()).await; + let id = patchula_config::PackageId::public(Source::Maven, "com.example:lib"); + let catalog = sources.lookup(&id).await.unwrap(); + let at = |catalog: &patchula_datasource::Catalog, version: &str| { + catalog + .versions + .iter() + .find(|v| v.version == version) + .cloned() + }; + let failed = at(&catalog, "3.3.1").unwrap(); + assert!(failed.unchecked && failed.published_at.is_none()); + assert_eq!( + at(&catalog, "3.0.0").unwrap().published_at.as_deref(), + Some("2025-01-01T00:00:00Z") + ); + let catalog = sources.lookup(&id).await.unwrap(); + let retried = at(&catalog, "3.3.1").unwrap(); + assert!(!retried.unchecked && retried.published_at.is_some()); + assert!(at(&catalog, "3.3.2").unwrap().published_at.is_some()); + assert!(at(&catalog, "3.3.3").is_none()); assert!(steps.0.lock().unwrap().is_empty()); } diff --git a/crates/datasource/tests/versions_host.rs b/crates/datasource/tests/versions_host.rs index 7ea2fff0..cd3743a1 100644 --- a/crates/datasource/tests/versions_host.rs +++ b/crates/datasource/tests/versions_host.rs @@ -331,3 +331,97 @@ async fn releases_github_marks_as_prereleases_stay_out_whichever_source_answers( assert!(catalog.versions.iter().all(|v| v.digest.is_none())); } } + +/// The mirror dates tags only by their releases. A tag it leaves undated may +/// be annotated, so GitHub's own tag catalog answers instead, and dates it +/// by its tagger. +#[tokio::test] +async fn a_tag_the_mirror_leaves_undated_is_dated_through_github() { + struct Annotated(Mutex>); + impl Transport for Annotated { + fn send(&self, request: reqwest::Request) -> Exchange<'_> { + Box::pin(async move { + let path = request.url().path().to_owned(); + self.0.lock().unwrap().push(path.clone()); + let commit = "a".repeat(40); + let annotated = "f".repeat(40); + let body = if request.url().host_str() != Some("api.github.com") { + if path.ends_with("tags.json") { + json!({"tags":[{"tag":"v2.0.0","sha":commit},{"tag":"v2.1.0","sha":"b".repeat(40)}],"truncated":false}) + } else { + json!({"releases":[{"tag":"v2.0.0","date":"2026-01-01T00:00:00Z","prerelease":false}],"truncated":false}) + } + } else if path == "/graphql" { + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()) + .unwrap(); + if body["query"].as_str().unwrap().contains("releases(") { + json!({"data":{"r0":{"releases":{"nodes":[ + {"tagName":"v2.0.0","publishedAt":"2026-01-01T00:00:00Z","isDraft":false,"isPrerelease":false} + ],"pageInfo":{"hasNextPage":false,"endCursor":null}}}}}) + } else { + let mut objects = serde_json::Map::new(); + for i in 0..2 { + let sha = body["variables"][format!("s{i}")].as_str().unwrap(); + let node = if sha == commit { + json!({"oid":sha,"__typename":"Commit"}) + } else { + json!({"oid":sha,"__typename":"Tag","tagger":{"date":"2026-03-01T00:00:00Z"},"target":{"__typename":"Commit","oid":"b".repeat(40)}}) + }; + objects.insert(format!("t{i}"), node); + } + json!({"data":{"repository":objects}}) + } + } else { + assert_eq!(path, "/repos/FiloSottile/age/git/matching-refs/tags/"); + json!([ + {"ref":"refs/tags/v2.0.0","object":{"sha":commit}}, + {"ref":"refs/tags/v2.1.0","object":{"sha":annotated}} + ]) + }; + Ok(Response { + status: reqwest::StatusCode::OK, + headers: Default::default(), + body: body.to_string().into_bytes(), + stale: None, + }) + }) + } + } + let cache = memory().await; + let transport = Arc::new(Annotated(Mutex::new(Vec::new()))); + let github = patchula_github::Client::with_transport( + "offline", + cache.clone(), + Default::default(), + transport.clone(), + ) + .unwrap(); + let catalog = Client::new( + github, + patchula_datasource::cached(cache.clone(), transport.clone(), 4).unwrap(), + cache, + ) + .lookup(&patchula_config::PackageId::public( + Source::GithubTags, + "FiloSottile/age", + )) + .await + .unwrap(); + let date = |tag: &str| { + catalog + .versions + .iter() + .find(|v| v.version == tag) + .and_then(|v| v.published_at.clone()) + }; + assert_eq!(date("v2.1.0").as_deref(), Some("2026-03-01T00:00:00Z")); + assert_eq!(date("v2.0.0").as_deref(), Some("2026-01-01T00:00:00Z")); + let requests = transport.0.lock().unwrap(); + assert!(requests.iter().any(|path| path.ends_with("tags.json"))); + assert!( + requests + .iter() + .any(|path| path.ends_with("/matching-refs/tags/")) + ); +} diff --git a/crates/engine/src/catalogs.rs b/crates/engine/src/catalogs.rs index 37abd0fb..ed9ec9e8 100644 --- a/crates/engine/src/catalogs.rs +++ b/crates/engine/src/catalogs.rs @@ -36,7 +36,7 @@ enum CatalogAccess { } impl State for Saved<'_> { const NAMESPACE: &'static str = "catalog"; - const VERSION: u32 = 15; + const VERSION: u32 = 16; // Code-versioned: the view is datasource output decoded into `Catalog`; // a stale-but-decodable view from a release whose transforms moved is // worse than a miss. @@ -177,7 +177,7 @@ pub(crate) struct Controls { } impl State for Controls { const NAMESPACE: &'static str = "catalog-controls"; - const VERSION: u32 = 15; + const VERSION: u32 = 16; // Code-versioned: the stored keys come from `identity` and `plan` // code, and a key written by older code that no longer matches its // dependency quietly un-retries a ticked box. diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index aa5f3b66..0376e8ce 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -389,25 +389,25 @@ impl State for BranchHint { } impl State for RepoConfig { const NAMESPACE: &'static str = "config"; - const VERSION: u32 = 13; + const VERSION: u32 = 14; const CODE_VERSIONED: bool = true; type Scope = RepoConfigScope; } impl State for CachedExtraction { const NAMESPACE: &'static str = "extract"; - const VERSION: u32 = 16; + const VERSION: u32 = 17; const CODE_VERSIONED: bool = true; type Scope = ExtractionScope; } impl State for FileExtraction { const NAMESPACE: &'static str = "files"; - const VERSION: u32 = 16; + const VERSION: u32 = 17; const CODE_VERSIONED: bool = true; type Scope = FilesScope; } impl State for EvaluatedConfig { const NAMESPACE: &'static str = "config-files"; - const VERSION: u32 = 13; + const VERSION: u32 = 14; const CODE_VERSIONED: bool = true; type Scope = ConfigFilesScope; } diff --git a/crates/lookup/src/lib.rs b/crates/lookup/src/lib.rs index 96d31f0b..95f53e39 100644 --- a/crates/lookup/src/lib.rs +++ b/crates/lookup/src/lib.rs @@ -503,7 +503,8 @@ impl Evaluator { let Ok(parsed) = scheme.parse(&value) else { continue; }; - if !parsed.0.pre.is_empty() || !permitted_matches(effective.permitted, scheme, &value)? + if prerelease(&parsed, version) + || !permitted_matches(effective.permitted, scheme, &value)? { continue; } @@ -713,6 +714,8 @@ impl Evaluator { let scheme = Ecosystem::of(dep.scheme, dep.source); let mut held_back = Vec::new(); let mut not_proposed = Vec::new(); + // Releases held because their source could not be asked for a date. + let mut unread: Vec<(String, Update)> = Vec::new(); let mut eligible = |version: &Version, selected: &str, kind: Update, @@ -740,6 +743,12 @@ impl Evaluator { } else { effective.policy.wait_hours }; + // An unchecked release's date is unknown, not absent: it waits + // until a lookup reads it, never skips the wait. + if version.unchecked && wait > 0 { + unread.push((selected.into(), kind)); + return Ok(false); + } let Some(published_at) = version.published_at.as_deref().filter(|_| wait > 0) else { return Ok(true); }; @@ -850,9 +859,9 @@ impl Evaluator { scheme.parse(&normalize(&v.version)?).ok()?, )) }) - .filter(|(_, _, p)| { + .filter(|(v, _, p)| { scheme.order(p, &ceiling) != Ordering::Greater - && (p.0.pre.is_empty() || !ceiling.0.pre.is_empty()) + && (!prerelease(p, v) || !ceiling.0.pre.is_empty()) }) .collect(); candidates.sort_by(|a, b| candidate_order(scheme, &b.2, b.0, &a.2, a.0)); @@ -915,6 +924,12 @@ impl Evaluator { dep.name, dep.current ) })?; + // The current version is a prerelease by its syntax, or because its + // source marks it one. + let current_prerelease = !baseline.0.pre.is_empty() + || catalog.versions.iter().any(|v| { + v.prerelease && normalize(&v.version).as_deref() == Some(current.as_str()) + }); let policies = [Update::Major, Update::Minor, Update::Patch, Update::Pin] .into_iter() .map(|kind| self.effective(dep, kind).map(|effective| (kind, effective))) @@ -1014,11 +1029,11 @@ impl Evaluator { // pin. Widening its ceiling needs compatibility evidence. continue; } - // Prereleases require a prerelease current or an explicit permitted - // filter. This permits deliberate nightly rules without a global flag. - if !parsed.0.pre.is_empty() - && baseline.0.pre.is_empty() - && effective.permitted.is_none() + // Prereleases require a prerelease current or a permitted filter + // that asks for them; one that only narrows or excludes does not. + if prerelease(&parsed, version) + && !current_prerelease + && !asks_for_prerelease(effective.permitted, scheme, &normalized, &parsed)? { continue; } @@ -1055,7 +1070,7 @@ impl Evaluator { .expect("version classification") .1; if !effective.policy.active - || (!parsed.0.pre.is_empty() && baseline.0.pre.is_empty()) + || (prerelease(&parsed, version) && !current_prerelease) || (proposing.range == Range::Support && effective.policy.versions == VersionPolicy::Preserve) { @@ -1286,6 +1301,27 @@ impl Evaluator { }); held_back.sort(); held_back.dedup(); + // Likewise for releases held because their date could not be read. + not_proposed.extend( + unread + .into_iter() + .filter(|(version, kind)| { + let unread = scheme.parse(version).ok(); + !result + .iter() + .filter(|u| u.update_type == *kind) + .filter_map(|u| u.new_version.as_deref().and_then(|v| scheme.parse(v).ok())) + .any(|v| { + unread + .as_ref() + .is_some_and(|h| scheme.order(h, &v) != Ordering::Greater) + }) + }) + .map(|(version, _)| NotProposed { + version, + reason: "the registry did not answer for its release date, so it waits until a later lookup reads one".into(), + }), + ); not_proposed.sort(); not_proposed.dedup(); Ok(Evaluation { @@ -1452,6 +1488,51 @@ fn any(values: &[T], value: T) -> bool { values.is_empty() || values.contains(&value) } /// Whether `text` passes the `permitted` filter, when one is set. +/// Whether a release is a prerelease: by its version syntax, or because its +/// source marks it one (a GitHub release flagged as a prerelease). +fn prerelease(parsed: &Parsed, version: &Version) -> bool { + !parsed.0.pre.is_empty() || version.prerelease +} +/// Whether `permitted` deliberately admits the prerelease `text` (see +/// `Filter::asks_for_prerelease`). Without a filter, nothing asks. +fn asks_for_prerelease( + permitted: Option<&Filter>, + scheme: Ecosystem, + text: &str, + parsed: &Parsed, +) -> Result { + let Some(filter) = permitted else { + return Ok(false); + }; + filter.asks_for_prerelease(scheme, text, &without_prerelease(scheme, text, parsed)) +} +/// `text` without its prerelease part: the longest prefix, cut before a +/// separator or where digits and letters meet, that parses as a release +/// (`2.2.0` for `2.2.0-rc.1`, `3.0` for `3.0rc1`, `1.5` for `1.5-beta1`). +/// A version that is a prerelease only by its source's flag, or one with +/// no such prefix, is its own release. +fn without_prerelease(scheme: Ecosystem, text: &str, parsed: &Parsed) -> String { + if parsed.0.pre.is_empty() { + return text.to_owned(); + } + let chars: Vec<(usize, char)> = text.char_indices().collect(); + for i in (1..chars.len()).rev() { + let (at, c) = chars[i]; + let previous = chars[i - 1].1; + let cut = matches!(c, '-' | '.' | '_' | '+') + || (c.is_ascii_digit() != previous.is_ascii_digit() + && c.is_ascii_alphanumeric() + && previous.is_ascii_alphanumeric()); + if !cut { + continue; + } + let prefix = text[..at].trim_end_matches(['-', '.', '_', '+']); + if !prefix.is_empty() && scheme.parse(prefix).is_ok_and(|p| p.0.pre.is_empty()) { + return prefix.to_owned(); + } + } + text.to_owned() +} fn permitted_matches(permitted: Option<&Filter>, scheme: Ecosystem, text: &str) -> Result { match permitted { Some(filter) => filter.matches(scheme, text), diff --git a/crates/lookup/tests/advisories.rs b/crates/lookup/tests/advisories.rs index b5289fea..7ce11696 100644 --- a/crates/lookup/tests/advisories.rs +++ b/crates/lookup/tests/advisories.rs @@ -331,6 +331,8 @@ fn a_fix_skips_the_release_age_wait_and_group_but_other_releases_still_wait() { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }) .collect(), dist_tags: Default::default(), @@ -390,6 +392,8 @@ fn the_lowest_release_fixing_the_most_is_proposed_past_permitted_filters() { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }) .collect(), dist_tags: Default::default(), @@ -549,6 +553,8 @@ fn a_fix_bypasses_a_committed_cargo_cooldown_and_the_proposal_says_so() { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }; let catalog = patchula_datasource::Catalog { versions: vec![ @@ -671,6 +677,8 @@ fn a_transitive_fix_inside_the_cooldown_is_marked_the_same_way() { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }, patchula_datasource::Version { version: "0.3.23".into(), @@ -678,6 +686,8 @@ fn a_transitive_fix_inside_the_cooldown_is_marked_the_same_way() { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }, ], dist_tags: Default::default(), @@ -744,6 +754,8 @@ fn a_transitive_crates_catalog_only_dates_its_fix() { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }; let catalog = patchula_datasource::Catalog { versions: vec![ @@ -794,6 +806,8 @@ fn a_transitive_fix_waits_out_security_applys_wait() { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }; let catalog = |ago: i64| patchula_datasource::Catalog { versions: vec![version("0.3.20", 4000), version("0.3.23", ago)], @@ -846,6 +860,8 @@ fn a_transitive_fix_inside_a_committed_cooldown_still_waits_for_security_apply() digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }; let catalog = |ago: i64| patchula_datasource::Catalog { versions: vec![version("0.3.20", 4000), version("0.3.23", ago)], diff --git a/crates/lookup/tests/corpus.rs b/crates/lookup/tests/corpus.rs index 74101e05..bb1b29bd 100644 --- a/crates/lookup/tests/corpus.rs +++ b/crates/lookup/tests/corpus.rs @@ -110,6 +110,8 @@ fn ladder(dep: &Dependency) -> Catalog { digest: Some(hash(format!("{}:next-revision", dep.package).as_bytes())), deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }], dist_tags: Default::default(), }; @@ -227,6 +229,8 @@ fn ladder(dep: &Dependency) -> Catalog { digest: Some(digest), deprecated: false, yanked: rung.yanked, + prerelease: false, + unchecked: false, }); } } @@ -257,6 +261,8 @@ fn ladder(dep: &Dependency) -> Catalog { digest: Some(hash(version.as_bytes())), deprecated: false, yanked: false, + prerelease: false, + unchecked: false, })); catalog } diff --git a/crates/lookup/tests/coverage.rs b/crates/lookup/tests/coverage.rs index 7785c459..bebfa928 100644 --- a/crates/lookup/tests/coverage.rs +++ b/crates/lookup/tests/coverage.rs @@ -49,6 +49,8 @@ fn every_example_has_implemented_dispatch_for_the_twelve_repo_matrix() { published_at: None, digest: None, yanked: false, + prerelease: false, + unchecked: false, deprecated: false, }); evaluator diff --git a/crates/lookup/tests/facts.rs b/crates/lookup/tests/facts.rs index e8d3142d..5e85d543 100644 --- a/crates/lookup/tests/facts.rs +++ b/crates/lookup/tests/facts.rs @@ -34,6 +34,8 @@ fn catalog(values: &[&str]) -> Catalog { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }) .collect(), dist_tags: Default::default(), diff --git a/crates/lookup/tests/pinned.rs b/crates/lookup/tests/pinned.rs index 775c6dd0..c679080b 100644 --- a/crates/lookup/tests/pinned.rs +++ b/crates/lookup/tests/pinned.rs @@ -34,6 +34,8 @@ fn catalog(values: &[&str]) -> Catalog { digest: Some("a".repeat(40)), deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }) .collect(), dist_tags: Default::default(), diff --git a/crates/lookup/tests/policy.rs b/crates/lookup/tests/policy.rs index 4b61db21..d6132ed2 100644 --- a/crates/lookup/tests/policy.rs +++ b/crates/lookup/tests/policy.rs @@ -32,6 +32,8 @@ fn catalog(values: &[&str]) -> Catalog { digest: Some("a".repeat(40)), deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }) .collect(), dist_tags: Default::default(), @@ -1335,6 +1337,8 @@ fn flake_inputs_get_digest_updates_after_the_release_age_wait() { digest: Some(digest), deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }], dist_tags: Default::default(), }; @@ -1596,6 +1600,8 @@ fn lock_only_updates_keep_the_wait_the_permitted_filter_and_exact_policy() { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }); let result = Evaluator::new(&filtered, Default::default()) .unwrap() @@ -1960,6 +1966,8 @@ fn a_tagged_action_with_a_short_tag_resolves_through_its_commit_digest() { digest: Some(digest.into()), deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }; let cat = Catalog { versions: vec![ @@ -2192,3 +2200,128 @@ fn a_rule_error_is_the_wait_error_never_a_wait_of_zero() { // crates.io. assert_eq!(eval.lockfile_wait_hours(&[dependency("left-pad")]), Ok(0)); } + +/// The versions `permitted` lets through for `dep` on `releases`, sorted, +/// with no release-age wait. +fn permitted_targets(dep: &Dependency, filter: &str, releases: &[&str]) -> Vec { + let mut cfg = config("renovate-migration"); + cfg.defaults.wait_hours = 0; + cfg.defaults.permitted = Some(filter.into()); + let eval = Evaluator::new(&cfg, Default::default()).unwrap(); + let mut targets: Vec<_> = eval + .propose(dep, &catalog(releases)) + .unwrap() + .into_iter() + .filter_map(|p| p.new_version) + .collect(); + targets.sort(); + targets.dedup(); + targets +} + +/// A `permitted` filter that only narrows or excludes keeps prereleases +/// out; one that names a prerelease, by a regex that matches the +/// prerelease part or a range naming a prerelease, lets them in. +#[test] +fn only_a_permitted_filter_that_asks_for_prereleases_admits_them() { + let mut npm = dependency("lib"); + npm.current = "2.1.0".into(); + npm.resolved = Some("2.1.0".into()); + let releases = &["2.1.0", "2.2.0-rc.1", "3.0.0"]; + assert!(permitted_targets(&npm, "!regex:^3\\.", releases).is_empty()); + assert!(permitted_targets(&npm, "regex:^2\\.", releases).is_empty()); + assert!(permitted_targets(&npm, "!semver:>=3", releases).is_empty()); + assert_eq!( + permitted_targets(&npm, "regex:-rc\\.", releases), + ["2.2.0-rc.1"] + ); + assert_eq!( + permitted_targets(&npm, "semver:>=2.2.0-rc.1", releases), + ["2.2.0-rc.1", "3.0.0"] + ); + + let mut maven = dependency("org.example:lib"); + maven.manager = Manager::Maven; + maven.source = Source::Maven; + maven.scheme = Scheme::Maven; + maven.current = "1.4".into(); + maven.resolved = Some("1.4".into()); + let releases = &["1.4", "1.5-beta1", "2.0"]; + assert!(permitted_targets(&maven, "semver:[1,2)", releases).is_empty()); + assert_eq!( + permitted_targets(&maven, "semver:[1.5-beta1,2)", releases), + ["1.5-beta1"] + ); + assert_eq!( + permitted_targets(&maven, "regex:-beta", releases), + ["1.5-beta1"] + ); + + let mut pypi = dependency("lib"); + pypi.manager = Manager::Pyproject; + pypi.source = Source::Pypi; + pypi.scheme = Scheme::Pep440; + pypi.current = "==2.1.0".into(); + pypi.resolved = Some("2.1.0".into()); + let releases = &["2.1.0", "2.2.0rc1", "3.0.0"]; + assert!(permitted_targets(&pypi, "semver:<3", releases).is_empty()); + assert_eq!( + permitted_targets(&pypi, "semver:>=2.2.0rc1,<3", releases), + ["2.2.0rc1"] + ); +} + +/// A release its source marks as a prerelease (mise passing on GitHub's +/// flag) is one, even when its version looks stable. +#[test] +fn a_release_the_source_marks_as_a_prerelease_is_held_like_one() { + let eval = Evaluator::new(&config("renovate-migration"), Default::default()).unwrap(); + let mut releases = catalog(&["1.0.0", "2.0.0"]); + releases.versions[1].prerelease = true; + let proposals = eval.propose(&prettier_tool("1.0.0"), &releases).unwrap(); + assert!(proposals.is_empty(), "{proposals:?}"); + // From a flagged current release, a newer flagged one is offered. + let mut releases = catalog(&["1.0.0", "2.0.0", "2.1.0"]); + releases.versions[1].prerelease = true; + releases.versions[2].prerelease = true; + let proposals = eval.propose(&prettier_tool("2.0.0"), &releases).unwrap(); + assert_eq!(proposals.len(), 1, "{proposals:?}"); + assert_eq!(proposals[0].new_version.as_deref(), Some("2.1.0")); +} + +/// A release whose date the source could not be asked for waits instead of +/// counting as undated, and is reported; with no wait it is proposed. +#[test] +fn an_unchecked_release_waits_instead_of_skipping_the_release_age_wait() { + let mut dep = dependency("lib"); + dep.current = "1.0.0".into(); + dep.resolved = Some("1.0.0".into()); + let mut releases = catalog(&["1.0.0", "1.0.1", "1.0.2"]); + releases.versions[1].published_at = Some("2026-01-01T00:00:00Z".into()); + releases.versions[2].unchecked = true; + let eval = Evaluator::new(&config("renovate-migration"), Default::default()).unwrap(); + let evaluation = eval + .evaluate_at(&dep, &releases, instant("2026-10-12T00:00:00Z")) + .unwrap(); + let targets: Vec<_> = evaluation + .updates + .iter() + .filter_map(|p| p.new_version.as_deref()) + .collect(); + assert_eq!(targets, ["1.0.1"]); + assert_eq!(evaluation.not_proposed.len(), 1); + assert_eq!(evaluation.not_proposed[0].version, "1.0.2"); + let mut cfg = config("renovate-migration"); + cfg.defaults.wait_hours = 0; + let evaluation = Evaluator::new(&cfg, Default::default()) + .unwrap() + .evaluate_at(&dep, &releases, instant("2026-10-12T00:00:00Z")) + .unwrap(); + assert!( + evaluation + .updates + .iter() + .any(|p| p.new_version.as_deref() == Some("1.0.2")) + ); + assert!(evaluation.not_proposed.is_empty()); +} diff --git a/crates/tools/src/lockfiles/tests/real_sparse_registry.rs b/crates/tools/src/lockfiles/tests/real_sparse_registry.rs index 34bf195a..7ae6b0f5 100644 --- a/crates/tools/src/lockfiles/tests/real_sparse_registry.rs +++ b/crates/tools/src/lockfiles/tests/real_sparse_registry.rs @@ -132,6 +132,8 @@ impl SparseRegistry { digest: None, deprecated: false, yanked: false, + prerelease: false, + unchecked: false, }) .collect() }) diff --git a/docs/src/content/docs/guide/configuration.md b/docs/src/content/docs/guide/configuration.md index d95df4f6..9ea794c7 100644 --- a/docs/src/content/docs/guide/configuration.md +++ b/docs/src/content/docs/guide/configuration.md @@ -169,7 +169,7 @@ lockfile is not an update type; it is configured under | `releaseNotes`, `bodyNotes` | release notes in the PR body, and extra lines of your own | | `waitHours` | release-age wait in whole hours; `0` turns it off | | `windows` | when proposals may be created | -| `permitted` | a version filter targets must match, such as `"semver:^24"`; an explicit filter also allows prereleases | +| `permitted` | a version filter targets must match, such as `"semver:^24"`; a filter that names a prerelease also allows prereleases ([prereleases](/guide/lookup-policy/)) | Windows are five-field cron expressions in an IANA timezone; weekdays count 0 or 7 for Sunday and 1 for Monday. Each time an expression names opens a turn: diff --git a/docs/src/content/docs/guide/lookup-policy.md b/docs/src/content/docs/guide/lookup-policy.md index b61db799..9293432b 100644 --- a/docs/src/content/docs/guide/lookup-policy.md +++ b/docs/src/content/docs/guide/lookup-policy.md @@ -7,6 +7,22 @@ proposes the newest version that is permitted, not deprecated, not yanked and old enough. Prereleases are only considered when the current version is already a prerelease, or when a rule's `permitted` filter asks for them. +A release is a prerelease when its version says so (`2.0.0-rc.1`, `3.0rc1`, +`1.5-beta1`), or when its source marks it one: a GitHub release flagged as a +prerelease, listed through mise. A `permitted` filter asks for a prerelease +only on purpose; one that just narrows or excludes versions does not: + +- A `regex:` filter asks when it matches the prerelease but not the same + version without its prerelease part, so the prerelease part is what + matched: `regex:-rc\.` and `regex:nightly` ask, `regex:^2\.` does not. + A version that is a prerelease only by its source's flag has no + prerelease part, so no `regex:` filter asks for it. +- A `semver:` range asks when it names a prerelease version itself and + the prerelease is inside it: `semver:>=3.0.0-rc.1` or `semver:[1.0-beta1,2)` ask, + `semver:^2` and `semver:[1,2)` do not. +- A negated filter (`!regex:...`, `!semver:...`) only excludes and never + asks. + The current version comes from the lockfile when there is one. An npm range on its own only tells the Count which class an update belongs to; he never pretends the range's floor is what you have installed. @@ -18,7 +34,13 @@ with a publication date becomes eligible that many hours after it was published. Brand-new releases are where broken and malicious versions tend to show up, and a day is usually enough for someone else to notice. -- Undated releases have no wait. `waitHours = 0` turns the wait off. +- Releases the registry gives no date have no wait. `waitHours = 0` turns + the wait off. +- A release whose date the Count could not read, because the request for it + failed or the registry has not described it yet, waits until a later + lookup reads one; the report lists it as a notice. Go, Maven and NuGet + date every version, not just the newest, so a new patch release on an + older line waits like any other. - [Reference data](/guide/configuration/#reference-data), such as CA root bundles, has no wait by default. - Versions you already use, and versions pinned in lockfiles, are never held diff --git a/docs/src/content/docs/guide/sources.md b/docs/src/content/docs/guide/sources.md index 6d8c7bd6..62b836a6 100644 --- a/docs/src/content/docs/guide/sources.md +++ b/docs/src/content/docs/guide/sources.md @@ -9,7 +9,7 @@ manager, and rules can select by it. |---|---|---| | `npm` | package name | the npm registry, full metadata for publication times, dist-tags and deprecations | | `crates` | crate name | the crates.io sparse index, with publication times; yanked releases are skipped | -| `go` | module path | the Go module proxy (`@v/list`, `.info` for new versions only) | +| `go` | module path | the Go module proxy (`@v/list`, and `.info` once for each version it has not dated yet) | | `pypi` | normalised name | the PyPI JSON API, falling back to the PEP 691 simple API | | `docker` | image | registry v2 tags and manifests on Docker Hub, GHCR and other registries | | `oci` | `registry/chart` | OCI registry tags for Helm charts | @@ -19,8 +19,8 @@ manager, and rules can select by it. | `github-runners` | runner family | a catalog of GitHub-hosted runner labels | | `forge-tags` | `host/path` | GitLab and Gitea/Forgejo tags, for pre-commit hooks and GitLab CI includes and components | | `mise` | tool name, as written in `mise.toml` | `mise ls-remote --json`, run by the Count's pinned mise ([mise lookups](#mise-lookups)) | -| `maven` | `group:artifact` | `maven-metadata.xml` on Maven Central, Google Maven or the Gradle plugin portal | -| `nuget` | package ID | the nuget.org flat container and registration pages | +| `maven` | `group:artifact` | `maven-metadata.xml` on Maven Central, Google Maven or the Gradle plugin portal; each POM's `Last-Modified`, read once per version, dates it | +| `nuget` | package ID | the nuget.org flat container and every registration page, for dates and unlisted and deprecated flags | | `rubygems` | gem name | the RubyGems versions API | | `packagist` | `vendor/name` | Packagist's p2 metadata | @@ -60,7 +60,9 @@ version, its date, whether it is a prerelease and, for GitHub-hosted tools, a release URL. Backends leave out what they do not know: Node lists a date only, and some `go:` versions have no date and so no release-age wait. Prereleases are listed, and the Count's [prerelease rules](/guide/lookup-policy/) decide -whether to offer them; mise's `prerelease` flag does not filter. +whether to offer them. A version mise marks `prerelease`, such as a GitHub +release flagged as one, counts as a prerelease even when it looks stable, just +as it does for `github-releases`. The Count runs mise the way he runs the lockfile tools, and more strictly: @@ -176,7 +178,10 @@ configuration. By default (`useVersionsHost = true`), public GitHub repositories that appear in mise's registry are read from the public [mise-versions](https://mise-versions.jdx.dev) mirror instead of the GitHub API: -`releases.json` and `tags.json`, validated by ETag. Repositories outside the +`releases.json` and `tags.json`, validated by ETag. The mirror dates a tag only +by its release, so when a `github-tags` lookup finds a tag without one, which +may be an annotated tag with a tagger date, that repository's tags are read +from GitHub instead. Repositories outside the registry, including private ones, go straight to GitHub. If the mirror fails with a transport error, 429 or 5xx, the Count falls back to GitHub; a 403, 404 or empty list does not, and a truncated list is reported as an incomplete lookup. diff --git a/docs/src/content/docs/internals/cache.md b/docs/src/content/docs/internals/cache.md index 04746639..5129470b 100644 --- a/docs/src/content/docs/internals/cache.md +++ b/docs/src/content/docs/internals/cache.md @@ -132,6 +132,13 @@ inserted version to `merge_release_page` as `after`, which tells a refresh prefix apart from initial pagination. The newest item is also kept in `release_heads`. +Sources that date one version per request (Go's `.info`, a Maven POM's +`Last-Modified`) keep each answer as `release-dates` state, per package and, +for Maven, per operator registry entry: a version's date, or that the +registry cannot serve it. A lookup asks only about listed versions with no +answer yet, wherever they sort; a failed request stores nothing, so the +version stays unchecked and the next lookup asks again. + Typed values the Count keeps between runs (hints, extractions, catalog views, receipts and the like) live in `state`; the server queue, installations and repositories keep their own tables. A state row is one value, keyed by a diff --git a/tests/derived-versions.tsv b/tests/derived-versions.tsv index 77f26d9a..dd15a142 100644 --- a/tests/derived-versions.tsv +++ b/tests/derived-versions.tsv @@ -22,26 +22,26 @@ # that every listed package is in Cargo.lock, so a typo fails. # # name version sources deps sha256 -npm-catalog-v2 1 crates/datasource/src/compact.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/http/src/derive.rs nodejs-semver,semver,serde,serde_json b43cb926b446d69581b66af1a582849e63725a213ec09aa4ffbe1b686182bcd5 -crates-catalog-v2 1 crates/datasource/src/compact.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json d01b9d453bb9357758141cb182dd33f5e524b808cb709473404b0b12615c50e0 -pypi-catalog-v2 1 crates/datasource/src/compact.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/versioning/src/extended.rs,crates/http/src/derive.rs pep440_rs,semver,serde,serde_json 0d0165e6d105c04d536a4d34e3cc7ae9b640ad7f25cffb8a754295c5dde56dab -terraform-catalog 3 crates/datasource/src/registries.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/versioning/src/extended.rs,crates/http/src/derive.rs pep440_rs,semver,serde,serde_json f0fec4e0f298a10ed8faa19915c663f347e27279572bfbcc157d2c73434287cb -helm-index 3 crates/datasource/src/helm.rs,crates/datasource/src/registries.rs,crates/datasource/src/catalog.rs,crates/datasource/src/upstream.rs,crates/versioning/src/lib.rs,crates/http/src/derive.rs libyaml-safer,nodejs-semver,semver,serde,serde_json ce3bdd17c7d5893229a489196948798676811ebec32c614c6c7de550763a86b2 -pypi-index-catalog 3 crates/datasource/src/registries.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/versioning/src/extended.rs,crates/http/src/derive.rs pep440_rs,semver,serde,serde_json f0fec4e0f298a10ed8faa19915c663f347e27279572bfbcc157d2c73434287cb -pypi-simple-catalog 3 crates/datasource/src/registries.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/versioning/src/extended.rs,crates/http/src/derive.rs pep440_rs,semver,serde,serde_json f0fec4e0f298a10ed8faa19915c663f347e27279572bfbcc157d2c73434287cb -nuget-flat 1 crates/datasource/src/nuget.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json bc42a5545769095282a0318d76ecf81fdc6575d3989e70e89f25367af67fac3a -nuget-registration 1 crates/datasource/src/nuget.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json bc42a5545769095282a0318d76ecf81fdc6575d3989e70e89f25367af67fac3a -nuget-registration-page 1 crates/datasource/src/nuget.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json bc42a5545769095282a0318d76ecf81fdc6575d3989e70e89f25367af67fac3a -maven-metadata 1 crates/datasource/src/maven.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json bbabbf88f0c0066693e051f39c4e0d11601a98a3336ebb5d090619b5a3ba78cc -packagist-p2 1 crates/datasource/src/packagist.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json 7bfa0d811a18ecf511a6820778224e0158687e4098cf5c49e6c6b822ca231f16 -rubygems-versions 1 crates/datasource/src/rubygems.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json 991f4be0f35501aafd852b1bd9c4686c269500a7accf0cf80c79ef520c454ef1 -forge-gitlab-tags 1 crates/datasource/src/forge.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json 6ed4867515d816f3986119b17d4e9fc207bacdb7db0dc715e595ff523fe9c25d -forge-gitea-tags 1 crates/datasource/src/forge.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json 6ed4867515d816f3986119b17d4e9fc207bacdb7db0dc715e595ff523fe9c25d +npm-catalog-v2 2 crates/datasource/src/compact.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/http/src/derive.rs nodejs-semver,semver,serde,serde_json 5565914c82bf17674e652500ea8b6ae48d6b4d7841704ac23f0e7cc89856baa0 +crates-catalog-v2 2 crates/datasource/src/compact.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json 61fa893d3ba59363121022d17e3fd2b2a0e29c3081afaad21b49c41beb413c4b +pypi-catalog-v2 2 crates/datasource/src/compact.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/versioning/src/extended.rs,crates/http/src/derive.rs pep440_rs,semver,serde,serde_json 33a2e9d47c1b166717f255a286cc2b0387fe7e9a93ad2f39f63f1fdd1db023cb +terraform-catalog 4 crates/datasource/src/registries.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/versioning/src/extended.rs,crates/http/src/derive.rs pep440_rs,semver,serde,serde_json a060e3369eaf350a758636da9bfdddc5448c6a0aada1ff9702b7f79dad286db9 +helm-index 4 crates/datasource/src/helm.rs,crates/datasource/src/registries.rs,crates/datasource/src/catalog.rs,crates/datasource/src/upstream.rs,crates/versioning/src/lib.rs,crates/http/src/derive.rs libyaml-safer,nodejs-semver,semver,serde,serde_json c459e22af35a68cbb76f0c066ec2043b42942af56990f3b4858240537cddfa2f +pypi-index-catalog 4 crates/datasource/src/registries.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/versioning/src/extended.rs,crates/http/src/derive.rs pep440_rs,semver,serde,serde_json a060e3369eaf350a758636da9bfdddc5448c6a0aada1ff9702b7f79dad286db9 +pypi-simple-catalog 4 crates/datasource/src/registries.rs,crates/datasource/src/catalog.rs,crates/versioning/src/lib.rs,crates/versioning/src/extended.rs,crates/http/src/derive.rs pep440_rs,semver,serde,serde_json a060e3369eaf350a758636da9bfdddc5448c6a0aada1ff9702b7f79dad286db9 +nuget-flat 2 crates/datasource/src/nuget.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json dc59e8569e8c7f3060305af841ec4c42dc078fcfc6ab34eff90657eead4a8e43 +nuget-registration 2 crates/datasource/src/nuget.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json dc59e8569e8c7f3060305af841ec4c42dc078fcfc6ab34eff90657eead4a8e43 +nuget-registration-page 2 crates/datasource/src/nuget.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json dc59e8569e8c7f3060305af841ec4c42dc078fcfc6ab34eff90657eead4a8e43 +maven-metadata 2 crates/datasource/src/maven.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json 1761448693fc2d552b5212ebe39caaa2bc041f22588ddfa54efb8bbf61f9ff2d +packagist-p2 2 crates/datasource/src/packagist.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json a3cf217ba3ada0e02ae7d86382095864e685db191b48d8ba4d927fabb49c9b0c +rubygems-versions 2 crates/datasource/src/rubygems.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json c49815987ca020561c6897137aaa2a053ccb22cbfa733e7daff87fc4d0dcc0a7 +forge-gitlab-tags 2 crates/datasource/src/forge.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json de1b16d97cecc11fcf452e427d4fedb3cf436a0a171186014bc4fb53fc77cf0f +forge-gitea-tags 2 crates/datasource/src/forge.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json de1b16d97cecc11fcf452e427d4fedb3cf436a0a171186014bc4fb53fc77cf0f oci-config-notes 1 crates/datasource/src/docker.rs,crates/datasource/src/upstream.rs,crates/http/src/derive.rs serde,serde_json 19b21631041de28b28ba16a6ab6f52dfae0e8336077fde4a9fde94f755fe8dc7 oci-tags 1 crates/datasource/src/docker.rs,crates/http/src/derive.rs serde,serde_json 1793d5ba9d90c2f7d4f7cc78cbdb2bf199bc463dbfa454c8296e3e13f6fa1808 -image-platform 3 crates/datasource/src/images.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json,sha2 27914c918d53cef3cd1fb7b1cb4882cb189b3658e2f719f07fe88ad208fdbe5e -sbom-os 3 crates/datasource/src/images.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs base64,serde,serde_json,sha2 27914c918d53cef3cd1fb7b1cb4882cb189b3658e2f719f07fe88ad208fdbe5e -github-release-head 5 crates/datasource/src/github.rs,crates/http/src/derive.rs serde,serde_json 97c8ecc48281b59d357a184d05b6528ba08417a939eba7e9a43875f3e74b600c +image-platform 4 crates/datasource/src/images.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs serde,serde_json,sha2 1c9dde1bd7781e14e0a02afe2256d2fad4044258a7a12ba9fe708422b81bd4d6 +sbom-os 4 crates/datasource/src/images.rs,crates/datasource/src/catalog.rs,crates/http/src/derive.rs base64,serde,serde_json,sha2 1c9dde1bd7781e14e0a02afe2256d2fad4044258a7a12ba9fe708422b81bd4d6 +github-release-head 6 crates/datasource/src/github.rs,crates/http/src/derive.rs serde,serde_json ebec9a065bad0efc69cf90fa79b64c3fd40ea93fe498c572b184f760bbfe1320 anonymous-registry-pull-token 1 crates/datasource/src/docker.rs chrono,serde,serde_json e8d7be4142dd9b7f394b4f633656f200b7003bd1fec689a051750576ef313cd3 notes-adapter-listing 7 crates/notes/src/adapters.rs,crates/http/src/derive.rs serde,serde_json 51afbd233292614a4563542f50198d2340ea23efe6128cb2cfb4d84de36f9d67 parsed-postgres-sgml 7 crates/notes/src/adapters.rs,crates/notes/src/changelog.rs,crates/http/src/derive.rs base64,serde,serde_json 0f919d21d51d698e46b47a0b76386d90295027ee1d5442efc3e315917d5cff95 diff --git a/tests/state-exempt.tsv b/tests/state-exempt.tsv index a97cb277..ca96f7d2 100644 --- a/tests/state-exempt.tsv +++ b/tests/state-exempt.tsv @@ -24,4 +24,5 @@ nix-prefetch refetched inside the one-hour freshness window osv-listing refetched within the advisory freshness window (`http.advisoryFreshnessSecs`, default 3600 s; the window bounds how long a listing derived by an older build is served) quiet-run the run's head, report hash and next-due instant — a stale hash fails toward a full run receipts-prefix names the prefix the receipts were saved under — a fact +release-dates a registry's answer about one immutable release (its date, or that it cannot be served) — a fact verified-write-receipts receipts of what a write proved, not derived bytes diff --git a/tests/state-versions.tsv b/tests/state-versions.tsv index 9d7d72d0..0a725c23 100644 --- a/tests/state-versions.tsv +++ b/tests/state-versions.tsv @@ -32,17 +32,17 @@ # and on a flag that disagrees with the table. # # name version sources deps sha256 -extract 16 crates/engine/src/lib.rs,crates/engine/src/repository.rs,crates/manager/src/,crates/mise/src/,crates/mise/registry.json,crates/config/src/,crates/words/src/,crates/versioning/src/,crates/update/src/plan.rs,crates/update/src/tooling.rs,PatchulaConfig.pkl,crates/github/src/model.rs,crates/github/src/blobs.rs,crates/engine/src/repository-head.graphql,crates/github/graphql/schema.docs.graphql chrono,chrono-tz,cron,globset,hcl-edit,jsonc-parser,nodejs-semver,pep440_rs,percent-encoding,pklr,regex,roxmltree,semver,serde,serde_json,serde_yaml_ng,sha2,toml_edit,url,yaml-rust2 5fd6e843cd12aa32fcc5b400df1fb01b51743b8a5b5bad9fe50abd388054ff6a -files 16 crates/engine/src/lib.rs,crates/engine/src/repository.rs,crates/manager/src/,crates/mise/src/,crates/mise/registry.json,crates/config/src/,crates/words/src/,crates/versioning/src/,crates/update/src/plan.rs,crates/update/src/tooling.rs,PatchulaConfig.pkl,crates/github/src/model.rs,crates/github/src/blobs.rs,crates/engine/src/repository-head.graphql,crates/github/graphql/schema.docs.graphql chrono,chrono-tz,cron,globset,hcl-edit,jsonc-parser,nodejs-semver,pep440_rs,percent-encoding,pklr,regex,roxmltree,semver,serde,serde_json,serde_yaml_ng,sha2,toml_edit,url,yaml-rust2 5fd6e843cd12aa32fcc5b400df1fb01b51743b8a5b5bad9fe50abd388054ff6a -config 13 crates/engine/src/lib.rs,crates/engine/src/repository.rs,crates/manager/src/,crates/mise/src/,crates/mise/registry.json,crates/config/src/,crates/words/src/,crates/versioning/src/,crates/update/src/plan.rs,crates/update/src/tooling.rs,PatchulaConfig.pkl,crates/github/src/model.rs,crates/github/src/blobs.rs,crates/engine/src/repository-head.graphql,crates/github/graphql/schema.docs.graphql chrono,chrono-tz,cron,globset,hcl-edit,jsonc-parser,nodejs-semver,pep440_rs,percent-encoding,pklr,regex,roxmltree,semver,serde,serde_json,serde_yaml_ng,sha2,toml_edit,url,yaml-rust2 5fd6e843cd12aa32fcc5b400df1fb01b51743b8a5b5bad9fe50abd388054ff6a -config-files 13 crates/engine/src/lib.rs,crates/engine/src/repository.rs,crates/manager/src/,crates/mise/src/,crates/mise/registry.json,crates/config/src/,crates/words/src/,crates/versioning/src/,crates/update/src/plan.rs,crates/update/src/tooling.rs,PatchulaConfig.pkl,crates/github/src/model.rs,crates/github/src/blobs.rs,crates/engine/src/repository-head.graphql,crates/github/graphql/schema.docs.graphql chrono,chrono-tz,cron,globset,hcl-edit,jsonc-parser,nodejs-semver,pep440_rs,percent-encoding,pklr,regex,roxmltree,semver,serde,serde_json,serde_yaml_ng,sha2,toml_edit,url,yaml-rust2 5fd6e843cd12aa32fcc5b400df1fb01b51743b8a5b5bad9fe50abd388054ff6a -catalog 15 crates/engine/src/catalogs.rs,crates/datasource/src/,crates/versioning/src/,crates/words/src/,crates/engine/src/lib.rs,crates/config/src/package.rs,crates/http/src/derive.rs,crates/http/src/cached.rs,crates/http/src/tracked.rs base64,chrono,nodejs-semver,pep440_rs,regex,semver,serde,serde_json,serde_yaml_ng,sha2 d473ce037f4e12fe96e9d6eada1d244f6692d87af67be32c80ecad11d59cd8e4 -catalog-controls 15 crates/engine/src/catalogs.rs,crates/update/src/identity.rs,crates/update/src/plan.rs,crates/update/src/plan/,crates/report/src/lib.rs,crates/manager/src/lib.rs,crates/words/src/,crates/engine/src/,crates/manager/src/,crates/config/src/,crates/lookup/src/,crates/lockfiles/src/ serde,serde_json,sha2 20bc743701bf12a1269dfc7f9ca0127f0a722178a0bdc3a2e773481f604734c9 +extract 17 crates/engine/src/lib.rs,crates/engine/src/repository.rs,crates/manager/src/,crates/mise/src/,crates/mise/registry.json,crates/config/src/,crates/words/src/,crates/versioning/src/,crates/update/src/plan.rs,crates/update/src/tooling.rs,PatchulaConfig.pkl,crates/github/src/model.rs,crates/github/src/blobs.rs,crates/engine/src/repository-head.graphql,crates/github/graphql/schema.docs.graphql chrono,chrono-tz,cron,globset,hcl-edit,jsonc-parser,nodejs-semver,pep440_rs,percent-encoding,pklr,regex,roxmltree,semver,serde,serde_json,serde_yaml_ng,sha2,toml_edit,url,yaml-rust2 28c309358f41554ace8bc0984edbb190ce004794606fb01592cb0d44dfc98046 +files 17 crates/engine/src/lib.rs,crates/engine/src/repository.rs,crates/manager/src/,crates/mise/src/,crates/mise/registry.json,crates/config/src/,crates/words/src/,crates/versioning/src/,crates/update/src/plan.rs,crates/update/src/tooling.rs,PatchulaConfig.pkl,crates/github/src/model.rs,crates/github/src/blobs.rs,crates/engine/src/repository-head.graphql,crates/github/graphql/schema.docs.graphql chrono,chrono-tz,cron,globset,hcl-edit,jsonc-parser,nodejs-semver,pep440_rs,percent-encoding,pklr,regex,roxmltree,semver,serde,serde_json,serde_yaml_ng,sha2,toml_edit,url,yaml-rust2 28c309358f41554ace8bc0984edbb190ce004794606fb01592cb0d44dfc98046 +config 14 crates/engine/src/lib.rs,crates/engine/src/repository.rs,crates/manager/src/,crates/mise/src/,crates/mise/registry.json,crates/config/src/,crates/words/src/,crates/versioning/src/,crates/update/src/plan.rs,crates/update/src/tooling.rs,PatchulaConfig.pkl,crates/github/src/model.rs,crates/github/src/blobs.rs,crates/engine/src/repository-head.graphql,crates/github/graphql/schema.docs.graphql chrono,chrono-tz,cron,globset,hcl-edit,jsonc-parser,nodejs-semver,pep440_rs,percent-encoding,pklr,regex,roxmltree,semver,serde,serde_json,serde_yaml_ng,sha2,toml_edit,url,yaml-rust2 28c309358f41554ace8bc0984edbb190ce004794606fb01592cb0d44dfc98046 +config-files 14 crates/engine/src/lib.rs,crates/engine/src/repository.rs,crates/manager/src/,crates/mise/src/,crates/mise/registry.json,crates/config/src/,crates/words/src/,crates/versioning/src/,crates/update/src/plan.rs,crates/update/src/tooling.rs,PatchulaConfig.pkl,crates/github/src/model.rs,crates/github/src/blobs.rs,crates/engine/src/repository-head.graphql,crates/github/graphql/schema.docs.graphql chrono,chrono-tz,cron,globset,hcl-edit,jsonc-parser,nodejs-semver,pep440_rs,percent-encoding,pklr,regex,roxmltree,semver,serde,serde_json,serde_yaml_ng,sha2,toml_edit,url,yaml-rust2 28c309358f41554ace8bc0984edbb190ce004794606fb01592cb0d44dfc98046 +catalog 16 crates/engine/src/catalogs.rs,crates/datasource/src/,crates/versioning/src/,crates/words/src/,crates/engine/src/lib.rs,crates/config/src/package.rs,crates/http/src/derive.rs,crates/http/src/cached.rs,crates/http/src/tracked.rs base64,chrono,nodejs-semver,pep440_rs,regex,semver,serde,serde_json,serde_yaml_ng,sha2 b8d4d669acee8484a5b5f1b61fdc8e12f95ceaf14b588f8daf56167a9bf2c9bb +catalog-controls 16 crates/engine/src/catalogs.rs,crates/update/src/identity.rs,crates/update/src/plan.rs,crates/update/src/plan/,crates/report/src/lib.rs,crates/manager/src/lib.rs,crates/words/src/,crates/engine/src/,crates/manager/src/,crates/config/src/,crates/lookup/src/,crates/lockfiles/src/ serde,serde_json,sha2 076a0e0a7e1722e0cda85cd311aa2452ba55a9f6b8f58597615a616da78f5bed not-ours 3 crates/update/src/cleanup.rs,crates/update/src/trailers.rs,crates/github/src/model.rs serde,serde_json 0d1832be99b85c57a8ee9348f778df2617bbedd1de714b469c0549a308818660 changelog 7 crates/notes/src/,crates/versioning/src/,crates/words/src/,crates/http/src/derive.rs,crates/http/src/cached.rs,crates/http/src/tracked.rs base64,nodejs-semver,pep440_rs,regex,semver,serde,serde_json,serde_yaml_ng da69de4854f91c5c6eb032c42bbabaf25ed85910308c8a9e0279920f522f7658 notes-adapter 7 crates/notes/src/,crates/versioning/src/,crates/words/src/,crates/http/src/derive.rs,crates/http/src/cached.rs,crates/http/src/tracked.rs base64,nodejs-semver,pep440_rs,regex,semver,serde,serde_json,serde_yaml_ng da69de4854f91c5c6eb032c42bbabaf25ed85910308c8a9e0279920f522f7658 -source-head 5 crates/datasource/src/github.rs,crates/datasource/src/releases.graphql,crates/github/src/,crates/github/graphql/schema.docs.graphql,crates/http/src/derive.rs graphql_client,serde,serde_json 2c60d1d2b06524c338586f91462609680a907108ccb1966b84f51e91fa7cd1d8 -tag-catalog 5 crates/datasource/src/github.rs,crates/datasource/src/catalog.rs,crates/github/src/ serde,serde_json,sha2 d15dd71df425938c0af4db6e7fddea620b07a2c3abd3af32a3678d48402ba1b1 -tag-object 5 crates/datasource/src/github.rs,crates/datasource/src/catalog.rs,crates/github/src/ serde,serde_json d15dd71df425938c0af4db6e7fddea620b07a2c3abd3af32a3678d48402ba1b1 +source-head 6 crates/datasource/src/github.rs,crates/datasource/src/releases.graphql,crates/github/src/,crates/github/graphql/schema.docs.graphql,crates/http/src/derive.rs graphql_client,serde,serde_json 962581169bae516cb338c37cf24a28a561431bfd21acefc52e3b3802315309ef +tag-catalog 6 crates/datasource/src/github.rs,crates/datasource/src/catalog.rs,crates/github/src/ serde,serde_json,sha2 a5140f7a0bfcaa8637173473960ca270940f1dc520de5fc9dc11bd3abb33da7a +tag-object 6 crates/datasource/src/github.rs,crates/datasource/src/catalog.rs,crates/github/src/ serde,serde_json a5140f7a0bfcaa8637173473960ca270940f1dc520de5fc9dc11bd3abb33da7a osv-advisory 2 crates/datasource/src/advisories.rs,crates/words/src/,crates/http/src/cached.rs,crates/http/src/tracked.rs cvss,serde,serde_json 146ab8efb9d29225d3096084f231506f0944b690a646577845a783baed03da4f release-notes 5 crates/notes/src/releases.rs,crates/github/src/ serde,serde_json ad92c09a13f291befa32fed2df1179cad45832ec1f7d2de155106ceeef9e0599 From 6ea82aae88471622a405a9da1a8ff87125a669f0 Mon Sep 17 00:00:00 2001 From: Sweepy Date: Mon, 16 Nov 2026 00:45:09 +0000 Subject: [PATCH 2/2] test(datasource): match registry protocol reads by URL, not by interleaving --- crates/datasource/tests/registry_protocol.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/datasource/tests/registry_protocol.rs b/crates/datasource/tests/registry_protocol.rs index 4a450215..fed63190 100644 --- a/crates/datasource/tests/registry_protocol.rs +++ b/crates/datasource/tests/registry_protocol.rs @@ -11,13 +11,16 @@ use std::{ struct Steps(Mutex>); impl Transport for Steps { fn send(&self, request: reqwest::Request) -> Exchange<'_> { - let (url, response) = self - .0 - .lock() - .unwrap() - .pop_front() - .expect("unexpected network read"); - assert_eq!(request.url().as_str(), url); + // Sources fan out reads (Go's `.info` dating, for one), so the + // requests and each URL's own sequence are under test, not the + // interleaving of different URLs. + let mut steps = self.0.lock().unwrap(); + let at = steps + .iter() + .position(|(url, _)| url == request.url().as_str()) + .unwrap_or_else(|| panic!("unexpected network read: {}", request.url())); + let (_, response) = steps.remove(at).unwrap(); + drop(steps); if response.status == 304 { assert_eq!(request.headers()["if-none-match"], "\"tags\""); }