Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions crates/config/src/matching.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,4 +129,39 @@ impl Filter {
};
Ok(yes != self.negated)
}
/// Whether the filter deliberately admits the prerelease `text`, whose
/// version without its prerelease part is `release`. A negated filter
/// only excludes, so it never asks. A `regex:` filter asks when it
/// matches the prerelease but not its release, so the match comes from
/// the prerelease part (`regex:-rc\.`, `regex:nightly`). A `semver:`
/// range asks when it names a prerelease version itself
/// (`semver:>=3.0.0-rc.1`, `semver:[1.0-beta1,2)`) and matches `text`.
pub fn asks_for_prerelease(
&self,
scheme: Ecosystem,
text: &str,
release: &str,
) -> Result<bool, String> {
if self.negated {
return Ok(false);
}
match &self.kind {
FilterKind::Regex(regex) => Ok(regex.is_match(text) && !regex.is_match(release)),
FilterKind::Semver(range) => {
Ok(names_prerelease(scheme, range) && self.matches(scheme, text)?)
}
}
}
}
/// Whether a range's text names a prerelease version in `scheme`'s syntax.
fn names_prerelease(scheme: Ecosystem, range: &str) -> bool {
use patchula_versioning::Versioning;
range
.split(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | '+')))
.filter(|token| !token.is_empty())
.any(|token| {
scheme
.parse(token)
.is_ok_and(|version| !version.0.pre.is_empty())
})
}
21 changes: 20 additions & 1 deletion crates/datasource/src/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,36 @@ pub struct Version {
pub digest: Option<String>,
pub deprecated: bool,
pub yanked: bool,
/// The source marks this release a prerelease, whatever its version
/// syntax says (a GitHub release flagged as one, listed through mise).
#[serde(default)]
pub prerelease: bool,
/// The source was not asked for this release's date and flags, or the
/// request failed: `published_at`, `deprecated` and `yanked` say
/// nothing. Policy holds such a release instead of reading it as
/// undated, so a missing answer never skips the release-age wait.
#[serde(default)]
pub unchecked: bool,
}
impl Version {
/// A listed version with no date or digest, neither deprecated nor
/// yanked.
/// yanked: the source has been asked and has nothing more to say.
pub fn new(version: String) -> Self {
Self {
version,
published_at: None,
digest: None,
deprecated: false,
yanked: false,
prerelease: false,
unchecked: false,
}
}
/// A listed version whose date and flags the source was not asked for.
pub fn unchecked(version: String) -> Self {
Self {
unchecked: true,
..Self::new(version)
}
}
}
Expand Down
6 changes: 3 additions & 3 deletions crates/datasource/src/compact.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ pub(super) fn npm(body: &mut dyn std::io::Read) -> Result<Vec<u8>, CacheError> {
sort(&mut catalog.versions, Ecosystem::Npm);
encode(catalog)
}
pub(crate) const NPM_CATALOG: Derived<CacheError> = Derived::new("npm-catalog-v2", 1, npm);
pub(crate) const NPM_CATALOG: Derived<CacheError> = Derived::new("npm-catalog-v2", 2, npm);

#[derive(Deserialize)]
struct CrateVersion {
Expand All @@ -85,7 +85,7 @@ pub(super) fn crates(body: &mut dyn std::io::Read) -> Result<Vec<u8>, CacheError
catalog.versions.reverse(); // Sparse index is append-only publication order.
encode(catalog)
}
pub(crate) const CRATES_CATALOG: Derived<CacheError> = Derived::new("crates-catalog-v2", 1, crates);
pub(crate) const CRATES_CATALOG: Derived<CacheError> = Derived::new("crates-catalog-v2", 2, crates);

#[derive(Deserialize)]
struct Pypi {
Expand Down Expand Up @@ -117,7 +117,7 @@ pub(super) fn pypi(body: &mut dyn std::io::Read) -> Result<Vec<u8>, CacheError>
sort(&mut catalog.versions, Ecosystem::Pep440);
encode(catalog)
}
pub(crate) const PYPI_CATALOG: Derived<CacheError> = Derived::new("pypi-catalog-v2", 1, pypi);
pub(crate) const PYPI_CATALOG: Derived<CacheError> = Derived::new("pypi-catalog-v2", 2, pypi);

#[cfg(test)]
mod tests {
Expand Down
101 changes: 101 additions & 0 deletions crates/datasource/src/dates.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
//! Per-version release dates for sources whose listing carries none (Go's
//! `.info`, a Maven POM's `Last-Modified`). Every listed version is asked
//! about once and the answer kept, so a warm lookup asks only about versions
//! it has not seen, wherever they sort: a patch release on an older line is
//! dated like a new major. A version whose request failed is marked
//! unchecked and asked about again by the next lookup.
use super::*;
use futures_util::{StreamExt, stream};

/// Per-version date requests a lookup has in flight at once.
const FAN_OUT: usize = 8;

/// What the source said about one version.
#[derive(Clone, Serialize, Deserialize)]
pub(crate) enum Dated {
/// The version exists; its publication time, when the source gives one.
At(Option<String>),
/// The source lists the version but cannot serve it: not installable.
Missing,
}
/// The answers kept for one package, by version.
#[derive(Default, Serialize, Deserialize)]
struct Dates(BTreeMap<String, Dated>);
#[derive(Serialize)]
pub(crate) struct DatesScope {
pub(crate) source: Source,
pub(crate) package: String,
/// The operator registry entry's scope when one answered, so a date read
/// with a credential is only reused under the same one.
pub(crate) credential: Option<String>,
}
impl State for Dates {
const NAMESPACE: &'static str = "release-dates";
const VERSION: u32 = 1;
type Scope = DatesScope;
}

impl Client {
/// Date every version in `versions` that has no date yet, asking `ask`
/// only about versions the store has no answer for. `ask` returns
/// `None` when the request failed: that version is marked unchecked and
/// nothing is stored for it. Missing versions are dropped.
pub(crate) async fn date_versions<F, Fut>(
&self,
scope: &DatesScope,
versions: &mut Vec<Version>,
ask: F,
) -> Result<()>
where
F: Fn(String) -> Fut,
Fut: std::future::Future<Output = Option<Dated>>,
{
let mut dates = self.cache.state::<Dates>(scope).await?.unwrap_or_default();
let wanted: Vec<String> = versions
.iter()
.filter(|v| v.published_at.is_none() && !dates.0.contains_key(&v.version))
.map(|v| v.version.clone())
.collect();
let answers: Vec<(String, Option<Dated>)> = stream::iter(wanted)
.map(|version| {
let answer = ask(version.clone());
async move { (version, answer.await) }
})
.buffer_unordered(FAN_OUT)
.collect()
.await;
let mut failed = std::collections::HashSet::new();
let mut learned = false;
for (version, answer) in answers {
match answer {
Some(dated) => {
dates.0.insert(version, dated);
learned = true;
}
None => {
failed.insert(version);
}
}
}
versions.retain_mut(|version| {
if failed.contains(&version.version) {
version.unchecked = true;
return true;
}
match dates.0.get(&version.version) {
Some(Dated::Missing) => false,
Some(Dated::At(time)) => {
if version.published_at.is_none() {
version.published_at.clone_from(time);
}
true
}
None => true,
}
});
if learned {
self.cache.set_state(scope, &dates).await?;
}
Ok(())
}
}
4 changes: 2 additions & 2 deletions crates/datasource/src/forge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ fn tags(
serde_json::to_vec(&versions).map_err(|_| patchula_http::CacheError::Document)
}
pub(crate) const FORGE_GITLAB_TAGS: patchula_http::Derived<patchula_http::CacheError> =
patchula_http::Derived::new("forge-gitlab-tags", 1, gitlab);
patchula_http::Derived::new("forge-gitlab-tags", 2, gitlab);
pub(crate) const FORGE_GITEA_TAGS: patchula_http::Derived<patchula_http::CacheError> =
patchula_http::Derived::new("forge-gitea-tags", 1, gitea);
patchula_http::Derived::new("forge-gitea-tags", 2, gitea);

fn forge(package: &str) -> Result<Forge> {
Repository::from_url(&format!("https://{package}"))
Expand Down
8 changes: 4 additions & 4 deletions crates/datasource/src/github.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ fn compact_release_head(
>(body)?)
}
pub(crate) const GITHUB_RELEASE_HEAD: patchula_http::Derived<serde_json::Error> =
patchula_http::Derived::new("github-release-head", 5, compact_release_head);
patchula_http::Derived::new("github-release-head", 6, compact_release_head);

/// The newest release's identity on a release source; a change refetches
/// history. `None` records a source with no releases.
Expand All @@ -68,7 +68,7 @@ pub(crate) struct SourceHeadScope {
}
impl State for SourceHead {
const NAMESPACE: &'static str = "source-head";
const VERSION: u32 = 5;
const VERSION: u32 = 6;
// Code-versioned: the stored fingerprint holds the newest release's
// fields as this build picks them, an entry built by code that picked
// differently must not compare equal.
Expand All @@ -85,7 +85,7 @@ pub(crate) struct TagCatalogScope {
}
impl State for TagCatalog {
const NAMESPACE: &'static str = "tag-catalog";
const VERSION: u32 = 5;
const VERSION: u32 = 6;
// Code-versioned: the fingerprint in the key names the refs, not the
// catalog this build's peel-and-map code derived from them.
const CODE_VERSIONED: bool = true;
Expand All @@ -101,7 +101,7 @@ pub(crate) struct TagObjectScope {
}
impl State for TagObject {
const NAMESPACE: &'static str = "tag-object";
const VERSION: u32 = 5;
const VERSION: u32 = 6;
// Code-versioned: the stored version is this build's decode of the
// object the scope's sha names.
const CODE_VERSIONED: bool = true;
Expand Down
2 changes: 1 addition & 1 deletion crates/datasource/src/helm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1036,7 +1036,7 @@ fn helm_walk(body: &mut dyn Read) -> std::result::Result<Vec<u8>, CacheError> {
.collect();
serde_json::to_vec(&charts).map_err(|_| CacheError::Document)
}
pub(crate) const HELM_INDEX: Derived<CacheError> = Derived::new("helm-index", 3, helm);
pub(crate) const HELM_INDEX: Derived<CacheError> = Derived::new("helm-index", 4, helm);

#[cfg(test)]
mod tests {
Expand Down
4 changes: 2 additions & 2 deletions crates/datasource/src/images.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ fn compact_platform(
.map_err(|_| Document)
}
pub(crate) const IMAGE_PLATFORM: patchula_http::Derived<patchula_http::CacheError> =
patchula_http::Derived::new("image-platform", 3, compact_platform);
patchula_http::Derived::new("image-platform", 4, compact_platform);

fn platform_name(os: &str, architecture: &str, variant: Option<&str>) -> String {
match variant {
Expand Down Expand Up @@ -304,7 +304,7 @@ fn compact_sbom(
serde_json::to_vec(&compact).map_err(|_| Document)
}
pub(crate) const SBOM_OS: patchula_http::Derived<patchula_http::CacheError> =
patchula_http::Derived::new("sbom-os", 3, compact_sbom);
patchula_http::Derived::new("sbom-os", 4, compact_sbom);

/// OSV's ecosystem name for an OS release, when OSV covers it.
fn ecosystem(kind: &str, namespace: &str, distro: Option<&(String, String)>) -> Option<String> {
Expand Down
81 changes: 24 additions & 57 deletions crates/datasource/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ mod catalog;
pub use catalog::{Catalog, Version};
pub(crate) use catalog::{sha256_hex, sort};
mod compact;
mod dates;
#[cfg(test)]
mod derive_parity;
mod docker;
Expand Down Expand Up @@ -144,8 +145,6 @@ impl std::fmt::Display for Failure {
f.write_str(&self.message)
}
}
/// Go releases dated through `.info` on a cold lookup, newest first.
const GO_DATED_RELEASES: usize = 20;
/// A stored listing or resolution is reused for this long. The tools keep
/// their own version lists for about as long, and a release-age wait is
/// evaluated against the dates in the listing, so a stored one never hides
Expand Down Expand Up @@ -437,48 +436,32 @@ impl Client {
latest_version
};
sort(&mut versions, Ecosystem::Go);
let known = self.cache.releases("go", package).await?;
let known: std::collections::HashMap<_, _> = known
.releases
.into_iter()
.map(|v| (v.version, v.published_at))
.collect();
// Fetch .info only for new releases; retain publication timestamps for
// older ones. Never refetch full per-version history on a warm lookup.
// A cold lookup dates only the newest releases: older ones are outside
// any release-age wait, and a module can list hundreds of versions.
// The proxy lists some versions it cannot describe (404); those are
// not installable releases and are dropped.
let mut reached_known = false;
let mut fetched = 0;
let mut missing = std::collections::HashSet::new();
for version in &mut versions {
if let Some(time) = known.get(&version.version) {
version.published_at.clone_from(time);
reached_known = true;
} else if !reached_known && fetched < GO_DATED_RELEASES {
fetched += 1;
let response = self
.http
.get(
&format!("{prefix}/{}.info", version.version),
"application/json",
)
.await;
let response = match response {
Ok(response) => response,
// Every listed version is dated through `.info`, once: the answer is
// kept, so a warm lookup asks only about versions it has not seen,
// including a patch release on an older line. The proxy lists some
// versions it cannot describe (404); those are not installable
// releases and are dropped.
let scope = dates::DatesScope {
source: Source::Go,
package: package.into(),
credential: None,
};
self.date_versions(&scope, &mut versions, |version| {
let url = format!("{prefix}/{version}.info");
async move {
match self.http.get(&url, "application/json").await {
Ok(response) => {
let info: Value = serde_json::from_slice(&response.body).ok()?;
Some(dates::Dated::At(info["Time"].as_str().map(str::to_owned)))
}
Err(patchula_http::CacheError::Status(404 | 410)) => {
missing.insert(version.version.clone());
continue;
Some(dates::Dated::Missing)
}
Err(error) => return Err(error.into()),
};
let info: Value = serde_json::from_slice(&response.body)?;
version.published_at = info["Time"].as_str().map(str::to_owned);
Err(_) => None,
}
}
}
versions.retain(|v| !missing.contains(&v.version));
self.merge("go", package, &versions).await?;
})
.await?;
Ok(Catalog {
versions,
dist_tags: BTreeMap::from([("latest".into(), latest_version)]),
Expand Down Expand Up @@ -509,22 +492,6 @@ impl Client {
};
Ok(serde_json::from_slice(&response.body)?)
}
async fn merge(&self, source: &str, package: &str, versions: &[Version]) -> Result<()> {
let head = self.cache.release_head(source, package).await?;
let mut page = Vec::new();
for v in versions {
page.push(Release {
version: v.version.clone(),
published_at: v.published_at.clone(),
digest: v.digest.clone(),
});
if head.as_deref() == Some(&v.version) {
break;
}
}
self.cache.merge_releases(source, package, &page).await?;
Ok(())
}
}
fn runners(package: &str) -> Result<Catalog> {
use std::sync::LazyLock;
Expand Down
Loading
Loading