diff --git a/driver/src/cargo.rs b/driver/src/cargo.rs index 5d7c861bb..000f91428 100644 --- a/driver/src/cargo.rs +++ b/driver/src/cargo.rs @@ -92,6 +92,9 @@ pub struct Check<'a> { pub manifest: &'a Path, /// The cargo package whose library the driver handles. pub package: &'a str, + /// The `Cargo.toml` of a package that depends on it, which cargo resolves it through + /// instead of its own workspace, if any, absolute. + pub root: Option<&'a Path>, /// The target to check for, or `None` for the host. pub target: Option<&'a str>, /// Passed on to `cargo check`, such as `--features`. @@ -113,7 +116,7 @@ impl Check<'_> { cmd.current_dir(package_dir).envs(self.env.iter().copied()); cmd.arg(subcommand) .arg("--manifest-path") - .arg(self.manifest) + .arg(self.root.unwrap_or(self.manifest)) .arg("--package") .arg(self.package) .arg("--target-dir") @@ -196,7 +199,7 @@ fn resolution(args: &[String]) -> Vec<&str> { } /// The directory of the workspace `manifest`, or the current directory, belongs to. -fn workspace_root(manifest: Option<&Path>) -> Result { +pub fn workspace_root(manifest: Option<&Path>) -> Result { let mut cmd = cargo(); cmd.args(["locate-project", "--workspace", "--message-format", "plain"]); if let Some(manifest) = manifest { diff --git a/ffi/justfile b/ffi/justfile index 6944613d9..058ba5934 100644 --- a/ffi/justfile +++ b/ffi/justfile @@ -8,6 +8,11 @@ set shell := ["bash", "-euo", "pipefail", "-c"] export PATH := if env("RUST2_TS_TOOLCHAIN", "") == "" { env("PATH") } else { env("RUST2_TS_TOOLCHAIN") / "bin:" + env("PATH") } +# The moq revision these recipes export by default, behind `just ts moq`'s: moq's later API +# needs moq/ffi.toml to follow it, and past moq-dev/moq#7032 moq-tokio's `Send` check +# overflows the recursion limit `moq-export` denies. +moq_rev := "30f25c04e6d0288d87893dc609ba582518be2cf6" + default: @just --list @@ -37,7 +42,7 @@ fix: run *args: cargo run --locked --quiet -- {{ args }} -# Census each package moq/ffi.toml exports into target/census/.md, from the moq revision `just ts moq` pins or a checkout. +# Census each package moq/ffi.toml exports into target/census/.md, from the pinned moq revision or a checkout. census checkout="": (standin checkout) cargo run --locked --quiet -- --config moq/ffi.toml --manifest-path target/moq-ffi/Cargo.toml --census target/census @@ -64,7 +69,7 @@ bench checkout="": (moq checkout) kotlin -Djava.library.path="$out/target/release" -cp "target/bench-kt:$KOTLINX_COROUTINES" ExportKt done -# Export moq's crates' C ABI, build it, and run moq/moq.c, moq/moq.cpp, and moq/moq.kt against it, from the moq revision `just ts moq` pins or a checkout. +# Export moq's crates' C ABI, build it, and run moq/moq.c, moq/moq.cpp, and moq/moq.kt against it, from the pinned moq revision or a checkout. moq checkout="": (moq-cpp checkout) #!/usr/bin/env bash set -euo pipefail @@ -107,14 +112,14 @@ moq-export checkout="": (standin checkout) RUSTFLAGS="-D recursion_depth_exceeding_limit --print=native-static-libs" cargo build --quiet --release --manifest-path "$out/Cargo.toml" --features kotlin 2>&1 | tee target/moq-build.log sed -n 's/^note: native-static-libs: //p' target/moq-build.log > target/moq-native -# Write target/moq-ffi, an empty library on every crate moq/ffi.toml exports, from the moq revision `just ts moq` pins or a checkout: the check builds with all of them, as moq-ffi's remainder will. +# Write target/moq-ffi, an empty library on every crate moq/ffi.toml exports, from the pinned moq revision or a checkout: the check builds with all of them, as moq-ffi's remainder will. [private] standin checkout="": #!/usr/bin/env bash set -euo pipefail src="{{ checkout }}" if [ -z "$src" ]; then - (cd ../ts && just moq-pin) + (cd ../ts && just moq-pin {{ moq_rev }}) src=../ts/target/moq fi src=$(realpath "$src") diff --git a/ffi/src/cli.rs b/ffi/src/cli.rs index 10f94dd83..7e1ea66b6 100644 --- a/ffi/src/cli.rs +++ b/ffi/src/cli.rs @@ -43,6 +43,7 @@ fn try_run(args: &[String]) -> Result { tool: "rust2-ffi", manifest: &args.manifest, package: &config.package, + root: None, // The host's, whatever cargo's config names: a C ABI is the platform's, and the // exports are found on it. target: Some(rustc_session::config::host_tuple()), diff --git a/flake.nix b/flake.nix index 2c3c1400a..558dfb7a9 100644 --- a/flake.nix +++ b/flake.nix @@ -304,13 +304,11 @@ RUST2_TS_SYSROOT = rust2-ts-std; KOTLINX_COROUTINES = "${kotlinx-coroutines}"; }; - # `just ts bench` adds Chrome and Firefox as engines, and wasm-bindgen for moq-wasm's - # package, which must match the moq revision's or fails. Apart for their size. + # `just ts bench` adds Chrome and Firefox as engines. Apart for their size. devShells.bench = self.devShells.${system}.default.overrideAttrs (old: { nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.chromium pkgs.firefox - pkgs.wasm-bindgen-cli ]; # `just ffi bench` builds moq's rs/moq-c, whose VAAPI bindings run bindgen. LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; diff --git a/quest/m1/ffi-export/recursion-limit.md b/quest/m1/ffi-export/recursion-limit.md index 48ef0b2be..4b894644a 100644 --- a/quest/m1/ffi-export/recursion-limit.md +++ b/quest/m1/ffi-export/recursion-limit.md @@ -14,5 +14,6 @@ quest in moq. ## Plan -Once it passes on moq's `main`, bump the exporter's `moq_rev` and update the config to what +Once it passes on moq's `main`, drop the exporter's own `moq_rev` for `just ts moq`'s, so the +translator and the exporter share one moq pin again, and update the config to what `just ffi census` then refuses. diff --git a/quest/m2/README.md b/quest/m2/README.md index 5b4ae8621..635d2f76b 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -26,7 +26,6 @@ The fork's feature quests folded into the [exporter](/quest/m1/ffi-export/README ## Required -- [Translate moq-net without moq-wasm](/quest/m2/ts-moq-pin.md) - the translator's moq pin moves past moq's WASM removal, back onto the exporter's -- [Translate moq-net's model to TypeScript](/quest/m2/ts-model/README.md) - the model translates with kio, moq-clock, and moq-pattern, for moq to compare against WASM +- [Translate moq-net's model to TypeScript](/quest/m2/ts-model/README.md) - the model translates with kio and moq-pattern, as moq's browser route - [Waiting: Swift 6.4 in nixpkgs-unstable](/quest/m2/uniffi-swift-arm.md) - the nightly UniFFI fixtures move back to arm - [Waiting: ubrn reads UniFFI 0.32](/quest/m2/ubrn-release.md) - the binding benchmark's browser runner pins a ubrn release, not an open PR diff --git a/quest/m2/ts-model/README.md b/quest/m2/ts-model/README.md index 97a8eba07..bbabea6b8 100644 --- a/quest/m2/ts-model/README.md +++ b/quest/m2/ts-model/README.md @@ -3,7 +3,7 @@ ## Goal [rust2-ts](/ts/README.md) translates moq-net's model layer, with the workspace crates it -calls (kio, moq-clock, moq-pattern), as it translates the codec today, so moq can ship +calls (kio, with its clock, and moq-pattern), as it translates the codec today, so moq can ship `@moq/net` as JavaScript translated from its Rust, in place of the handwritten package. ## Plan @@ -49,5 +49,6 @@ Public API: the translator's output for moq-net's model. ## Required +- [The std calls moq's main added to the model](/quest/m2/ts-model/main-std.md) - `VecDeque::range`, and `str::rsplit_once` and `split_terminator` by a `char`, translate - [A crate's own hasher](/quest/m2/ts-model/own-hasher.md) - hashlink's default hasher, a crate's own `Hasher`, hashes keys as native's does - [A crate's own iterator traits](/quest/m2/ts-model/iter-traits.md) - hashlink's double-ended, exact-size, and fused iterators, and their `size_hint` diff --git a/quest/m2/ts-model/main-std.md b/quest/m2/ts-model/main-std.md new file mode 100644 index 000000000..91d4bfe67 --- /dev/null +++ b/quest/m2/ts-model/main-std.md @@ -0,0 +1,17 @@ +# [S] The std calls moq's main added to the model + +## Goal + +rust2-ts translates the std calls that moq-net's model reaches at moq's `main` and refused +there first, proven by differential tests against native Rust: `VecDeque::range`, and +`str::rsplit_once` and `str::split_terminator` by a `char`. + +## Plan + +Found moving the translator's moq pin past moq's WASM removal: each is one refusal in the +census of `ts/moq/model.toml` (`model/group.rs`, `model/origin.rs`, `path/mod.rs`) that the +previous pin's census did not have. Translate each as std's others of its kind are, with +no config. The model's new `dyn DoubleEndedIterator` in `model/track.rs` is +[trait objects](/quest/m3/ts-dyn.md)' work. + +Public API: the translator's output. diff --git a/quest/m2/ts-moq-pin.md b/quest/m2/ts-moq-pin.md deleted file mode 100644 index bf582d08b..000000000 --- a/quest/m2/ts-moq-pin.md +++ /dev/null @@ -1,21 +0,0 @@ -# [M] Translate moq-net without moq-wasm - -## Goal - -`just ts moq` and `just ts census` translate moq-net from moq's `main`, and the translator -and the exporter share one moq pin again ([#390](https://github.com/moq-dev/rust2/pull/390) -holds until then). moq dropped WASM in -[moq-dev/moq#7475](https://github.com/moq-dev/moq/pull/7475), and with it moq-wasm, which -picked the browser's randomness for moq-net's `rand` and `uuid` on wasm32. Without it, -getrandom and uuid refuse to build there, so the translator's moq pin cannot move past it. - -## Plan - -The browser's randomness is a platform boundary, so the driver may pick it for a stock -crate, as a wasm32 binary would. Prefer what getrandom and uuid document for wasm32 over a -stand-in crate. - -`just ts bench`'s `moq-wasm package` row has no source past that pin either: decide with -the maintainer whether it goes, or stays on the last revision with moq-wasm. The -[model's](/quest/m2/ts-model/README.md) comparison against moq's WASM build may change -with it. diff --git a/ts/Cargo.toml b/ts/Cargo.toml index 63f2d4def..063f12f3c 100644 --- a/ts/Cargo.toml +++ b/ts/Cargo.toml @@ -40,7 +40,7 @@ toml = "1.1.8" [lints] workspace = true -# moq's profile for moq-wasm, which ships size-optimized: the benchmark's WASM route. +# The size-optimized profile moq's WASM shipped with: the benchmark's WASM route. [profile.wasm-release] inherits = "release" lto = "fat" diff --git a/ts/README.md b/ts/README.md index b41c85a75..fd69092d1 100644 --- a/ts/README.md +++ b/ts/README.md @@ -9,7 +9,13 @@ rust2-ts --config ts.toml (--out dir | --census file.md) [--manifest-path Cargo. It runs `cargo check` on the package with itself as `RUSTC_WRAPPER`, for `wasm32-unknown-unknown`, against std checked from source with every function's MIR. The flake's package and dev shell check it once per toolchain and pass it as `RUST2_TS_SYSROOT`; without -that, each target directory checks it with `-Zbuild-std`, which needs the toolchain's `rust-src`. Every crate compiles as rustc +that, each target directory checks it with `-Zbuild-std`, which needs the toolchain's `rust-src`. Where the +package builds with getrandom or uuid, which refuse wasm32 until a binary picks their source of randomness, cargo checks it +as a dependency of a root package that picks `crypto.getRandomValues` by the feature each documents, as a browser's binary +would, in a workspace that resolves as the package's own: its lockfile, patches, and dev profile, and its paths, +relative to its root as rustc reads a member's. `--features` and `--no-default-features` pass to that dependency; +`--all-features`, a feature of the package's dependency, and a `"*"` dev-profile override, which skips workspace members +but would reach the package there, are refused. Every crate compiles as rustc would; the configured library is also translated, after type and borrow checking, starting from its roots and following what they reach. Every crate it depends on but std's translates with it as one program, as far as the roots reach: each compiles with every function's MIR, and its types and @@ -968,8 +974,8 @@ comes up `Short` and retries, iterator chains over bytes, a `BTreeMap` of 16, 25 4096 keys, and timestamps converted between timescales as moq-net's `Timestamp::convert` does, whose products pass 2^53, then added by its `checked_add` and ordered by its `Ord`, whose cross products do too, and integers sorted, searched for their greatest and least, -and kept in a `BinaryHeap`. Each runs four ways: natively, as wasm32 at moq-wasm's -size-optimized profile, hand-written (moq's `js/net` in `bench/moq.ts` and its `Timestamp` +and kept in a `BinaryHeap`. Each runs four ways: natively, as wasm32 at the +size-optimized profile moq's WASM shipped with, hand-written (moq's `js/net` in `bench/moq.ts` and its `Timestamp` in `bench/timestamp.ts`, ES iterator helpers in `bench/iter.ts`, a sorted array in `bench/sorted.ts`, `bigint` in `bench/rescale.ts`, as moq's `js/hang` converts timescales, and typed arrays and a heap of its own in `bench/order.ts`), and translated; every route's output must match @@ -1007,4 +1013,4 @@ workloads still settle into one of two speeds per worker, up to 2x apart, so on their A/A stayed 6-35% wide while the median workload's was 1-3%. The report's A/A line also gives the noise of a family's geometric mean, the steadier gauge. -The bench shell adds Chromium, Firefox, and the wasm-bindgen that builds moq-wasm's package. +The bench shell adds Chromium and Firefox. diff --git a/ts/bench/main.ts b/ts/bench/main.ts index 33245e755..07b660ece 100644 --- a/ts/bench/main.ts +++ b/ts/bench/main.ts @@ -2,7 +2,7 @@ // Node, Bun, Chrome, and Firefox, and reports the numbers with each route's bundle size. // // usage: bun main.ts --translated --wasm --native --moq -// --moq-wasm --task --task-wasm --task-native +// --task --task-wasm --task-native // --out [--sample-ms 2] [--rounds 25] [--seed ] [--route =] // // The task fixture's async handoff runs beside the codec, against moq's way with promises. @@ -30,7 +30,6 @@ const { values: args } = parseArgs({ wasm: { type: "string" }, native: { type: "string" }, moq: { type: "string" }, - "moq-wasm": { type: "string" }, task: { type: "string" }, "task-wasm": { type: "string" }, "task-native": { type: "string" }, @@ -160,13 +159,11 @@ await Promise.all([ extra && Bun.write(join(out, "task-route.js"), extra.task), ]); -const moqWasm = arg("moq-wasm"); const sizes = [ { route: "wasm", files: [wasmBytes] }, { route: "hand-written", files: [handBundle] }, { route: "translated", files: [translation] }, ...(extra ? [{ route: extra.name, files: [extra.codec] }] : []), - { route: "moq-wasm package", files: [await read(join(moqWasm, "moq_bg.wasm")), await read(join(moqWasm, "moq.js"))] }, ].map(measureSize); // The task fixture: its channel, its async functions, and their runtime, as each route ships them. @@ -366,7 +363,7 @@ function render(): string { `of ${rounds} samples of at least ${sampleMs} ms, ± half their interquartile range. Each round runs`, `the JS routes and workloads in a new random order, from seed ${seed}. \`native\` is a release`, "build on the host, its workloads in a fixed order. `wasm` is the same", - "Rust at moq-wasm's size-optimized profile, run with its input already in memory;", + "Rust at the size-optimized profile moq's WASM shipped with, run with its input already in memory;", "copying the input in and the output out are timed apart. `hand-written` is moq's", "`js/net`, called as its own benchmarks call it, for `iter/` the same chain with ES", "iterator helpers (`bench/iter.ts`), for `btree/` a sorted array (`bench/sorted.ts`), for `hash/` a `Map` (`bench/hashed.ts`), and", @@ -380,8 +377,7 @@ function render(): string { "", "## Bundle size", "", - "Bytes for the same workloads, as timed: minified by Bun where JS. `moq-wasm package` is what moq", - "ships as WASM today, its bindgen glue included: the whole consume path, not only the codec.", + "Bytes for the same workloads, as timed: minified by Bun where JS.", "", "| route | raw | gzip |", "| --- | ---: | ---: |", diff --git a/ts/justfile b/ts/justfile index 7bfded524..8a1315ac2 100644 --- a/ts/justfile +++ b/ts/justfile @@ -9,7 +9,7 @@ set shell := ["bash", "-euo", "pipefail", "-c"] export PATH := if env("RUST2_TS_TOOLCHAIN", "") == "" { env("PATH") } else { env("RUST2_TS_TOOLCHAIN") / "bin:" + env("PATH") } # The moq revision `moq` translates by default. -moq_rev := "30f25c04e6d0288d87893dc609ba582518be2cf6" +moq_rev := "4b8e37bd44e407b3416a17992720870ea3f9221c" default: @just --list @@ -51,8 +51,7 @@ moq checkout="": src=target/moq fi out=target/moq-ts - # moq-wasm comes along to select getrandom's wasm backend for moq-net. - cargo run --locked --quiet -- --config moq/ts.toml --manifest-path "$src/rs/moq-net/Cargo.toml" --out "$out" -- --package moq-wasm + cargo run --locked --quiet -- --config moq/ts.toml --manifest-path "$src/rs/moq-net/Cargo.toml" --out "$out" printf '{ "extends": "%s/runtime/tsconfig.json", "files": [], "include": ["**/*.ts"] }\n' "$PWD" > "$out/tsconfig.json" tsc -p "$out" @@ -65,24 +64,24 @@ census checkout="": just moq-pin src=target/moq fi - cargo run --locked --quiet -- --config moq/net.toml --manifest-path "$src/rs/moq-net/Cargo.toml" --census target/census/moq-net.md -- --package moq-wasm - cargo run --locked --quiet -- --config moq/model.toml --manifest-path "$src/rs/moq-net/Cargo.toml" --census target/census/model.md -- --package moq-wasm + cargo run --locked --quiet -- --config moq/net.toml --manifest-path "$src/rs/moq-net/Cargo.toml" --census target/census/moq-net.md + cargo run --locked --quiet -- --config moq/model.toml --manifest-path "$src/rs/moq-net/Cargo.toml" --census target/census/model.md cargo run --locked --quiet -- --config moq/kio.toml --manifest-path "$src/rs/kio/Cargo.toml" --census target/census/kio.md -# Check out the pinned moq revision in target/moq. +# Check out the pinned moq revision, or another, in target/moq. [private] -moq-pin: +moq-pin rev=moq_rev: [ -d target/moq/.git ] || git clone --quiet --filter=blob:none https://github.com/moq-dev/moq target/moq # Fetch only a missing pin; the partial clone would otherwise fetch it lazily to look it up. - GIT_NO_LAZY_FETCH=1 git -C target/moq cat-file -e {{ moq_rev }}^{commit} 2> /dev/null || git -C target/moq fetch --quiet --depth 1 origin {{ moq_rev }} - git -C target/moq checkout --quiet --detach {{ moq_rev }} + GIT_NO_LAZY_FETCH=1 git -C target/moq cat-file -e {{ rev }}^{commit} 2> /dev/null || git -C target/moq fetch --quiet --depth 1 origin {{ rev }} + git -C target/moq checkout --quiet --detach {{ rev }} # Time the codec natively, as moq's WASM, as moq's hand-written TS, and translated, in Node, Bun, Chrome, and Firefox; `--route name=dir` adds another run's translations. [arg("route", long)] bench checkout="" route="": (moq checkout) #!/usr/bin/env bash set -euo pipefail - for tool in bun chromium firefox wasm-bindgen; do + for tool in bun chromium firefox; do command -v "$tool" > /dev/null || { echo "$tool is missing: run this in the bench shell, nix develop .#bench" >&2; exit 1; } done src="{{ checkout }}" @@ -98,10 +97,7 @@ bench checkout="" route="": (moq checkout) cargo build --locked --quiet --package task-fixture --example handoff --release # moq's hand-written TS imports its npm dependencies; its postinstall builds libav. (cd "$src" && bun install --frozen-lockfile --ignore-scripts --filter @moq/net) - # moq-wasm's package, as moq's `just js wasm` builds it. - (cd "$src" && cargo build --locked --quiet --package moq-wasm --target wasm32-unknown-unknown --profile wasm-release) - wasm-bindgen --target web --out-name moq --out-dir "$out/moq-wasm" "$src/target/wasm32-unknown-unknown/wasm-release/moq_wasm.wasm" - bun bench/main.ts --translated "$out/translated" --moq "$src" --moq-wasm "$out/moq-wasm" --out "$out" \ + bun bench/main.ts --translated "$out/translated" --moq "$src" --out "$out" \ --wasm target/wasm32-unknown-unknown/wasm-release/codec_fixture.wasm --native target/release/examples/bench \ --task "$out/task" \ --task-wasm target/wasm32-unknown-unknown/wasm-release/task_fixture.wasm --task-native target/release/examples/handoff \ diff --git a/ts/src/browser.rs b/ts/src/browser.rs new file mode 100644 index 000000000..7ed1adb35 --- /dev/null +++ b/ts/src/browser.rs @@ -0,0 +1,538 @@ +//! The choice a crate leaves to the wasm32 binary that depends on it, which translation +//! makes as a browser's binary would: the source of randomness. getrandom and uuid refuse +//! to build for wasm32-unknown-unknown until the binary enables the feature that picks +//! `crypto.getRandomValues`, and only a package that depends on them can, so translation +//! checks the package through a root package of its own that does. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::cli::TARGET; + +/// The feature that picks `crypto.getRandomValues`, by crate and the version prefix it is +/// documented for. +const PICKS: [(&str, &str, &str); 4] = [ + ("getrandom", "0.2.", "js"), + ("getrandom", "0.3.", "wasm_js"), + ("getrandom", "0.4.", "wasm_js"), + ("uuid", "1.", "js"), +]; + +/// A cache directory's tag, as specifies it. +const CACHEDIR_TAG: &str = "Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag created by rust2-ts. +# For information about cache directory tags see https://bford.info/cachedir/ +"; + +/// The root package that checks `package` with the browser's picks. +pub struct Root { + /// Its `Cargo.toml`, absolute. + pub manifest: PathBuf, + /// The cargo args for the check through it: the package's features move into the + /// root's dependency on it, since cargo takes none on the command line for a package + /// outside the workspace. + pub cargo: Vec, +} + +/// The root package for `package` of `manifest`, written in `dir`, if what it builds with +/// for [`TARGET`] holds a crate that leaves the browser's randomness to the binary. +pub fn root(manifest: &Path, package: &str, dir: &Path, cargo: &[String]) -> Result, String> { + let args = Args::parse(cargo)?; + let packages = tree(manifest, package, "normal", &args)?; + let picks: Vec<_> = (packages.iter()) + .filter_map(|p| { + let pick = PICKS + .iter() + .find(|(krate, prefix, _)| *krate == p.name && p.version.starts_with(prefix)); + pick.map(|&(_, _, feature)| (p, feature)) + }) + .collect(); + if picks.is_empty() { + return Ok(None); + } + if args.rest.iter().any(|arg| arg == "--all-features") { + return Err(format!( + "--all-features: the browser's randomness checks {package} as a dependency, which takes only the features --features names" + )); + } + // `cargo tree` lists the package first, wherever its directory is. + let Some(Source::Path(package_dir)) = packages.first().map(|p| &p.source) else { + return Err(format!( + "{package} is not a path package, which the browser's randomness can depend on" + )); + }; + let workspace = rust2_driver::cargo::workspace_root(Some(manifest))?; + // Cargo cleans only a target directory it made, which it tags as a cache, as it + // would here had it run first. + if !dir.exists() { + std::fs::create_dir_all(dir).map_err(|err| format!("creating {}: {err}", dir.display()))?; + write(&dir.join("CACHEDIR.TAG"), CACHEDIR_TAG)?; + } + let dir = dir.join("browser").join(package); + std::fs::create_dir_all(&dir).map_err(|err| format!("creating {}: {err}", dir.display()))?; + if config_overrides_all(manifest, &args)? { + return Err(overrides_all(package, "cargo's config")); + } + // What cargo compiles, build scripts' dependencies too, which the profile overrides. + let built = tree(manifest, package, "normal,build", &args)?; + let text = manifest_text(package, package_dir, &workspace, &args, &picks, &built)?; + write(&dir.join("Cargo.toml"), &text)?; + write(&dir.join("lib.rs"), "")?; + // The workspace's own versions, wherever the root needs no other. + let lock = workspace.join("Cargo.lock"); + if lock.exists() { + std::fs::copy(&lock, dir.join("Cargo.lock")).map_err(|err| format!("copying {}: {err}", lock.display()))?; + } + // The lockfile gains the root and what its picks build with, so `--locked` holds the + // workspace's own lockfile to account in `cargo tree` above, not the root's. + let mut cargo: Vec = (args.rest.into_iter()) + .filter(|arg| arg != "--locked") + .map(|arg| match arg.as_str() { + "--frozen" => "--offline".to_string(), + _ => arg, + }) + .collect(); + // The workspace's paths, as rustc reads its members': relative to its root, so + // `file!()` and the census read as they would there. + let remap = toml::Value::from(format!("--remap-path-prefix={}/=", workspace.display())); + cargo.extend(["--config".into(), format!("profile.dev.rustflags=[{remap}]")]); + // getrandom 0.3 before 0.3.4 takes its backend from a cfg beside the feature, which + // the others ignore. + if picks + .iter() + .any(|(p, _)| p.name == "getrandom" && p.version.starts_with("0.3.")) + { + let cfg = r#"profile.dev.package.getrandom.rustflags=["--cfg", 'getrandom_backend="wasm_js"']"#; + cargo.extend(["--config".into(), cfg.into()]); + } + Ok(Some(Root { + manifest: dir.join("Cargo.toml"), + cargo, + })) +} + +/// A package `cargo tree` lists. +struct Package { + name: String, + version: String, + source: Source, +} + +/// Where a package comes from. +enum Source { + /// crates.io, which a dependency names by its version alone. + Registry, + /// This directory. + Path(PathBuf), + /// A Git repository at a branch, tag, or revision, if its URL names one. + Git { + url: String, + reference: Option<(String, String)>, + }, + /// Another, as `cargo tree` prints it. + Other(String), +} + +/// The cargo args, with the package's features apart. +struct Args { + /// Each feature `--features` and `-F` name. + features: Vec, + /// Whether `--no-default-features` is given. + no_default: bool, + /// The rest. + rest: Vec, +} + +impl Args { + fn parse(cargo: &[String]) -> Result { + let mut parsed = Self { + features: Vec::new(), + no_default: false, + rest: Vec::new(), + }; + let mut args = cargo.iter(); + while let Some(arg) = args.next() { + let value = match arg.as_str() { + "--no-default-features" => { + parsed.no_default = true; + continue; + } + "--features" | "-F" => args.next().ok_or_else(|| format!("{arg} needs a value"))?, + _ => match arg.strip_prefix("--features=").or_else(|| arg.strip_prefix("-F")) { + Some(value) => value.strip_prefix('=').unwrap_or(value), + None => { + parsed.rest.push(arg.clone()); + continue; + } + }, + }; + let named = value.split([',', ' ']).filter(|f| !f.is_empty()); + parsed.features.extend(named.map(String::from)); + } + Ok(parsed) + } + + /// Each `--config` arg, with its value. + fn configs(&self) -> Vec { + let mut args = Vec::new(); + let mut rest = self.rest.iter(); + while let Some(arg) = rest.next() { + match arg.as_str() { + "--config" => args.extend([Some(arg), rest.next()].into_iter().flatten().cloned()), + _ if arg.starts_with("--config=") => args.push(arg.clone()), + _ => {} + } + } + args + } + + /// The args that pick how `package` resolves, as `cargo tree` takes them. + fn resolution(&self) -> Vec { + let flags = ["--all-features", "--locked", "--frozen", "--offline"]; + let mut args: Vec = (self.rest.iter()) + .filter(|arg| flags.contains(&arg.as_str())) + .cloned() + .collect(); + args.extend(self.configs()); + if self.no_default { + args.push("--no-default-features".into()); + } + if !self.features.is_empty() { + args.extend(["--features".into(), self.features.join(",")]); + } + args + } +} + +/// Each package `package` builds with for [`TARGET`] along `edges`, itself first. +fn tree(manifest: &Path, package: &str, edges: &str, args: &Args) -> Result, String> { + let tree = ["--package", package, "--edges", edges, "--target", TARGET]; + let output = cargo_tree(manifest, &tree, args)?; + let mut seen = std::collections::HashSet::new(); + let mut packages = Vec::new(); + for line in output.lines() { + let line = line.trim_end_matches(" (*)").replace(" (proc-macro)", ""); + // A package is its name, version, and source together. + if !seen.insert(line.clone()) { + continue; + } + let Some((name, rest)) = line.split_once(' ') else { + continue; + }; + let (version, source) = rest.split_once(' ').unwrap_or((rest, "")); + let version = version.strip_prefix('v').unwrap_or(version); + let source = match source.strip_prefix('(').and_then(|s| s.strip_suffix(')')) { + None => Source::Registry, + Some(dir) if Path::new(dir).is_absolute() => Source::Path(dir.into()), + Some(git) if git.contains("://") => { + let url = git.split('#').next().unwrap_or(git); + let (url, query) = url.split_once('?').unwrap_or((url, "")); + let reference = query.split_once('=').map(|(k, v)| (k.to_string(), v.to_string())); + Source::Git { + url: url.to_string(), + reference, + } + } + Some(other) => Source::Other(other.to_string()), + }; + packages.push(Package { + name: name.to_string(), + version: version.to_string(), + source, + }); + } + Ok(packages) +} + +/// Whether cargo's config, read as the check reads it, sets the dev profile's `"*"` +/// override. +fn config_overrides_all(manifest: &Path, args: &Args) -> Result { + let output = Command::new(std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into())) + .current_dir(manifest.parent().expect("a manifest path has a parent")) + .arg("-Zunstable-options") + .args(args.configs()) + .args(["config", "get", "profile.dev.package"]) + .output() + .map_err(|err| format!("running cargo config: {err}"))?; + let stderr = String::from_utf8_lossy(&output.stderr); + match output.status.success() { + true => Ok(String::from_utf8_lossy(&output.stdout).contains(".\"*\".")), + false if stderr.contains("is not set") => Ok(false), + false => Err(stderr.into_owned()), + } +} + +/// The error for a `"*"` dev-profile override that `from` sets: it overrides every package +/// but a workspace member, and the package is no member of the root's workspace, so it +/// would reach the package there as it never does natively. +fn overrides_all(package: &str, from: &str) -> String { + format!( + "{from} overrides every package's dev profile with `\"*\"`, which skips workspace members but would reach {package} through the root that picks the browser's randomness: override the packages by name instead" + ) +} + +/// What `cargo tree` prints for the workspace `manifest` belongs to with `tree`'s args and +/// the check's resolution: each package as ` v`, then its source in +/// parentheses unless it is crates.io, plain, so a repeat's `(*)` reads as one. +fn cargo_tree(manifest: &Path, tree: &[&str], args: &Args) -> Result { + let output = Command::new(std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into())) + // Cargo reads its config from where it runs, as the check does. + .current_dir(manifest.parent().expect("a manifest path has a parent")) + .args(["tree", "--manifest-path"]) + .arg(manifest) + .args(tree) + .args(["--prefix", "none", "--format", "{p}", "--color", "never"]) + .args(args.resolution()) + .output() + .map_err(|err| format!("running cargo tree: {err}"))?; + if !output.status.success() { + return Err(String::from_utf8_lossy(&output.stderr).into_owned()); + } + String::from_utf8(output.stdout).map_err(|err| format!("reading cargo tree: {err}")) +} + +/// The root's `Cargo.toml`: a library on `package`, in `dir`, and on each pick, in a +/// workspace that resolves and compiles the `built` packages as `workspace`, the package's +/// own, does. +fn manifest_text( + package: &str, + dir: &Path, + workspace: &Path, + args: &Args, + picks: &[(&Package, &str)], + built: &[Package], +) -> Result { + let mut dependencies = toml::Table::new(); + // A feature named as the package's own, `/`, by its name there. + let own = format!("{package}/"); + let features: Vec<&str> = (args.features.iter()) + .map(|f| f.strip_prefix(&own).unwrap_or(f)) + .collect(); + // Cargo takes none of a dependency's own dependencies' features. + if let Some(feature) = features.iter().find(|f| f.contains('/')) { + return Err(format!( + "--features {feature}: the browser's randomness checks {package} as a dependency, which takes only its own features" + )); + } + let dependency = toml::toml! { + path = (dir.display().to_string()) + default-features = (!args.no_default) + features = features + }; + dependencies.insert(package.to_string(), dependency.into()); + for (i, (pick, feature)) in picks.iter().enumerate() { + // Its feature alone, so the crate builds as the package has it otherwise, from the + // source it resolved to, which a version alone would name another of. + let mut dependency = toml::toml! { + package = (pick.name.clone()) + version = (format!("={}", pick.version)) + default-features = false + features = (vec![feature.to_string()]) + }; + match &pick.source { + Source::Registry => {} + // Another registry, which the root would have to name by the key cargo's config + // gives it. + Source::Other(source) => { + return Err(format!( + "{} {} is from {source}, which the browser's randomness can't depend on", + pick.name, pick.version + )); + } + Source::Path(dir) => { + dependency.insert("path".into(), dir.display().to_string().into()); + } + Source::Git { url, reference } => { + dependency.insert("git".into(), url.clone().into()); + if let Some((key, value)) = reference { + dependency.insert(key.clone(), value.clone().into()); + } + } + } + // Unique, since a graph may hold one version of a crate from two sources. + let key = format!("{}-{}-{i}", pick.name, pick.version.replace(['.', '+'], "_")); + dependencies.insert(key, dependency.into()); + } + let mut root = toml::toml! { + [package] + name = "rust2-ts-browser" + version = "0.0.0" + edition = "2024" + publish = false + + [lib] + path = "lib.rs" + }; + root.insert("dependencies".into(), dependencies.into()); + let path = workspace.join("Cargo.toml"); + let text = std::fs::read_to_string(&path).map_err(|err| format!("reading {}: {err}", path.display()))?; + let mut theirs: toml::Table = toml::from_str(&text).map_err(|err| format!("parsing {}: {err}", path.display()))?; + root.insert( + "workspace".into(), + toml::toml! { resolver = (resolver(&theirs)) }.into(), + ); + // What else picks how the workspace resolves, each path absolute. + for key in ["patch", "replace"] { + if let Some(mut value) = theirs.remove(key) { + absolute(&mut value, workspace); + root.insert(key.into(), value); + } + } + let dev = theirs + .get_mut("profile") + .and_then(|p| p.as_table_mut()) + .and_then(|p| p.remove("dev")); + if let Some(toml::Value::Table(dev)) = dev { + let dev = dev_profile(dev, package, built)?; + root.insert("profile".into(), toml::toml! { dev = dev }.into()); + } + toml::to_string(&root).map_err(|err| format!("writing the root package: {err}")) +} + +/// The workspace's dev profile `dev` as the root's, for the `built` packages, but its +/// overrides of packages never built, which cargo would warn of. +fn dev_profile(mut dev: toml::Table, package: &str, built: &[Package]) -> Result { + let Some(toml::Value::Table(mut overrides)) = dev.remove("package") else { + return Ok(dev); + }; + if overrides.contains_key("*") { + return Err(overrides_all(package, "the workspace's Cargo.toml")); + } + // A spec of another form, such as a URL, may match one. + overrides.retain(|spec, _| { + let Some((name, version)) = spec_parts(spec) else { + return true; + }; + (built.iter()).any(|p| { + p.name == name && version.is_none_or(|v| p.version == v || p.version.starts_with(&format!("{v}."))) + }) + }); + dev.insert("package".into(), overrides.into()); + Ok(dev) +} + +/// The name and version, if any, of a package spec of the form `name`, `name@version`, or +/// `name:version`. +fn spec_parts(spec: &str) -> Option<(&str, Option<&str>)> { + // A URL's scheme reads as a name before a `:`. + if spec.contains('/') { + return None; + } + let (name, version) = match spec.split_once(['@', ':']) { + Some((name, version)) => (name, Some(version)), + None => (spec, None), + }; + let crate_name = !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); + crate_name.then_some((name, version)) +} + +/// The feature resolver a workspace's root manifest picks: its own, or its package's +/// edition's, or a virtual manifest's 1. +fn resolver(root: &toml::Table) -> String { + let field = |table: &str, key: &str| root.get(table).and_then(|t| t.get(key)).and_then(toml::Value::as_str); + if let Some(resolver) = field("workspace", "resolver").or_else(|| field("package", "resolver")) { + return resolver.to_string(); + } + // An edition the package inherits is the workspace's. + let inherited = || root.get("workspace")?.get("package")?.get("edition")?.as_str(); + let edition = root.get("package").and(field("package", "edition").or_else(inherited)); + match edition { + Some("2024") => "3", + Some("2021") => "2", + _ => "1", + } + .to_string() +} + +/// `value` with each `path` in it absolute, relative to `dir` where it was not. +fn absolute(value: &mut toml::Value, dir: &Path) { + let Some(table) = value.as_table_mut() else { + return; + }; + for (key, value) in table.iter_mut() { + match value { + toml::Value::String(path) if key == "path" => *path = dir.join(&*path).display().to_string(), + _ => absolute(value, dir), + } + } +} + +fn write(path: &Path, text: &str) -> Result<(), String> { + std::fs::write(path, text).map_err(|err| format!("writing {}: {err}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn takes_the_features_apart() { + for spelling in [ + &["--features", "a,dep/b"][..], + &["--features=a dep/b"], + &["-F", "a", "-F", "dep/b"], + &["-Fa,dep/b"], + &["-F=a", "--features", "dep/b"], + ] { + let cargo: Vec = spelling.iter().map(|s| s.to_string()).collect(); + let args = Args::parse(&cargo).unwrap(); + assert_eq!( + (args.features, args.rest), + (vec!["a".into(), "dep/b".into()], vec![]), + "{spelling:?}" + ); + } + let cargo = ["--no-default-features", "--locked", "--config", "a=1", "--quiet"]; + let args = Args::parse(&cargo.map(String::from)).unwrap(); + assert!(args.no_default); + assert_eq!(args.rest, ["--locked", "--config", "a=1", "--quiet"]); + assert_eq!( + args.resolution(), + ["--locked", "--config", "a=1", "--no-default-features"] + ); + for trailing in ["--features", "-F"] { + assert!(Args::parse(&[trailing.to_string()]).is_err(), "{trailing}"); + } + } + + #[test] + fn keeps_the_overrides_cargo_applies() { + let package = |name: &str| Package { + name: name.into(), + version: "1.0.0".into(), + source: Source::Registry, + }; + let built = [package("named"), package("versioned")]; + let dev: toml::Table = toml::from_str( + r#" + debug = 1 + [package.named] + opt-level = 3 + [package.unbuilt] + opt-level = 3 + [package."versioned@1"] + opt-level = 2 + [package."versioned@2"] + opt-level = 2 + [package."https://example.com/index#other@1.0.0"] + opt-level = 2 + "#, + ) + .unwrap(); + let expected: toml::Table = toml::from_str( + r#" + debug = 1 + [package.named] + opt-level = 3 + [package."versioned@1"] + opt-level = 2 + [package."https://example.com/index#other@1.0.0"] + opt-level = 2 + "#, + ) + .unwrap(); + assert_eq!(dev_profile(dev, "pkg", &built), Ok(expected)); + let star: toml::Table = toml::from_str("[package.\"*\"]\ndebug-assertions = false").unwrap(); + assert!(dev_profile(star, "pkg", &built).is_err()); + } +} diff --git a/ts/src/cli.rs b/ts/src/cli.rs index a18af2b22..b98f634db 100644 --- a/ts/src/cli.rs +++ b/ts/src/cli.rs @@ -5,6 +5,7 @@ use std::process::ExitCode; use rust2_driver::cargo::{Args, Check, Output}; +use crate::browser; use crate::config::{self, Config}; /// Translation type-checks for this target: its 32-bit `usize` is what a JS number @@ -15,7 +16,8 @@ const USAGE: &str = "usage: rust2-ts --config (--out | --census Translates the library of the package that ts.toml names to TypeScript in . The package's Cargo.toml is beside ts.toml unless --manifest-path says where. Cargo args, -such as --features, are passed to `cargo check`, which checks with the dev profile. +such as --features, are passed to `cargo check`, which checks with the dev profile, and +picks getrandom's and uuid's browser randomness for wasm32 as a binary would. --census writes a Markdown report of every construct translation refuses in the library, reached from each of its items as if each were a root, instead of translating it."; @@ -52,15 +54,18 @@ fn try_run(args: &[String]) -> Result { ("CARGO_CACHE_RUSTC_INFO", "0".as_ref()), ("CARGO_UNSTABLE_SEPARATE_NIGHTLIES", "true".as_ref()), ]; + let target_dir = rust2_driver::cargo::target_dir("rust2-ts", Some(&args.manifest))?; + let root = browser::root(&args.manifest, &config.package, &target_dir, &args.cargo)?; let mut cargo = profile_args(sysroot.is_none()); // The profile `PROFILE` sets, which cargo refuses to combine with one the cargo args // pick. cargo.extend(["--profile".into(), "dev".into()]); - cargo.extend(args.cargo); + cargo.extend(root.as_ref().map_or(args.cargo, |root| root.cargo.clone())); Check { tool: "rust2-ts", manifest: &args.manifest, package: &config.package, + root: root.as_ref().map(|root| root.manifest.as_path()), target: Some(TARGET), args: &cargo, env: &env, diff --git a/ts/src/main.rs b/ts/src/main.rs index e29eed78b..f21afc9ca 100644 --- a/ts/src/main.rs +++ b/ts/src/main.rs @@ -25,6 +25,7 @@ extern crate rustc_span; extern crate rustc_target; extern crate rustc_trait_selection; +mod browser; mod census; mod check; mod cli; diff --git a/ts/tests/fixtures.rs b/ts/tests/fixtures.rs index 79b3b00fa..2f0b2d825 100644 --- a/ts/tests/fixtures.rs +++ b/ts/tests/fixtures.rs @@ -632,19 +632,13 @@ fn sysroot() { /// wasm32 has no OS to read. Only getrandom's flags change, so nothing else rebuilds. const GETRANDOM: &str = r#"profile.dev.package.getrandom.rustflags=["--cfg", 'getrandom_backend="custom"']"#; -/// `bytes`, `rand`, and `tracing`, which translation maps without a binding or config. +/// `bytes`, `rand`, and `tracing`, which translation maps without a binding or config: +/// it picks getrandom's backend itself, as a browser's binary would. #[test] fn stock() { let out = scratch().join("stock").join("out"); let config = fixture("stock").join("ts.toml"); - let args: [&std::ffi::OsStr; 5] = [ - "--out".as_ref(), - out.as_ref(), - "--".as_ref(), - "--config".as_ref(), - GETRANDOM.as_ref(), - ]; - let output = driver("stock", &config, &args, "build"); + let output = driver("stock", &config, &["--out".as_ref(), out.as_ref()], "build"); assert!( output.status.success(), "translating stock failed:\n{}",