From f56232bd51ad8fa7b73c83da4ab897dfdb33f7cc Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Sat, 26 Sep 2026 18:14:56 -0400 Subject: [PATCH 1/4] Add in-memory hosted scan engine Let hosts run hosted-mode patching without a working directory or network access. The new hosted_memory engine takes a map of repository files, finds every project root, looks up patches once across all roots through a pluggable PatchApi provider, and returns the rewritten lockfiles and redirect ledgers. Lockfile inventory now reads through a ProjectView, so the same parsers serve disk and in-memory projects; disk behavior is unchanged. The existing scan --mode hosted path keeps using ApiClient directly, which also implements PatchApi. Parity tests compare the engine against the real CLI binary across npm, pnpm, yarn, bun, pypi, cargo, composer, gem and golang fixtures. A hidden hosted-bundle command exposes the engine for debugging. Assisted-by: Claude Code:claude-opus-5-5 --- crates/socket-patch-cli/CLI_CONTRACT.md | 2 + crates/socket-patch-cli/Cargo.toml | 2 + .../src/commands/hosted_bundle.rs | 158 +++ crates/socket-patch-cli/src/commands/mod.rs | 1 + .../src/commands/scan/hosted.rs | 55 +- .../socket-patch-cli/src/commands/scan/mod.rs | 2 +- .../src/hosted_memory/discover.rs | 452 +++++++ .../src/hosted_memory/ledger.rs | 177 +++ .../src/hosted_memory/limits.rs | 409 ++++++ .../socket-patch-cli/src/hosted_memory/mod.rs | 1113 ++++++++++++++++ .../src/hosted_memory/redirect.rs | 1154 +++++++++++++++++ .../src/hosted_memory/roots.rs | 263 ++++ .../src/hosted_memory/select.rs | 478 +++++++ .../src/hosted_memory/types.rs | 403 ++++++ crates/socket-patch-cli/src/lib.rs | 8 + crates/socket-patch-cli/src/main.rs | 1 + .../socket-patch-cli/tests/cli_global_args.rs | 3 + .../tests/hosted_memory_common/mod.rs | 413 ++++++ .../tests/hosted_memory_engine.rs | 932 +++++++++++++ .../tests/hosted_memory_parity.rs | 616 +++++++++ crates/socket-patch-core/src/api/client.rs | 300 ++++- .../src/utils/cargo_workspace.rs | 138 +- .../src/vendor/lock_inventory/bun.rs | 25 +- .../src/vendor/lock_inventory/cargo.rs | 26 +- .../src/vendor/lock_inventory/composer.rs | 21 +- .../src/vendor/lock_inventory/gem.rs | 19 +- .../src/vendor/lock_inventory/golang.rs | 17 +- .../src/vendor/lock_inventory/mod.rs | 47 +- .../src/vendor/lock_inventory/npm.rs | 12 +- .../src/vendor/lock_inventory/npm_family.rs | 80 +- .../src/vendor/lock_inventory/pnpm.rs | 49 +- .../src/vendor/lock_inventory/pypi.rs | 67 +- .../src/vendor/lock_inventory/view.rs | 498 +++++++ .../src/vendor/lock_inventory/yarn.rs | 19 +- crates/socket-patch-core/src/vendor/pypi.rs | 2 +- 35 files changed, 7793 insertions(+), 169 deletions(-) create mode 100644 crates/socket-patch-cli/src/commands/hosted_bundle.rs create mode 100644 crates/socket-patch-cli/src/hosted_memory/discover.rs create mode 100644 crates/socket-patch-cli/src/hosted_memory/ledger.rs create mode 100644 crates/socket-patch-cli/src/hosted_memory/limits.rs create mode 100644 crates/socket-patch-cli/src/hosted_memory/mod.rs create mode 100644 crates/socket-patch-cli/src/hosted_memory/redirect.rs create mode 100644 crates/socket-patch-cli/src/hosted_memory/roots.rs create mode 100644 crates/socket-patch-cli/src/hosted_memory/select.rs create mode 100644 crates/socket-patch-cli/src/hosted_memory/types.rs create mode 100644 crates/socket-patch-cli/tests/hosted_memory_common/mod.rs create mode 100644 crates/socket-patch-cli/tests/hosted_memory_engine.rs create mode 100644 crates/socket-patch-cli/tests/hosted_memory_parity.rs create mode 100644 crates/socket-patch-core/src/vendor/lock_inventory/view.rs diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index f6991b705..ac4ec2d8b 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -27,6 +27,8 @@ This document defines the **public surface** of the `socket-patch` binary. Anyth **Root `--update` flag.** `socket-patch --update [VERSION]` updates the binary itself from GitHub Releases. It is a root flag, not a subcommand: argv is rewritten (the same mechanism as the bare-UUID fallback) onto an internal hidden subcommand whose name carries no stability guarantee — script the flag, never the internal name. Combining the flag with a subcommand (`socket-patch --update scan`) is a usage error (exit 2). Full contract: [Self-update contract](#self-update-contract-socket-patch---update). +**Internal `hosted-bundle` subcommand.** `socket-patch hosted-bundle` is a hidden, INTERNAL parity/debug harness for the in-memory hosted engine (`src/hosted_memory/`, the engine the Node addon embeds): it reads a JSON bundle `{"files": {path: text}, "binaryFiles"?: {path: base64}, "presentOnly"?: [path], "symlinks"?: [path], "projectRoots"?: [dir], "pipenvMajor"?: n, "batchSize"?: n}` on stdin, queries the authenticated org API built from `--api-url` / `--api-token` / `--org` only (both of the latter are required; no public-proxy fallback), and prints the engine result — or `{"status":"error","error":{"code","message"}}` with exit 1 (exit 2 for unusable input or missing credentials). It never touches the filesystem. Its name, input and output carry NO stability guarantee; do not script it. + ## Global arguments In v3.0 every subcommand accepts the same set of "global" flags via a single shared `GlobalArgs` struct that's `#[command(flatten)]`-ed into each per-command struct (`crates/socket-patch-cli/src/args.rs`). Subcommands that don't actually consume a given flag accept it silently — e.g. `list --global` parses fine and is a no-op. Every flag also has an environment-variable binding; precedence is **CLI arg > env var > default** — and for exactly three keys (`--api-token`, `--org`, `--api-url`) the JS socket-cli's persisted login sits between env var and default: **CLI arg > env var (canonical, then `SOCKET_CLI_*` alias) > socket-cli `config.json` > default**. See "Persisted configuration" under Environment variables. diff --git a/crates/socket-patch-cli/Cargo.toml b/crates/socket-patch-cli/Cargo.toml index 00046ebe9..6da25c657 100644 --- a/crates/socket-patch-cli/Cargo.toml +++ b/crates/socket-patch-cli/Cargo.toml @@ -22,6 +22,8 @@ clap = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } +base64 = { workspace = true } console = { workspace = true } dialoguer = { workspace = true } uuid = { workspace = true } diff --git a/crates/socket-patch-cli/src/commands/hosted_bundle.rs b/crates/socket-patch-cli/src/commands/hosted_bundle.rs new file mode 100644 index 000000000..67fe2bb10 --- /dev/null +++ b/crates/socket-patch-cli/src/commands/hosted_bundle.rs @@ -0,0 +1,158 @@ +//! `socket-patch hosted-bundle` (hidden, internal): run the in-memory +//! hosted engine over a JSON bundle read from stdin and print its result. +//! A parity/debugging harness for [`crate::hosted_memory`]: the patch API +//! is the authenticated org API built from the global `--api-url` / +//! `--api-token` / `--org` arguments, which (like every command's) fall back +//! to `SOCKET_API_URL` / `SOCKET_API_TOKEN` / `SOCKET_ORG_SLUG` — so an +//! exported production token is used, reference grants included. It never +//! uses the public proxy, and the engine itself reads no environment. +//! +//! Stdin: `{"files": {path: text}, "binaryFiles"?: {path: base64}, +//! "presentOnly"?: [path], "symlinks"?: [path], "projectRoots"?: [dir], +//! "pipenvMajor"?: n, "batchSize"?: n}`. Stdout: the engine result +//! (`HostedScanResult`, binary contents base64), or +//! `{"status":"error","error":{"code","message"}}` with exit 1. + +use std::collections::BTreeMap; +use std::io::Read; +use std::sync::Arc; + +use base64::Engine; +use clap::Args; +use serde::Deserialize; +use socket_patch_core::api::client::{ApiClient, ApiClientOptions}; +use socket_patch_core::constants::DEFAULT_SOCKET_API_URL; +use tokio_util::sync::CancellationToken; + +use crate::args::GlobalArgs; +use crate::hosted_memory::{ + run_in_memory, EngineError, HostedScanOptions, MarkKind, PresentKind, SessionBuilder, +}; + +#[derive(Args)] +pub struct HostedBundleArgs { + #[command(flatten)] + pub common: GlobalArgs, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Bundle { + #[serde(default)] + files: BTreeMap, + #[serde(default)] + binary_files: BTreeMap, + #[serde(default)] + present_only: Vec, + #[serde(default)] + symlinks: Vec, + #[serde(default)] + project_roots: Option>, + #[serde(default)] + pipenv_major: Option, + #[serde(default)] + batch_size: Option, +} + +fn print_error(code: &str, message: &str) { + println!( + "{}", + serde_json::json!({ "status": "error", "error": { "code": code, "message": message } }) + ); +} + +fn build_input( + bundle: Bundle, + options: HostedScanOptions, +) -> Result { + let mut builder = SessionBuilder::new(options)?; + for (path, text) in &bundle.files { + builder.add_text(path, text)?; + } + for (path, encoded) in &bundle.binary_files { + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|e| EngineError::InvalidInput { + code: "invalid_base64", + message: format!("binaryFiles[{path}]: {e}"), + })?; + builder.add_binary(path, &bytes)?; + } + for path in &bundle.present_only { + builder.mark_present(path, MarkKind::Present(PresentKind::Present))?; + } + for path in &bundle.symlinks { + builder.mark_present(path, MarkKind::Symlink)?; + } + builder.finish() +} + +pub async fn run(args: HostedBundleArgs) -> i32 { + let common = &args.common; + let (Some(token), Some(org)) = ( + common.api_token.clone().filter(|t| !t.is_empty()), + common.org.clone().filter(|o| !o.is_empty()), + ) else { + print_error( + "missing_credentials", + "hosted-bundle requires --api-token and --org", + ); + return 2; + }; + let mut raw = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut raw) { + print_error("invalid_bundle", &format!("cannot read stdin: {e}")); + return 2; + } + let bundle: Bundle = match serde_json::from_str(&raw) { + Ok(bundle) => bundle, + Err(e) => { + print_error("invalid_bundle", &e.to_string()); + return 2; + } + }; + let options = HostedScanOptions { + org_slug: org.clone(), + ecosystems: common.ecosystems.clone(), + batch_size: bundle.batch_size, + dry_run: common.dry_run, + pipenv_major: bundle.pipenv_major, + trust_lockfile_config: Some(!common.no_trust_lockfile_config), + npm_allow_remote_config: Some(!common.no_npm_allow_remote_config), + project_roots: bundle.project_roots.clone(), + ..HostedScanOptions::default() + }; + let input = match build_input(bundle, options) { + Ok(input) => input, + Err(e) => { + print_error(e.code(), &e.to_string()); + return 2; + } + }; + let client = ApiClient::new(ApiClientOptions { + api_url: common + .api_url + .clone() + .filter(|u| !u.is_empty()) + .unwrap_or_else(|| DEFAULT_SOCKET_API_URL.to_string()), + api_token: Some(token), + use_public_proxy: false, + org_slug: Some(org), + }); + match run_in_memory(input, Arc::new(client), CancellationToken::new()).await { + Ok(output) => match serde_json::to_string_pretty(&output) { + Ok(text) => { + println!("{text}"); + 0 + } + Err(e) => { + print_error("engine_internal", &e.to_string()); + 1 + } + }, + Err(e) => { + print_error(e.code(), &e.to_string()); + 1 + } + } +} diff --git a/crates/socket-patch-cli/src/commands/mod.rs b/crates/socket-patch-cli/src/commands/mod.rs index 11ce26343..60dba5854 100644 --- a/crates/socket-patch-cli/src/commands/mod.rs +++ b/crates/socket-patch-cli/src/commands/mod.rs @@ -2,6 +2,7 @@ pub mod apply; pub(crate) mod bun_preflight; pub(crate) mod fetch_stage; pub mod get; +pub mod hosted_bundle; pub mod list; pub(crate) mod lock_cli; pub mod remove; diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 2c218c539..39ad86bb9 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -22,9 +22,10 @@ mod python; /// Fragment-edit kinds whose lockfile the package manager re-lays in place /// (keeping the Socket source) — a re-scan REBASES their ledger edits instead /// of appending; see the ledger merge below. -const REBASE_KINDS: &[&str] = &["redirect_poetry_lock_package", "redirect_pdm_lock_package"]; +pub(crate) const REBASE_KINDS: &[&str] = + &["redirect_poetry_lock_package", "redirect_pdm_lock_package"]; -const REDIRECT_CANDIDATE_FILES: &[&str] = &[ +pub(crate) const REDIRECT_CANDIDATE_FILES: &[&str] = &[ "package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", @@ -95,7 +96,7 @@ const REDIRECT_CANDIDATE_FILES: &[&str] = &[ /// dependency (reqwest is dev-only here); per RFC 3986 a raw `@` in the /// authority can ONLY be the userinfo terminator (it is percent-encoded /// everywhere else), so the tail after the LAST `@` is exactly host[:port]. -fn url_host(url: &str) -> Option<&str> { +pub(crate) fn url_host(url: &str) -> Option<&str> { let rest = url.split_once("://").map_or(url, |(_, r)| r); let authority = rest.split(['/', '?', '#']).next().unwrap_or(rest); let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h); @@ -105,7 +106,7 @@ fn url_host(url: &str) -> Option<&str> { /// Repo-relative path of the pnpm workspace manifest the trustLockfile /// auto-config edits (the same file the vendor backend's override surface /// uses). -const PNPM_WORKSPACE_REL: &str = "pnpm-workspace.yaml"; +pub(crate) const PNPM_WORKSPACE_REL: &str = "pnpm-workspace.yaml"; /// `FileEdit.kind` recorded when the hosted flow ensures `trustLockfile: /// true` in pnpm-workspace.yaml. `action: "created"` — the workspace file @@ -113,14 +114,14 @@ const PNPM_WORKSPACE_REL: &str = "pnpm-workspace.yaml"; /// `trustLockfile: true` line was appended to an existing file (a revert /// removes exactly that line). Additive ledger vocabulary: older ledgers /// without it load unchanged (kind is an opaque string to the loader). -const REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND: &str = "redirect_pnpm_workspace_trust"; +pub(crate) const REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND: &str = "redirect_pnpm_workspace_trust"; /// The honest-tradeoff + don't-rebuild tail shared by every trustLockfile /// warning variant. The tradeoff sentence is a security disclosure, not /// prose garnish: `trustLockfile: true` disables pnpm's lockfile /// re-verification for the WHOLE lock, so it must be stated wherever the /// setting is written or recommended. -const PNPM_TRUST_TRADEOFF_AND_CAUTION: &str = +pub(crate) const PNPM_TRUST_TRADEOFF_AND_CAUTION: &str = "Note: trustLockfile makes pnpm skip its lockfile re-verification \ (minimumReleaseAge / trustPolicy re-checks) for ALL lockfile entries, \ not just the patched ones — the per-entry sha512 integrity pins are \ @@ -131,7 +132,7 @@ const PNPM_TRUST_TRADEOFF_AND_CAUTION: &str = /// The policy preamble shared by every trustLockfile warning variant: /// what was repointed, and how pnpm >=11 fails without trust. -fn pnpm_trust_policy_preamble(server: &str) -> String { +pub(crate) fn pnpm_trust_policy_preamble(server: &str) -> String { format!( "pnpm-lock.yaml was repointed at {server}; pnpm >=11 rejects the \ rewritten lock (pnpm 11: ERR_PNPM_TARBALL_URL_MISMATCH, pnpm 12: \ @@ -142,7 +143,7 @@ fn pnpm_trust_policy_preamble(server: &str) -> String { /// The pre-auto-config guidance, kept verbatim for the runs where the /// auto-config does not apply (legacy 5.x/6.0 locks, Rush nested locks, /// `--no-trust-lockfile-config`): both verified recoveries, spelled exactly. -fn pnpm_trust_manual_guidance(server: &str) -> String { +pub(crate) fn pnpm_trust_manual_guidance(server: &str) -> String { format!( "{}. Install with `pnpm install --trust-lockfile`, or commit \ `trustLockfile: true` in pnpm-workspace.yaml so every install \ @@ -160,7 +161,7 @@ fn pnpm_trust_manual_guidance(server: &str) -> String { /// no trust step exists or is needed. Deliberately NEVER mentions /// `pnpm install --trust-lockfile`: pnpm 7/8 reject the flag as an unknown /// option, so headlining it here would hand users a command that errors. -fn pnpm_trust_legacy_detail(server: &str) -> String { +pub(crate) fn pnpm_trust_legacy_detail(server: &str) -> String { format!( "The pnpm lockfile was repointed at {server}. This is a legacy \ lock read by pnpm 1–8, which have no \ @@ -178,7 +179,7 @@ fn pnpm_trust_legacy_detail(server: &str) -> String { /// here would OVERWRITE the user's file with the root-only scaffold — /// destroying their `packages:` globs — so the auto-config stands down and /// the warning names the file, the error, and both manual recoveries. -fn pnpm_trust_workspace_unreadable_detail(server: &str, err: &std::io::Error) -> String { +pub(crate) fn pnpm_trust_workspace_unreadable_detail(server: &str, err: &std::io::Error) -> String { format!( "{}. {PNPM_WORKSPACE_REL} exists but could not be read ({err}); it \ was left untouched — auto-configuring trust would risk overwriting \ @@ -237,7 +238,7 @@ fn pnpm_lock_carries_hosted_redirect( /// block engages anyway. Legacy (<9) and unparseable-version locks stay /// `None` (fail closed: never write config for a lock era we can't read), /// as does a root lock this run DID splice (the splice path covers it). -fn pnpm_heal_root<'a>( +pub(crate) fn pnpm_heal_root<'a>( root_spliced: bool, disk_root: Option<&'a String>, overrides: &[socket_patch_core::patch::redirect::DepOverride], @@ -253,7 +254,7 @@ fn pnpm_heal_root<'a>( /// The auto-config variant: trust was (or, on `--dry-run`, would be) /// configured in pnpm-workspace.yaml, so installs need no flags. -fn pnpm_trust_configured_detail(server: &str, created: bool, dry_run: bool) -> String { +pub(crate) fn pnpm_trust_configured_detail(server: &str, created: bool, dry_run: bool) -> String { let how = match (created, dry_run) { (true, false) => "`trustLockfile: true` was written to a new", (false, false) => "`trustLockfile: true` was merged into the existing", @@ -273,7 +274,7 @@ fn pnpm_trust_configured_detail(server: &str, created: bool, dry_run: bool) -> S /// `'6.0'`, pnpm 7 an unquoted `5.4`. `None` when no parseable version line /// exists — callers treat that as "not trust-policy era" and stay /// hands-off (fail closed: never write config for a lock we can't read). -fn pnpm_lock_version_major(lock_text: &str) -> Option { +pub(crate) fn pnpm_lock_version_major(lock_text: &str) -> Option { lock_text.lines().find_map(|line| { let rest = line.strip_prefix("lockfileVersion:")?; let value = rest.trim().trim_matches(|c| c == '\'' || c == '"'); @@ -285,7 +286,7 @@ fn pnpm_lock_version_major(lock_text: &str) -> Option { /// `--store` (pnpm 1–3 can silently ignore `--store-dir`; early pnpm 4 /// rejects it): a `shrinkwrapVersion` lock (pnpm 1–2) or lockfileVersion /// 5.0–5.2 (pnpm 3–5). Later locks never get the `--store` note. -fn pnpm_lock_may_need_store_flag(lock_text: &str) -> bool { +pub(crate) fn pnpm_lock_may_need_store_flag(lock_text: &str) -> bool { lock_text.lines().any(|line| { if line.starts_with("shrinkwrapVersion:") { return true; @@ -305,7 +306,7 @@ fn pnpm_lock_may_need_store_flag(lock_text: &str) -> bool { } /// The planned pnpm-workspace.yaml `trustLockfile: true` edit. -enum TrustPlan { +pub(crate) enum TrustPlan { /// No workspace file: create it (root-only `packages` scaffold — pnpm 9 /// refuses a workspace file with no `packages` field — plus the trust /// key; the same scaffold shape the vendor backend creates). @@ -325,7 +326,7 @@ enum TrustPlan { /// Line splices only (never a YAML library), mirroring the vendor backend's /// workspace surgery: untouched lines stay byte-identical, so a revert can /// remove exactly what was added. -fn plan_workspace_trust(existing: Option<&str>) -> TrustPlan { +pub(crate) fn plan_workspace_trust(existing: Option<&str>) -> TrustPlan { let Some(text) = existing else { return TrustPlan::Create("packages:\n - '.'\ntrustLockfile: true\n".to_string()); }; @@ -355,7 +356,7 @@ fn plan_workspace_trust(existing: Option<&str>) -> TrustPlan { /// The root npm locks the hosted rewriter edits (`rewrite_npm_lock` rewrites /// every one present — npm 12 installs from package-lock.json beside a /// committed shrinkwrap). -const NPM_LOCKS: [&str; 2] = ["npm-shrinkwrap.json", "package-lock.json"]; +pub(crate) const NPM_LOCKS: [&str; 2] = ["npm-shrinkwrap.json", "package-lock.json"]; /// The honest-tradeoff + opt-out tail shared by every `allow-remote` /// warning variant. The tradeoff sentence is a security disclosure, not @@ -382,7 +383,11 @@ fn npm_allow_remote_preamble(hosts: &[&str]) -> String { /// The auto-config variant: `allow-remote=all` was (or, on `--dry-run`, /// would be) written to the project `.npmrc`, so installs need no flags. -fn npm_allow_remote_configured_detail(hosts: &[&str], created: bool, dry_run: bool) -> String { +pub(crate) fn npm_allow_remote_configured_detail( + hosts: &[&str], + created: bool, + dry_run: bool, +) -> String { let how = match (created, dry_run) { (true, false) => "`allow-remote=all` was written to a new", (false, false) => "`allow-remote=all` was appended to the existing", @@ -399,7 +404,7 @@ fn npm_allow_remote_configured_detail(hosts: &[&str], created: bool, dry_run: bo } /// The project `.npmrc` already resolves to `allow-remote=all`. -fn npm_allow_remote_already_detail(hosts: &[&str]) -> String { +pub(crate) fn npm_allow_remote_already_detail(hosts: &[&str]) -> String { format!( "{}, and the project .npmrc already sets `allow-remote=all` — keep it committed \ alongside the lock; `npm ci` needs no extra flags. {NPM_ALLOW_REMOTE_TRADEOFF}", @@ -410,7 +415,7 @@ fn npm_allow_remote_already_detail(hosts: &[&str]) -> String { /// The user explicitly set another value: respected, never flipped (the /// pnpm `trustLockfile: false` precedent) — the warning names the manual /// recoveries instead. -fn npm_allow_remote_user_set_detail(hosts: &[&str], value: &str) -> String { +pub(crate) fn npm_allow_remote_user_set_detail(hosts: &[&str], value: &str) -> String { format!( "{}. The project .npmrc explicitly sets `allow-remote={value}`, which was respected \ and left untouched — set `allow-remote=all` there yourself (or install with \ @@ -423,7 +428,7 @@ fn npm_allow_remote_user_set_detail(hosts: &[&str], value: &str) -> String { /// An `npm_config_allow_remote` environment variable sets another value. /// npm's env layer beats every `.npmrc`, so a project write could not take /// effect in this environment — and an explicit setting is respected. -fn npm_allow_remote_env_set_detail(hosts: &[&str], var: &str, value: &str) -> String { +pub(crate) fn npm_allow_remote_env_set_detail(hosts: &[&str], var: &str, value: &str) -> String { format!( "{}. The environment variable {var}={value} explicitly sets `allow-remote`, which \ was respected: npm's environment layer overrides every .npmrc, so a project \ @@ -438,7 +443,7 @@ fn npm_allow_remote_env_set_detail(hosts: &[&str], var: &str, value: &str) -> St /// another value. A committed project `allow-remote=all` would silently /// override that machine / org policy on every checkout, so it is /// respected like a project value and the override is left to the user. -fn npm_allow_remote_outer_set_detail( +pub(crate) fn npm_allow_remote_outer_set_detail( hosts: &[&str], layer: &str, path: &std::path::Path, @@ -458,7 +463,7 @@ fn npm_allow_remote_outer_set_detail( /// The opt-out (`--no-npm-allow-remote-config`) variant: nothing written, /// both manual recoveries spelled out. -fn npm_allow_remote_manual_detail(hosts: &[&str]) -> String { +pub(crate) fn npm_allow_remote_manual_detail(hosts: &[&str]) -> String { format!( "{}. Commit `allow-remote=all` in the project .npmrc (or install with \ `npm ci --allow-remote=all`) so npm >=12 installs the patched artifacts. \ @@ -471,7 +476,7 @@ fn npm_allow_remote_manual_detail(hosts: &[&str]) -> String { /// be read, or is a symlink / non-regular file the atomic writer would /// replace. Planning a Create here would OVERWRITE the user's registry / /// auth config, so the auto-config stands down and names the problem. -fn npm_allow_remote_unreadable_detail(hosts: &[&str], why: &str) -> String { +pub(crate) fn npm_allow_remote_unreadable_detail(hosts: &[&str], why: &str) -> String { format!( "{}. The project .npmrc exists but {why}; it was left untouched. Add \ `allow-remote=all` to it yourself (or install with `npm ci --allow-remote=all`) \ @@ -561,7 +566,7 @@ fn build_redirect_json_envelope( /// `mode` is `"hosted"` (the final mode name for `--redirect`): an additive /// key so consumers dispatch on the mode without inferring it from which /// sub-object is present. -pub(super) fn redirect_json_block( +pub(crate) fn redirect_json_block( redirected: usize, rewritten: Vec, skipped: Vec, diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 0e727c382..fedc13f8b 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -32,7 +32,7 @@ use super::get::{download_and_apply_patches_with, select_patches, DownloadParams mod discovery; mod gc; -mod hosted; +pub(crate) mod hosted; pub(crate) mod render; pub(crate) mod vendor_flow; diff --git a/crates/socket-patch-cli/src/hosted_memory/discover.rs b/crates/socket-patch-cli/src/hosted_memory/discover.rs new file mode 100644 index 000000000..43572ecea --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/discover.rs @@ -0,0 +1,452 @@ +//! Patch discovery across every project root with one provider lookup per +//! purl / uuid / url: the union of the roots' purls is batch-searched in +//! sorted, deterministic chunks; packages with patches get their details +//! fetched with bounded concurrency; each root then takes the disk JSON +//! flow's selection (accessible patches only, top-ranked per purl through +//! [`cmp_search_results`]). + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::Poll; +use std::time::Duration; + +use socket_patch_core::api::client::{ApiError, ApiFuture, PatchApi}; +use socket_patch_core::api::ranking::cmp_search_results; +use socket_patch_core::api::types::{ + BatchPackagePatches, PackageVendorResult, PatchResponse, PatchSearchResult, SearchResponse, +}; +use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers}; + +use super::types::MAX_REFERENCE_BATCH; + +/// A boxed future the bounded joiner drives. +pub(crate) type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// Drive `futures` with at most `limit` in flight, returning their outputs +/// in input order. Futures are polled in place (no task spawning), so +/// dropping the returned future cancels every one of them. +pub(crate) async fn join_bounded<'a, T>(futures: Vec>, limit: usize) -> Vec { + let limit = limit.max(1); + let total = futures.len(); + let mut queued: VecDeque<(usize, BoxFuture<'a, T>)> = futures.into_iter().enumerate().collect(); + let mut active: Vec<(usize, BoxFuture<'a, T>)> = Vec::new(); + let mut out: Vec> = (0..total).map(|_| None).collect(); + let mut done = 0usize; + std::future::poll_fn(|cx| loop { + while active.len() < limit { + match queued.pop_front() { + Some(next) => active.push(next), + None => break, + } + } + let mut progressed = false; + let mut i = 0; + while i < active.len() { + if let Poll::Ready(value) = active[i].1.as_mut().poll(cx) { + let (index, _) = active.swap_remove(i); + out[index] = Some(value); + done += 1; + progressed = true; + } else { + i += 1; + } + } + if done == total { + return Poll::Ready(()); + } + if !progressed || queued.is_empty() { + return Poll::Pending; + } + }) + .await; + out.into_iter().flatten().collect() +} + +/// The host's [`PatchApi`] behind per-call timeouts and call counting. +pub(crate) struct Provider { + api: Arc, + timeout: Duration, + pub(crate) concurrency: usize, + calls: Mutex>, +} + +impl Provider { + pub(crate) fn new(api: Arc, timeout: Duration, concurrency: usize) -> Self { + Self { + api, + timeout, + concurrency, + calls: Mutex::new(BTreeMap::new()), + } + } + + pub(crate) fn calls(&self) -> BTreeMap { + self.calls.lock().map(|c| c.clone()).unwrap_or_default() + } + + async fn call(&self, method: &'static str, fut: ApiFuture<'_, T>) -> Result { + if let Ok(mut calls) = self.calls.lock() { + *calls.entry(method.to_string()).or_insert(0) += 1; + } + match tokio::time::timeout(self.timeout, fut).await { + Ok(result) => result, + Err(_) => Err(ApiError::Network(format!( + "{method} timed out after {} ms", + self.timeout.as_millis() + ))), + } + } + + pub(crate) async fn search_patches_batch( + &self, + purls: &[String], + ) -> Result { + let mut response = self + .call("searchPatchesBatch", self.api.search_patches_batch(purls)) + .await?; + socket_patch_core::api::client::sort_batch_response(&mut response); + Ok(response) + } + + pub(crate) async fn search_patches_by_package( + &self, + purl: &str, + ) -> Result { + let mut response = self + .call( + "searchPatchesByPackage", + self.api.search_patches_by_package(purl), + ) + .await?; + response.patches.sort_by(cmp_search_results); + Ok(response) + } + + pub(crate) async fn fetch_registry_references( + &self, + uuids: &[String], + ) -> Result, ApiError> { + self.call( + "fetchRegistryReferences", + self.api.fetch_registry_references(uuids), + ) + .await + } + + pub(crate) async fn fetch_patch(&self, uuid: &str) -> Result, ApiError> { + self.call("fetchPatch", self.api.fetch_patch(uuid)).await + } + + pub(crate) async fn download_artifact( + &self, + url: &str, + max_bytes: u64, + ) -> Result, ApiError> { + let bytes = self + .call( + "downloadArtifact", + self.api.download_artifact(url, max_bytes), + ) + .await?; + if bytes.len() as u64 > max_bytes { + return Err(ApiError::Other(format!( + "artifact exceeds the {max_bytes}-byte limit" + ))); + } + Ok(bytes) + } +} + +/// The literal, qualifier-free purl the disk flow's lockfile supplement +/// queries for an inventory entry (`crawled_from_purl`'s shape rule), or +/// `None` for a purl it drops. +pub(crate) fn supplement_purl(purl: &str) -> Option { + let decoded = normalize_purl(strip_purl_qualifiers(purl)).into_owned(); + let rest = decoded.strip_prefix("pkg:")?; + let (_eco, rest) = rest.split_once('/')?; + rest.rfind('@').filter(|&i| i > 0)?; + Some(decoded) +} + +/// One root's batch-search outcome. +#[derive(Debug, Default)] +pub(crate) struct RootBatch { + /// Packages with at least one patch, purl-sorted. + pub(crate) packages: Vec, + /// How many of the root's purls sat in a failed chunk. + pub(crate) failed_purls: usize, + pub(crate) last_error: Option, +} + +/// The run-level batch outcome. +#[derive(Debug, Default)] +pub(crate) struct BatchOutcome { + pub(crate) roots: BTreeMap, + pub(crate) can_access_paid_patches: bool, +} + +/// Batch-search the union of every root's purls (`root_purls` values are +/// sorted, deduplicated supplement purls). A response package is credited +/// to every root that asked for it; one whose purl matches no request +/// spelling goes to every root that had a purl in its chunk. +pub(crate) async fn batch_search( + provider: &Provider, + root_purls: &BTreeMap>, + batch_size: usize, +) -> BatchOutcome { + let mut owners: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (root, purls) in root_purls { + for purl in purls { + owners.entry(purl.as_str()).or_default().push(root.as_str()); + } + } + let union: Vec = owners.keys().map(|p| p.to_string()).collect(); + let chunks: Vec> = union + .chunks(batch_size.max(1)) + .map(<[String]>::to_vec) + .collect(); + let futures: Vec> = chunks + .iter() + .map(|chunk| -> BoxFuture<'_, _> { Box::pin(provider.search_patches_batch(chunk)) }) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + + let mut outcome = BatchOutcome::default(); + for root in root_purls.keys() { + outcome.roots.insert(root.clone(), RootBatch::default()); + } + for (chunk, result) in chunks.iter().zip(results) { + let chunk_roots: BTreeSet<&str> = chunk + .iter() + .flat_map(|p| owners.get(p.as_str()).into_iter().flatten().copied()) + .collect(); + match result { + Ok(response) => { + if response.can_access_paid_patches { + outcome.can_access_paid_patches = true; + } + for pkg in response.packages { + if pkg.patches.is_empty() { + continue; + } + let key = normalize_purl(strip_purl_qualifiers(&pkg.purl)).into_owned(); + let targets: Vec<&str> = match owners.get(key.as_str()) { + Some(roots) => roots.clone(), + None => chunk_roots.iter().copied().collect(), + }; + for root in targets { + if let Some(entry) = outcome.roots.get_mut(root) { + entry.packages.push(pkg.clone()); + } + } + } + } + Err(error) => { + let message = error.to_string(); + for purl in chunk { + for root in owners.get(purl.as_str()).into_iter().flatten() { + if let Some(entry) = outcome.roots.get_mut(*root) { + entry.failed_purls += 1; + entry.last_error = Some(message.clone()); + } + } + } + } + } + } + for entry in outcome.roots.values_mut() { + entry.packages.sort_by(|a, b| a.purl.cmp(&b.purl)); + } + outcome +} + +/// `search_patches_by_package` once per distinct purl. +pub(crate) async fn fetch_details( + provider: &Provider, + purls: &BTreeSet, +) -> BTreeMap> { + let ordered: Vec<&String> = purls.iter().collect(); + let futures: Vec> = ordered + .iter() + .map(|purl| -> BoxFuture<'_, _> { Box::pin(provider.search_patches_by_package(purl)) }) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + ordered + .into_iter() + .cloned() + .zip(results.into_iter().map(|r| r.map_err(|e| e.to_string()))) + .collect() +} + +/// The disk `--json` selection over one root's merged detail results: +/// accessible patches only, then the top-ranked patch per purl, sorted by +/// purl, as `(purl, uuid)`. +pub(crate) fn select_top_ranked( + results: &[PatchSearchResult], + can_access_paid: bool, +) -> Vec<(String, String)> { + let mut by_purl: BTreeMap<&str, Vec<&PatchSearchResult>> = BTreeMap::new(); + for patch in results + .iter() + .filter(|p| can_access_paid || p.tier == "free") + { + by_purl.entry(patch.purl.as_str()).or_default().push(patch); + } + by_purl + .into_iter() + .filter_map(|(purl, mut group)| { + group.sort_by(|a, b| cmp_search_results(a, b)); + group.first().map(|p| (purl.to_string(), p.uuid.clone())) + }) + .collect() +} + +/// Reference grants for every distinct uuid (`MAX_REFERENCE_BATCH` per +/// request): the merged results, plus the uuids whose request failed. +pub(crate) async fn fetch_references( + provider: &Provider, + uuids: &BTreeSet, +) -> ( + HashMap, + BTreeMap, +) { + let ordered: Vec = uuids.iter().cloned().collect(); + let chunks: Vec> = ordered + .chunks(MAX_REFERENCE_BATCH) + .map(<[String]>::to_vec) + .collect(); + let futures: Vec> = chunks + .iter() + .map(|chunk| -> BoxFuture<'_, _> { Box::pin(provider.fetch_registry_references(chunk)) }) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + let mut merged: HashMap = HashMap::new(); + let mut failed: BTreeMap = BTreeMap::new(); + for (chunk, result) in chunks.iter().zip(results) { + match result { + Ok(map) => merged.extend(map), + Err(error) => { + let message = error.to_string(); + for uuid in chunk { + failed.insert(uuid.clone(), message.clone()); + } + } + } + } + (merged, failed) +} + +/// Hosted wheel metadata once per distinct `(url, sha256)`: the disk +/// flow's `fetch_hosted_wheel_metadata` over the provider. +pub(crate) async fn fetch_wheel_metadata( + provider: &Provider, + wanted: &BTreeSet<(String, String)>, + max_bytes: u64, +) -> BTreeMap, String>> { + let ordered: Vec<&(String, String)> = wanted.iter().collect(); + let futures: Vec> = ordered + .iter() + .map( + |(url, sha256)| -> BoxFuture<'_, Result, String>> { + Box::pin(async move { + let bytes = provider + .download_artifact(url, max_bytes) + .await + .map_err(|error| format!("cannot fetch hosted wheel metadata: {error}"))?; + socket_patch_core::vendor::pypi::decode_hosted_wheel_metadata(&bytes, sha256) + }) + }, + ) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + ordered + .into_iter() + .map(|(url, _)| url.clone()) + .zip(results) + .collect() +} + +/// Patch views for every distinct confirmed uuid (wet runs only). +pub(crate) async fn fetch_records( + provider: &Provider, + uuids: &BTreeSet, +) -> BTreeMap> { + let ordered: Vec<&String> = uuids.iter().collect(); + let futures: Vec> = ordered + .iter() + .map(|uuid| -> BoxFuture<'_, _> { Box::pin(provider.fetch_patch(uuid)) }) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + ordered + .into_iter() + .cloned() + .zip(results.into_iter().map(|r| r.ok().flatten())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[tokio::test] + async fn join_bounded_keeps_order_and_caps_concurrency() { + let in_flight = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let futures: Vec> = (0..20usize) + .map(|i| -> BoxFuture<'static, usize> { + let in_flight = in_flight.clone(); + let peak = peak.clone(); + Box::pin(async move { + let now = in_flight.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis((20 - i as u64) % 7)).await; + in_flight.fetch_sub(1, Ordering::SeqCst); + i + }) + }) + .collect(); + let out = join_bounded(futures, 3).await; + assert_eq!(out, (0..20).collect::>()); + assert!(peak.load(Ordering::SeqCst) <= 3); + } + + #[test] + fn supplement_purl_matches_the_disk_shape_rule() { + assert_eq!( + supplement_purl("pkg:npm/%40scope/x@1.0.0?arch=x").as_deref(), + Some("pkg:npm/@scope/x@1.0.0") + ); + assert_eq!(supplement_purl("pkg:npm/x"), None); + assert_eq!(supplement_purl("npm/x@1"), None); + } + + fn result(purl: &str, uuid: &str, tier: &str, severity: &str) -> PatchSearchResult { + serde_json::from_value(serde_json::json!({ + "uuid": uuid, "purl": purl, "publishedAt": "2024-01-01T00:00:00Z", + "description": "", "license": "MIT", "tier": tier, + "vulnerabilities": {"GHSA-x": {"cves": [], "summary": "", "severity": severity, "description": ""}} + })) + .unwrap() + } + + #[test] + fn selection_filters_paid_and_takes_the_top_ranked() { + let results = vec![ + result("pkg:npm/b@1", "b-low", "free", "low"), + result("pkg:npm/b@1", "b-crit", "free", "critical"), + result("pkg:npm/a@1", "a-paid", "paid", "critical"), + result("pkg:npm/a@1", "a-free", "free", "low"), + ]; + assert_eq!( + select_top_ranked(&results, false), + vec![ + ("pkg:npm/a@1".to_string(), "a-free".to_string()), + ("pkg:npm/b@1".to_string(), "b-crit".to_string()) + ] + ); + assert_eq!(select_top_ranked(&results, true)[0].1, "a-paid"); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/ledger.rs b/crates/socket-patch-cli/src/hosted_memory/ledger.rs new file mode 100644 index 000000000..9a876a0c9 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/ledger.rs @@ -0,0 +1,177 @@ +//! The redirect ledger (`.socket/vendor/redirect-state.json`) in memory: +//! loaded strictly (a malformed ledger is a project error, never a fresh +//! start), merged exactly like the disk flow (edits appended unless already +//! recorded, `REBASE_KINDS` rebased, records extended newest-wins), and +//! serialized with the disk writer's bytes (`to_vec_pretty` + `\n`). + +use std::collections::BTreeMap; + +use socket_patch_core::manifest::schema::PatchRecord; +use socket_patch_core::patch::redirect::{ + CorruptRedirectState, FileEdit, RedirectState, REDIRECT_STATE_REL, +}; +use socket_patch_core::vendor::lock_inventory::{MemoryEntry, MemoryProject}; + +use crate::commands::scan::hosted::REBASE_KINDS; + +/// Load the project's ledger: `Ok(None)` when absent, `Err` (the disk +/// message) when present but unreadable or malformed. +pub(crate) fn load(project: &MemoryProject, root: &str) -> Result, String> { + let path = super::roots::join_root(root, REDIRECT_STATE_REL); + let corrupt = |detail: String, unreadable: bool| { + CorruptRedirectState { + path: path.clone().into(), + detail, + quarantined_to: None, + unreadable, + } + .to_string() + }; + let bytes: &[u8] = match project.get(REDIRECT_STATE_REL) { + None => return Ok(None), + Some(MemoryEntry::Text(text)) => text.as_bytes(), + Some(MemoryEntry::Binary(bytes)) => bytes, + Some(MemoryEntry::Present) => { + return Err(corrupt("file content was not provided".into(), true)) + } + Some(MemoryEntry::Symlink) => return Err(corrupt("is a symbolic link".into(), true)), + }; + serde_json::from_slice(bytes) + .map(Some) + .map_err(|e| corrupt(format!("invalid JSON: {e}"), false)) +} + +/// Merge this run's `edits` and `records` into `ledger` (the disk flow's +/// merge, verbatim). `files` are the pre-rewrite candidate contents the +/// rebase drift check reads. +pub(crate) fn merge( + ledger: &mut RedirectState, + edits: &[FileEdit], + records: BTreeMap, + files: &BTreeMap, +) { + ledger.mode = "hosted".to_string(); + let mut rebased: Vec = Vec::new(); + for edit in edits + .iter() + .filter(|e| REBASE_KINDS.contains(&e.kind.as_str())) + { + let siblings: Vec = ledger + .edits + .iter() + .enumerate() + .filter(|(_, old)| { + old.path == edit.path && old.kind == edit.kind && old.key == edit.key + }) + .map(|(i, _)| i) + .collect(); + let before = files.get(&edit.path).map(String::as_str).unwrap_or(""); + let drifted = !siblings.is_empty() + && siblings.iter().all(|&i| { + ledger.edits[i] + .new + .as_ref() + .and_then(serde_json::Value::as_str) + .is_none_or(|new| !before.contains(new)) + }); + if !drifted { + continue; + } + let nth = edits + .iter() + .filter(|e| e.path == edit.path && e.kind == edit.kind && e.key == edit.key) + .position(|e| std::ptr::eq(e, edit)) + .unwrap_or(0); + if let Some(&target) = siblings.get(nth) { + if !rebased.contains(&target) { + if edit.kind == "redirect_pdm_lock_package" { + ledger.edits[target].original = edit.original.clone(); + } + ledger.edits[target].new = edit.new.clone(); + ledger.edits[target].action = edit.action.clone(); + rebased.push(target); + } + } + } + let recorded = ledger.edits.len(); + for edit in edits { + let is_rebased = REBASE_KINDS.contains(&edit.kind.as_str()) + && rebased.iter().any(|&t| { + let old = &ledger.edits[t]; + old.path == edit.path + && old.kind == edit.kind + && old.key == edit.key + && old.new == edit.new + }); + if !is_rebased && !ledger.edits[..recorded].contains(edit) { + ledger.edits.push(edit.clone()); + } + } + ledger.records.extend(records); +} + +/// The ledger's on-disk bytes. +pub(crate) fn serialize(ledger: &RedirectState) -> Result { + let mut bytes = serde_json::to_vec_pretty(ledger).map_err(|e| e.to_string())?; + bytes.push(b'\n'); + String::from_utf8(bytes).map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn edit(kind: &str, new: &str) -> FileEdit { + FileEdit { + path: "poetry.lock".into(), + kind: kind.into(), + action: "replaced".into(), + key: Some("k".into()), + original: Some(serde_json::json!("orig")), + new: Some(serde_json::json!(new)), + } + } + + #[test] + fn corrupt_and_absent_ledgers() { + let mut p = MemoryProject::new(); + assert!(load(&p, "").unwrap().is_none()); + p.insert_text(REDIRECT_STATE_REL, "{not json"); + let err = load(&p, "sub").unwrap_err(); + assert!( + err.contains("sub/.socket/vendor/redirect-state.json"), + "{err}" + ); + assert!(err.contains("malformed"), "{err}"); + p.insert_symlink(REDIRECT_STATE_REL); + assert!(load(&p, "").unwrap_err().contains("cannot be read")); + } + + #[test] + fn merge_appends_new_edits_and_rebases_drifted_fragments() { + let mut ledger = RedirectState::new(); + ledger.edits.push(edit("redirect_npm_lock_entry", "a")); + ledger + .edits + .push(edit("redirect_poetry_lock_package", "old-new")); + let files = BTreeMap::from([("poetry.lock".to_string(), "relocked".to_string())]); + merge( + &mut ledger, + &[ + edit("redirect_npm_lock_entry", "a"), + edit("redirect_npm_lock_entry", "b"), + edit("redirect_poetry_lock_package", "fresh"), + ], + BTreeMap::new(), + &files, + ); + let news: Vec<&str> = ledger + .edits + .iter() + .map(|e| e.new.as_ref().and_then(|v| v.as_str()).unwrap()) + .collect(); + assert_eq!(news, vec!["a", "fresh", "b"]); + let text = serialize(&ledger).unwrap(); + assert!(text.ends_with("}\n")); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/limits.rs b/crates/socket-patch-cli/src/hosted_memory/limits.rs new file mode 100644 index 000000000..4bcdb6359 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/limits.rs @@ -0,0 +1,409 @@ +//! Option validation and the streaming session builder: every size and +//! count limit on what a host may hand the engine is enforced here, before +//! any parser sees a byte. + +use std::collections::BTreeMap; + +use super::select::{is_binary_candidate, safe_repo_path, safe_root_path}; +use super::types::{ + EngineError, EngineWarning, HostedScanInput, HostedScanOptions, InputFile, MarkKind, + PresentKind, ResolvedLimits, DEFAULT_BATCH_SIZE, DEFAULT_PROVIDER_CONCURRENCY, + DEFAULT_REQUEST_TIMEOUT_MS, ECOSYSTEMS, MAX_BATCH_SIZE, +}; + +/// Upper bound on concurrent provider calls, whatever the host asks for. +const MAX_PROVIDER_CONCURRENCY: u32 = 64; + +/// The options every engine entry point validates, with defaults applied. +#[derive(Debug, Clone)] +pub(crate) struct ResolvedOptions { + pub(crate) ecosystems: Option>, + pub(crate) batch_size: usize, + pub(crate) dry_run: bool, + pub(crate) pipenv_major: Option, + pub(crate) trust_lockfile_config: bool, + pub(crate) npm_allow_remote_config: bool, + pub(crate) project_roots: Option>, + pub(crate) provider_concurrency: usize, + pub(crate) request_timeout: std::time::Duration, + pub(crate) limits: ResolvedLimits, +} + +pub(crate) fn resolve_options(options: &HostedScanOptions) -> Result { + if options.org_slug.trim().is_empty() { + return Err(EngineError::invalid( + "invalid_org_slug", + "orgSlug must be a non-empty string", + )); + } + let batch_size = options.batch_size.unwrap_or(DEFAULT_BATCH_SIZE); + if !(1..=MAX_BATCH_SIZE).contains(&batch_size) { + return Err(EngineError::invalid( + "invalid_batch_size", + format!("batchSize must be between 1 and {MAX_BATCH_SIZE}"), + )); + } + if let Some(ecosystems) = &options.ecosystems { + if let Some(bad) = ecosystems + .iter() + .find(|e| !ECOSYSTEMS.contains(&e.as_str())) + { + return Err(EngineError::invalid( + "invalid_ecosystem", + format!("unknown ecosystem `{bad}`"), + )); + } + } + let project_roots = match &options.project_roots { + Some(roots) => { + let mut out: Vec = Vec::with_capacity(roots.len()); + for root in roots { + let normalized = safe_root_path(root).ok_or_else(|| { + EngineError::invalid( + "invalid_project_root", + format!( + "projectRoots entry `{root}` is not a safe repo-relative directory" + ), + ) + })?; + out.push(normalized); + } + out.sort(); + out.dedup(); + Some(out) + } + None => None, + }; + let provider_concurrency = options + .provider_concurrency + .unwrap_or(DEFAULT_PROVIDER_CONCURRENCY); + if provider_concurrency == 0 { + return Err(EngineError::invalid( + "invalid_provider_concurrency", + "providerConcurrency must be at least 1", + )); + } + let timeout_ms = options + .request_timeout_ms + .unwrap_or(DEFAULT_REQUEST_TIMEOUT_MS); + if timeout_ms == 0 { + return Err(EngineError::invalid( + "invalid_request_timeout", + "requestTimeoutMs must be at least 1", + )); + } + Ok(ResolvedOptions { + ecosystems: options.ecosystems.clone(), + batch_size: batch_size as usize, + dry_run: options.dry_run, + pipenv_major: options.pipenv_major, + trust_lockfile_config: options.trust_lockfile_config.unwrap_or(true), + npm_allow_remote_config: options.npm_allow_remote_config.unwrap_or(true), + project_roots, + provider_concurrency: provider_concurrency.min(MAX_PROVIDER_CONCURRENCY) as usize, + request_timeout: std::time::Duration::from_millis(timeout_ms), + limits: options.limits.clone().unwrap_or_default().resolve(), + }) +} + +/// Accumulates a host's streamed files into a [`HostedScanInput`], +/// enforcing `maxFiles`, `maxFileBytes` and `maxTotalBytes` on every chunk. +/// `bun.lockb` is kept as bytes; every other file must be UTF-8 (a file +/// that is not is kept as present-but-unreadable with an `invalid_utf8` +/// warning, exactly like an unreadable file on disk). +#[derive(Debug)] +pub struct SessionBuilder { + options: HostedScanOptions, + limits: ResolvedLimits, + open: BTreeMap>, + files: BTreeMap, + total_bytes: u64, + warnings: Vec, +} + +impl SessionBuilder { + pub fn new(options: HostedScanOptions) -> Result { + let resolved = resolve_options(&options)?; + Ok(Self { + options, + limits: resolved.limits, + open: BTreeMap::new(), + files: BTreeMap::new(), + total_bytes: 0, + warnings: Vec::new(), + }) + } + + pub fn options(&self) -> &HostedScanOptions { + &self.options + } + + pub fn total_bytes(&self) -> u64 { + self.total_bytes + } + + fn checked_path(&self, path: &str) -> Result { + let normalized = safe_repo_path(path).ok_or_else(|| { + EngineError::invalid( + "invalid_path", + format!("`{path}` is not a safe repo-relative path"), + ) + })?; + if self.files.contains_key(&normalized) { + return Err(EngineError::invalid( + "duplicate_path", + format!("`{normalized}` was already provided"), + )); + } + Ok(normalized) + } + + fn check_file_count(&self, adding_new: bool) -> Result<(), EngineError> { + let count = (self.files.len() + self.open.len() + usize::from(adding_new)) as u64; + if count > self.limits.max_files { + return Err(EngineError::limit( + "max_files", + format!("more than {} files were provided", self.limits.max_files), + )); + } + Ok(()) + } + + /// Append `chunk` to `path`'s content. + pub fn push_chunk(&mut self, path: &str, chunk: &[u8]) -> Result<(), EngineError> { + let path = self.checked_path(path)?; + let is_new = !self.open.contains_key(&path); + self.check_file_count(is_new)?; + let current = self.open.get(&path).map_or(0, Vec::len) as u64; + let len = chunk.len() as u64; + if current + len > self.limits.max_file_bytes { + return Err(EngineError::limit( + "max_file_bytes", + format!( + "`{path}` exceeds the {}-byte per-file limit", + self.limits.max_file_bytes + ), + )); + } + if self.total_bytes + len > self.limits.max_total_bytes { + return Err(EngineError::limit( + "max_total_bytes", + format!( + "the session exceeds the {}-byte total limit", + self.limits.max_total_bytes + ), + )); + } + self.total_bytes += len; + self.open.entry(path).or_default().extend_from_slice(chunk); + Ok(()) + } + + /// Complete `path` (a path with no chunks is an empty file). + pub fn end_file(&mut self, path: &str) -> Result<(), EngineError> { + let path = self.checked_path(path)?; + let bytes = match self.open.remove(&path) { + Some(bytes) => bytes, + None => { + self.check_file_count(true)?; + Vec::new() + } + }; + let file = if is_binary_candidate(&path) { + InputFile::Binary(bytes) + } else { + match String::from_utf8(bytes) { + Ok(text) => InputFile::Text(text), + Err(_) => { + self.warnings.push(EngineWarning::new( + "invalid_utf8", + format!("{path} is not valid UTF-8 and was treated as unreadable"), + None, + )); + InputFile::Present(PresentKind::BinarySkipped) + } + } + }; + self.files.insert(path, file); + Ok(()) + } + + /// Record that `path` exists without providing its content. + pub fn mark_present(&mut self, path: &str, kind: MarkKind) -> Result<(), EngineError> { + let path = self.checked_path(path)?; + if self.open.contains_key(&path) { + return Err(EngineError::invalid( + "duplicate_path", + format!("`{path}` already has streamed content"), + )); + } + self.check_file_count(true)?; + let file = match kind { + MarkKind::Symlink => InputFile::Symlink, + MarkKind::Present(kind) => { + let code = match kind { + PresentKind::Present => None, + PresentKind::BinarySkipped => Some("binary_skipped"), + PresentKind::Oversize => Some("file_oversize"), + PresentKind::LfsPointer => Some("lfs_pointer"), + }; + if let Some(code) = code { + self.warnings.push(EngineWarning::new( + code, + format!("{path} was not provided ({code}) and was treated as unreadable"), + None, + )); + } + InputFile::Present(kind) + } + }; + self.files.insert(path, file); + Ok(()) + } + + /// Whole-file convenience over [`Self::push_chunk`] + [`Self::end_file`]. + pub fn add_text(&mut self, path: &str, text: &str) -> Result<(), EngineError> { + self.push_chunk(path, text.as_bytes())?; + self.end_file(path) + } + + /// Whole-file convenience for binary content. + pub fn add_binary(&mut self, path: &str, bytes: &[u8]) -> Result<(), EngineError> { + let normalized = self.checked_path(path)?; + self.push_chunk(&normalized, bytes)?; + let bytes = self.open.remove(&normalized).unwrap_or_default(); + self.files.insert(normalized, InputFile::Binary(bytes)); + Ok(()) + } + + /// The finished input. Fails when a streamed file was never ended. + pub fn finish(self) -> Result { + if let Some(path) = self.open.keys().next() { + return Err(EngineError::invalid( + "unterminated_file", + format!("`{path}` received chunks but endFile was never called"), + )); + } + Ok(HostedScanInput { + options: self.options, + files: self.files, + warnings: self.warnings, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hosted_memory::types::HostedScanLimits; + + fn options(limits: HostedScanLimits) -> HostedScanOptions { + HostedScanOptions { + org_slug: "org".into(), + limits: Some(limits), + ..HostedScanOptions::default() + } + } + + #[test] + fn rejects_bad_options() { + let mut o = options(HostedScanLimits::default()); + o.batch_size = Some(0); + assert_eq!( + resolve_options(&o).unwrap_err().code(), + "invalid_batch_size" + ); + o.batch_size = Some(501); + assert_eq!( + resolve_options(&o).unwrap_err().code(), + "invalid_batch_size" + ); + o.batch_size = None; + o.ecosystems = Some(vec!["deno".into()]); + assert_eq!(resolve_options(&o).unwrap_err().code(), "invalid_ecosystem"); + o.ecosystems = None; + o.project_roots = Some(vec!["../x".into()]); + assert_eq!( + resolve_options(&o).unwrap_err().code(), + "invalid_project_root" + ); + o.project_roots = None; + o.org_slug = " ".into(); + assert_eq!(resolve_options(&o).unwrap_err().code(), "invalid_org_slug"); + } + + #[test] + fn enforces_session_limits() { + let mut b = SessionBuilder::new(options(HostedScanLimits { + max_file_bytes: Some(4), + max_total_bytes: Some(6), + max_files: Some(2), + ..HostedScanLimits::default() + })) + .unwrap(); + b.push_chunk("a", b"12").unwrap(); + b.push_chunk("a", b"34").unwrap(); + assert_eq!( + b.push_chunk("a", b"5").unwrap_err().code(), + "max_file_bytes" + ); + b.end_file("a").unwrap(); + b.push_chunk("b", b"12").unwrap(); + assert_eq!( + b.push_chunk("b", b"3").unwrap_err().code(), + "max_total_bytes" + ); + b.end_file("b").unwrap(); + assert_eq!(b.end_file("c").unwrap_err().code(), "max_files"); + assert_eq!( + b.push_chunk("a", b"x").unwrap_err().code(), + "duplicate_path" + ); + assert_eq!( + b.push_chunk("../evil", b"x").unwrap_err().code(), + "invalid_path" + ); + } + + #[test] + fn classifies_finished_files() { + let mut b = SessionBuilder::new(options(HostedScanLimits::default())).unwrap(); + b.add_text("package-lock.json", "{}").unwrap(); + b.push_chunk("bun.lockb", &[0xff, 0x00]).unwrap(); + b.end_file("bun.lockb").unwrap(); + b.push_chunk("yarn.lock", &[0xff, 0xfe]).unwrap(); + b.end_file("yarn.lock").unwrap(); + b.mark_present(".pnp.cjs", MarkKind::Present(PresentKind::Present)) + .unwrap(); + b.mark_present(".npmrc", MarkKind::Symlink).unwrap(); + b.push_chunk("open.txt", b"x").unwrap(); + assert_eq!(b.finish().unwrap_err().code(), "unterminated_file"); + } + + #[test] + fn finished_input_keeps_every_kind() { + let mut b = SessionBuilder::new(options(HostedScanLimits::default())).unwrap(); + b.add_text("package-lock.json", "{}").unwrap(); + b.push_chunk("bun.lockb", &[0xff, 0x00]).unwrap(); + b.end_file("bun.lockb").unwrap(); + b.push_chunk("yarn.lock", &[0xff, 0xfe]).unwrap(); + b.end_file("yarn.lock").unwrap(); + b.mark_present(".npmrc", MarkKind::Symlink).unwrap(); + let input = b.finish().unwrap(); + assert_eq!( + input.files.get("package-lock.json"), + Some(&InputFile::Text("{}".into())) + ); + assert_eq!( + input.files.get("bun.lockb"), + Some(&InputFile::Binary(vec![0xff, 0x00])) + ); + assert_eq!( + input.files.get("yarn.lock"), + Some(&InputFile::Present(PresentKind::BinarySkipped)) + ); + assert_eq!(input.files.get(".npmrc"), Some(&InputFile::Symlink)); + assert_eq!(input.warnings.len(), 1); + assert_eq!(input.warnings[0].code, "invalid_utf8"); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/mod.rs b/crates/socket-patch-cli/src/hosted_memory/mod.rs new file mode 100644 index 000000000..b5f8b0f3d --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/mod.rs @@ -0,0 +1,1113 @@ +//! The hosted-mode redirect engine over an in-memory repository: no +//! filesystem, no subprocesses, no environment reads, no telemetry. Every +//! patch lookup goes through the caller's [`PatchApi`]; the caller hands +//! in the repository's candidate files (chosen by [`select_paths`]) and +//! gets back the changed files, ledger included. +//! +//! Per project root the result matches `scan --mode hosted --json` over a +//! checkout holding the same files (the parity tests hold the two paths to +//! it), with these differences by design: patch lookups are shared across +//! roots (one call per purl / uuid / url), a 401/403 is a project error +//! instead of a public-proxy fallback, the Pipenv major is an option +//! instead of a `pipenv --version` probe, the npm allow-remote planner sees +//! no user/global npm config, and a vendored→hosted takeover is refused +//! (`vendored_takeover_unsupported_in_memory`). Maven and NuGet have no +//! lockfile inventory (disk discovers them only through installed-tree +//! crawlers), so their files raise `ecosystem_unsupported_in_memory`. A dry +//! run still requests reference grants, exactly like the disk dry run. +//! +//! Cargo member manifests are chosen by content on disk (`members`, +//! `exclude`, path dependencies and `[patch]` paths, anywhere but +//! `target/`), which a tree listing cannot evaluate. So [`select_paths`] +//! asks for every `Cargo.toml` under a Cargo root outside `target/` +//! segments (`vendor/`, `node_modules/` and dot-directories included) and +//! the engine repeats the disk walk over them; manifests the walk never +//! reaches are read and ignored. The one exception is `cargo vendor` +//! output: a manifest in or below a directory holding +//! `.cargo-checksum.json` is registry source the walk never reads, so it +//! is not fetched; a path dependency or `[patch]` path into such a crate +//! then fails closed (`redirect_cargo_transitive_dependents`). The cost is +//! over-fetch: a Rust repo with many committed fixture or example crates +//! streams all of them, and each counts toward `maxFiles` / +//! `maxTotalBytes`, so such a repo can fail with a `limit` error where the +//! disk run would succeed. Fetching fewer would instead silently drop +//! manifests the disk run pins. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; +use std::time::Instant; + +use socket_patch_core::api::client::PatchApi; +use socket_patch_core::api::types::{PatchResponse, PatchSearchResult}; +use socket_patch_core::crawlers::Ecosystem; +use socket_patch_core::manifest::schema::PatchRecord; +use socket_patch_core::patch::redirect::{RedirectState, REDIRECT_STATE_REL}; +use socket_patch_core::utils::cargo_workspace::member_manifests_in; +use socket_patch_core::vendor::lock_inventory::{ + inventory_project_diagnosed_in, MemoryEntry, MemoryProject, ProjectView, +}; +use tokio_util::sync::CancellationToken; + +pub(crate) mod discover; +pub(crate) mod ledger; +pub mod limits; +pub(crate) mod redirect; +pub(crate) mod roots; +pub mod select; +pub mod types; + +pub use limits::SessionBuilder; +pub use select::{candidate_files, safe_repo_path, select_paths}; +pub use types::*; + +use discover::Provider; +use redirect::{Planned, Refused, Rewritten, StageOptions}; + +/// `"+"`; the sha comes from the +/// `SOCKET_PATCH_GIT_SHA` build-time variable. +pub fn engine_version() -> String { + format!( + "{}+{}", + env!("CARGO_PKG_VERSION"), + option_env!("SOCKET_PATCH_GIT_SHA").unwrap_or("unknown") + ) +} + +/// Run the engine. Resolves with the result, or rejects with an +/// [`EngineError`]: a limit breach, unusable input, cancellation through +/// `cancel` (checked around every await and between roots in the CPU-bound +/// inventory / plan / rewrite phases — one root's running parser is not +/// interrupted), or an engine bug. Provider failures never reject; they +/// become project errors and warnings. +pub async fn run_in_memory( + input: HostedScanInput, + provider: Arc, + cancel: CancellationToken, +) -> Result { + tokio::select! { + biased; + _ = cancel.cancelled() => Err(EngineError::Cancelled), + result = engine(input, provider, cancel.clone()) => result, + } +} + +/// Yield to the runtime between roots of a CPU-bound phase (memory reads +/// never pend, so without it neither `cancel` nor other tasks on this +/// worker would run until the phase ends). +async fn checkpoint(cancel: &CancellationToken) -> Result<(), EngineError> { + tokio::task::yield_now().await; + if cancel.is_cancelled() { + return Err(EngineError::Cancelled); + } + Ok(()) +} + +/// One root's working state across the phases. +struct RootState { + root: String, + project: Option, + /// Root-relative paths that exist but whose content was not provided + /// (oversize, LFS pointers, presence-only): the disk flow would read + /// them, so a rewrite that depends on one is refused. + unreadable: BTreeSet, + ledger: Option, + purls: Vec, + summary: ProjectSummary, + packages: Vec, + selected: Vec<(String, String)>, + skipped: Vec, + error: Option, +} + +impl RootState { + fn fail(&mut self, code: &str, message: String) { + if self.error.is_none() { + self.error = Some(ProjectError { + code: code.to_string(), + message, + }); + } + } +} + +struct Phases { + at: Instant, + ms: BTreeMap, +} + +impl Phases { + fn mark(&mut self, name: &str) { + let now = Instant::now(); + self.ms.insert( + name.to_string(), + now.duration_since(self.at).as_millis() as u64, + ); + self.at = now; + } +} + +fn validate_input(input: &HostedScanInput, limits: &ResolvedLimits) -> Result { + if input.files.len() as u64 > limits.max_files { + return Err(EngineError::limit( + "max_files", + format!("more than {} files were provided", limits.max_files), + )); + } + let mut total = 0u64; + for (path, file) in &input.files { + if safe_repo_path(path).as_deref() != Some(path.as_str()) { + return Err(EngineError::invalid( + "invalid_path", + format!("`{path}` is not a safe repo-relative path"), + )); + } + let len = match file { + InputFile::Text(text) => text.len() as u64, + InputFile::Binary(bytes) => bytes.len() as u64, + InputFile::Present(_) | InputFile::Symlink => 0, + }; + if len > limits.max_file_bytes { + return Err(EngineError::limit( + "max_file_bytes", + format!( + "`{path}` exceeds the {}-byte per-file limit", + limits.max_file_bytes + ), + )); + } + total += len; + } + if total > limits.max_total_bytes { + return Err(EngineError::limit( + "max_total_bytes", + format!( + "the input exceeds the {}-byte total limit", + limits.max_total_bytes + ), + )); + } + Ok(total) +} + +/// One input file, stored once: every root's project shares its bytes. +struct SharedFile { + entry: MemoryEntry, + /// Exists, but its content (which disk would read) was not provided. + /// A non-UTF-8 text file is not: disk cannot read it either. + unreadable: bool, +} + +fn share(file: InputFile) -> SharedFile { + let (entry, unreadable) = match file { + InputFile::Text(text) => (MemoryEntry::Text(Arc::from(text)), false), + InputFile::Binary(bytes) => (MemoryEntry::Binary(Arc::from(bytes)), false), + InputFile::Present(kind) => (MemoryEntry::Present, kind != PresentKind::BinarySkipped), + InputFile::Symlink => (MemoryEntry::Symlink, false), + }; + SharedFile { entry, unreadable } +} + +/// The root-relative in-memory project for `root` and its unreadable +/// paths. +fn project_for( + root: &str, + files: &BTreeMap, +) -> (MemoryProject, BTreeSet) { + let mut project = MemoryProject::new(); + let mut unreadable = BTreeSet::new(); + for (path, file) in files { + let Some(rel) = roots::strip_root(root, path) else { + continue; + }; + if file.unreadable { + unreadable.insert(rel.to_string()); + } + project.insert(rel, file.entry.clone()); + } + (project, unreadable) +} + +/// A root that is a member of an enclosing root's Cargo workspace builds +/// through that workspace's Cargo.lock, which the enclosing root already +/// pins its manifest against; the member's own Cargo.lock (which cargo +/// ignores) is dropped so only the workspace root redirects cargo packages. +fn demote_cargo_members(states: &mut [RootState], warnings: &mut Vec) { + let mut member_of: BTreeMap = BTreeMap::new(); + for state in states.iter() { + let Some(project) = state.project.as_ref() else { + continue; + }; + if !project.contains("Cargo.lock") { + continue; + } + for rel in member_manifests_in(&ProjectView::Memory(project)) { + if let Some(dir) = rel.strip_suffix("/Cargo.toml") { + member_of + .entry(roots::join_root(&state.root, dir)) + .or_insert_with(|| state.root.clone()); + } + } + } + for state in states.iter_mut() { + let Some(owner) = member_of.get(&state.root) else { + continue; + }; + let Some(project) = state.project.as_mut() else { + continue; + }; + if project.remove("Cargo.lock").is_some() { + state.unreadable.remove("Cargo.lock"); + warnings.push(EngineWarning::new( + "cargo_member_lock_ignored", + format!( + "{} is a member of the Cargo workspace at `{owner}`, whose Cargo.lock \ + cargo builds it from; its own Cargo.lock was not scanned", + if state.root.is_empty() { + "." + } else { + state.root.as_str() + } + ), + Some(&state.root), + )); + } + } +} + +fn ecosystem_allowed(ecosystems: Option<&[String]>, purl: &str) -> bool { + match ecosystems { + None => true, + Some(list) => { + Ecosystem::from_purl(purl).is_some_and(|eco| list.iter().any(|a| a == eco.cli_name())) + } + } +} + +fn unsupported_ecosystem_warnings( + root: &str, + project: &MemoryProject, + ecosystems: Option<&[String]>, + out: &mut Vec, +) { + let allowed = |eco: &str| ecosystems.is_none_or(|list| list.iter().any(|e| e == eco)); + for (eco, markers) in roots::UNSUPPORTED_MARKERS { + if !allowed(eco) { + continue; + } + if let Some(found) = markers.iter().find(|m| project.contains(m)) { + out.push(EngineWarning::new( + "ecosystem_unsupported_in_memory", + format!( + "{found} is present, but {eco} dependencies are discovered only from an \ + installed tree, which the in-memory hosted scan does not have; {eco} \ + dependencies were not scanned" + ), + Some(root), + )); + } + } +} + +/// Maven / NuGet marker files outside every root (a repo with only those +/// has no root at all): one run-level warning per ecosystem. +fn unrooted_unsupported_warnings<'a>( + paths: impl Iterator, + roots: &[String], + ecosystems: Option<&[String]>, + out: &mut Vec, +) { + let root_set: BTreeSet<&str> = roots.iter().map(String::as_str).collect(); + let mut found: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for path in paths { + let (dir, base) = roots::split_path(path); + let Some(&(eco, _)) = roots::UNSUPPORTED_MARKERS + .iter() + .find(|(_, markers)| markers.contains(&base)) + else { + continue; + }; + if root_set.contains(dir) + || !ecosystems.is_none_or(|list| list.iter().any(|e| e == eco)) + || dir + .split('/') + .any(|seg| roots::EXCLUDED_ROOT_SEGMENTS.contains(&seg)) + { + continue; + } + found.entry(eco).or_default().push(path); + } + for (eco, paths) in found { + let more = match paths.len() { + 1 => String::new(), + n => format!(" (and {} more)", n - 1), + }; + out.push(EngineWarning::new( + "ecosystem_unsupported_in_memory", + format!( + "{}{more} is present, but {eco} dependencies are discovered only from an \ + installed tree, which the in-memory hosted scan does not have; {eco} \ + dependencies were not scanned", + paths[0] + ), + None, + )); + } +} + +async fn engine( + input: HostedScanInput, + api: Arc, + cancel: CancellationToken, +) -> Result { + let options = limits::resolve_options(&input.options)?; + let bytes_input = validate_input(&input, &options.limits)?; + let files_input = input.files.len() as u64; + let files: BTreeMap = input + .files + .into_iter() + .map(|(path, file)| (path, share(file))) + .collect(); + let mut phases = Phases { + at: Instant::now(), + ms: BTreeMap::new(), + }; + let mut warnings: Vec = input.warnings; + let ecosystems = options.ecosystems.as_deref(); + let provider = Provider::new(api, options.request_timeout, options.provider_concurrency); + + let root_list: Vec = match &options.project_roots { + Some(roots) => roots.clone(), + None => roots::detect_roots(files.keys().map(String::as_str), ecosystems).0, + }; + if root_list.len() as u64 > options.limits.max_projects { + return Err(EngineError::limit( + "max_projects", + format!( + "{} project roots exceed the {}-project limit", + root_list.len(), + options.limits.max_projects + ), + )); + } + unrooted_unsupported_warnings( + files.keys().map(String::as_str), + &root_list, + ecosystems, + &mut warnings, + ); + let mut states: Vec = root_list + .iter() + .map(|root| { + let (project, unreadable) = project_for(root, &files); + (root, project, unreadable) + }) + .map(|(root, project, unreadable)| RootState { + root: root.clone(), + project: Some(project), + unreadable, + ledger: None, + purls: Vec::new(), + summary: ProjectSummary::default(), + packages: Vec::new(), + selected: Vec::new(), + skipped: Vec::new(), + error: None, + }) + .collect(); + drop(files); + demote_cargo_members(&mut states, &mut warnings); + phases.mark("roots"); + + for state in &mut states { + checkpoint(&cancel).await?; + let Some(project) = state.project.as_ref() else { + continue; + }; + match ledger::load(project, &state.root) { + Ok(loaded) => state.ledger = loaded, + Err(message) => { + state.fail("corrupt_ledger", message); + continue; + } + } + let (entries, unsupported) = + inventory_project_diagnosed_in(&ProjectView::Memory(project)).await; + for (code, detail) in crate::commands::scan::unsupported_layout_warnings(&unsupported) { + warnings.push(EngineWarning::new(code, detail, Some(&state.root))); + } + unsupported_ecosystem_warnings(&state.root, project, ecosystems, &mut warnings); + let purls: BTreeSet = entries + .iter() + .filter_map(|e| discover::supplement_purl(&e.purl)) + .filter(|p| ecosystem_allowed(ecosystems, p)) + .collect(); + state.purls = purls.into_iter().collect(); + state.summary.scanned_packages = state.purls.len() as u64; + } + let union_purls: BTreeSet<&str> = states + .iter() + .filter(|s| s.error.is_none()) + .flat_map(|s| s.purls.iter().map(String::as_str)) + .collect(); + if union_purls.len() as u64 > options.limits.max_purls { + return Err(EngineError::limit( + "max_purls", + format!( + "{} distinct packages exceed the {}-package limit", + union_purls.len(), + options.limits.max_purls + ), + )); + } + phases.mark("inventory"); + + let root_purls: BTreeMap> = states + .iter() + .filter(|s| s.error.is_none() && !s.purls.is_empty()) + .map(|s| (s.root.clone(), s.purls.clone())) + .collect(); + let batch = discover::batch_search(&provider, &root_purls, options.batch_size).await; + let can_access_paid = batch.can_access_paid_patches; + for state in states.iter_mut().filter(|s| s.error.is_none()) { + state.summary.can_access_paid_patches = can_access_paid; + let Some(outcome) = batch.roots.get(&state.root) else { + continue; + }; + let error = outcome + .last_error + .clone() + .unwrap_or_else(|| "all batches failed".to_string()); + if outcome.failed_purls > 0 && outcome.failed_purls >= state.purls.len() { + state.fail("patch_lookup_failed", error); + continue; + } + if outcome.failed_purls > 0 { + warnings.push(EngineWarning::new( + "batch_failed", + format!( + "{} of {} packages could not be queried for patches: {error}", + outcome.failed_purls, + state.purls.len() + ), + Some(&state.root), + )); + } + state.packages = outcome.packages.clone(); + state.summary.packages_with_patches = state.packages.len() as u64; + for pkg in &state.packages { + for patch in &pkg.patches { + if patch.tier == "free" { + state.summary.free_patches += 1; + } else { + state.summary.paid_patches += 1; + } + } + } + state.summary.total_patches = state.summary.free_patches + state.summary.paid_patches; + } + phases.mark("batch"); + + let detail_purls: BTreeSet = states + .iter() + .filter(|s| s.error.is_none()) + .flat_map(|s| s.packages.iter().map(|p| p.purl.clone())) + .collect(); + let details = discover::fetch_details(&provider, &detail_purls).await; + for state in states.iter_mut().filter(|s| s.error.is_none()) { + if state.packages.is_empty() { + continue; + } + let mut results: Vec = Vec::new(); + let mut failures: Vec = Vec::new(); + for pkg in &state.packages { + match details.get(&pkg.purl) { + Some(Ok(response)) => results.extend(response.patches.iter().cloned()), + Some(Err(error)) => failures.push(error.clone()), + None => failures.push("patch details were not fetched".to_string()), + } + } + if !failures.is_empty() && failures.len() == state.packages.len() { + let last = failures.last().cloned().unwrap_or_default(); + state.fail( + "patch_lookup_failed", + format!("all {} patch-detail queries failed: {last}", failures.len()), + ); + continue; + } + if !failures.is_empty() { + warnings.push(EngineWarning::new( + "detail_lookup_failed", + format!( + "patch details could not be fetched for {} of {} packages", + failures.len(), + state.packages.len() + ), + Some(&state.root), + )); + } + state.selected = discover::select_top_ranked(&results, can_access_paid); + } + phases.mark("details"); + + let uuids: BTreeSet = states + .iter() + .filter(|s| s.error.is_none()) + .flat_map(|s| s.selected.iter().map(|(_, u)| u.clone())) + .collect(); + let (references, failed_refs) = if uuids.is_empty() { + (HashMap::new(), BTreeMap::new()) + } else { + discover::fetch_references(&provider, &uuids).await + }; + for state in states.iter_mut().filter(|s| s.error.is_none()) { + if let Some(error) = state + .selected + .iter() + .find_map(|(_, uuid)| failed_refs.get(uuid)) + { + state.fail( + "reference_lookup_failed", + format!("failed to resolve patch references: {error}"), + ); + } + } + phases.mark("references"); + + let mut planned: Vec<(usize, Planned)> = Vec::new(); + for (index, state) in states.iter_mut().enumerate() { + if state.error.is_some() { + continue; + } + checkpoint(&cancel).await?; + let Some(project) = state.project.take() else { + continue; + }; + let unreadable = std::mem::take(&mut state.unreadable); + match redirect::plan(project, unreadable, &state.selected, &references) { + Ok(plan) => planned.push((index, plan)), + Err(Refused { error }) => state.error = Some(error), + } + } + let wheels: BTreeSet<(String, String)> = planned + .iter() + .flat_map(|(_, p)| p.wheels.iter().cloned()) + .collect(); + let wheel_metadata = if wheels.is_empty() { + BTreeMap::new() + } else { + discover::fetch_wheel_metadata(&provider, &wheels, options.limits.max_artifact_bytes).await + }; + phases.mark("plan"); + + let stage = StageOptions { + dry_run: options.dry_run, + pipenv_major: options.pipenv_major, + trust_lockfile_config: options.trust_lockfile_config, + npm_allow_remote_config: options.npm_allow_remote_config, + }; + let mut rewritten: Vec<(usize, Rewritten)> = Vec::new(); + for (index, plan) in planned { + checkpoint(&cancel).await?; + let skipped_before = plan.skipped.clone(); + match redirect::rewrite(plan, &wheel_metadata, stage) { + Ok(done) => rewritten.push((index, done)), + Err(Refused { error }) => { + states[index].skipped = skipped_before; + states[index].error = Some(error); + } + } + } + phases.mark("rewrite"); + + let record_uuids: BTreeSet = if options.dry_run { + BTreeSet::new() + } else { + rewritten + .iter() + .flat_map(|(_, r)| r.confirmed.iter().map(|(_, u)| u.clone())) + .collect() + }; + let records: BTreeMap> = if record_uuids.is_empty() { + BTreeMap::new() + } else { + discover::fetch_records(&provider, &record_uuids).await + }; + phases.mark("records"); + + let mut changed: BTreeMap = BTreeMap::new(); + let mut changed_binary: BTreeMap)> = BTreeMap::new(); + let mut results: BTreeMap = BTreeMap::new(); + for (index, done) in rewritten { + let state = &mut states[index]; + let result = finish_root( + state, + done, + &records, + options.dry_run, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + results.insert(index, result); + } + let mut projects: Vec = Vec::with_capacity(states.len()); + for (index, state) in states.iter().enumerate() { + if let Some(result) = results.remove(&index) { + projects.push(result); + continue; + } + let redirect = match &state.error { + Some(_) => serde_json::json!({ "mode": "hosted" }), + None => crate::commands::scan::hosted::redirect_json_block( + 0, + Vec::new(), + Vec::new(), + Vec::new(), + options.dry_run, + ), + }; + projects.push(ProjectResult { + root: state.root.clone(), + redirect, + summary: state.summary.clone(), + redirected: Vec::new(), + skipped: state.skipped.clone(), + error: state.error.clone(), + }); + } + phases.mark("finish"); + + let changed_files: Vec = changed + .into_iter() + .map(|(path, (_, content))| ChangedFile { path, content }) + .collect(); + let changed_binary_files: Vec = changed_binary + .into_iter() + .map(|(path, (_, content))| ChangedBinaryFile { path, content }) + .collect(); + let stats = EngineStats { + projects: projects.len() as u64, + files_input, + bytes_input, + packages_scanned: projects.iter().map(|p| p.summary.scanned_packages).sum(), + packages_with_patches: projects + .iter() + .map(|p| p.summary.packages_with_patches) + .sum(), + patches_selected: states.iter().map(|s| s.selected.len() as u64).sum(), + patches_redirected: projects.iter().map(|p| p.redirected.len() as u64).sum(), + files_changed: (changed_files.len() + changed_binary_files.len()) as u64, + provider_calls: provider.calls(), + phase_ms: phases.ms, + }; + Ok(HostedScanOutput { + projects, + changed_files, + changed_binary_files, + deleted_files: Vec::new(), + warnings, + stats, + engine_version: engine_version(), + }) +} + +/// Records → ledger merge → the project's result and changed files. +fn finish_root( + state: &mut RootState, + done: Rewritten, + records: &BTreeMap>, + dry_run: bool, + changed: &mut BTreeMap, + changed_binary: &mut BTreeMap)>, + warnings: &mut Vec, +) -> ProjectResult { + let Rewritten { + planned, + rewrite, + rewritten, + confirmed, + rush_warnings, + pnpm_warnings, + npm_warnings, + } = done; + let root = state.root.clone(); + let mut record_map: BTreeMap = BTreeMap::new(); + let mut record_warnings: Vec = Vec::new(); + if !dry_run { + for (purl, uuid) in &confirmed { + match records.get(uuid) { + Some(Some(response)) => { + let (rec_purl, record) = + crate::commands::get::record_from_patch_response(response); + record_map.insert(rec_purl, record); + } + _ => record_warnings.push(serde_json::json!({ + "code": "record_fetch_failed", + "detail": format!( + "{purl} redirected, but its patch record could not be fetched; \ + it will be missing from VEX until `socket-patch scan --mode \ + hosted` is re-run" + ), + })), + } + } + } + + let mut project_changes: Vec<(String, String)> = Vec::new(); + let mut ledger_error: Option = None; + if !dry_run && (!rewrite.edits.is_empty() || !record_map.is_empty()) { + let mut ledger = state.ledger.take().unwrap_or_default(); + ledger::merge(&mut ledger, &rewrite.edits, record_map, &planned.files); + match ledger::serialize(&ledger) { + Ok(text) => { + if planned.project.text(REDIRECT_STATE_REL) != Some(text.as_str()) { + project_changes.push((REDIRECT_STATE_REL.to_string(), text)); + } + } + Err(message) => { + ledger_error = Some(ProjectError { + code: "ledger_serialize_failed".into(), + message, + }) + } + } + } + if let Some(error) = ledger_error { + return ProjectResult { + root, + redirect: serde_json::json!({ "mode": "hosted" }), + summary: state.summary.clone(), + redirected: Vec::new(), + skipped: planned.skipped, + error: Some(error), + }; + } + for (rel, content) in &rewrite.files { + if planned.project.text(rel) != Some(content.as_str()) { + project_changes.push((rel.clone(), content.clone())); + } + } + let text_writes: Vec<(String, String)> = project_changes + .into_iter() + .map(|(rel, content)| (roots::join_root(&root, &rel), content)) + .collect(); + let binary_writes: Vec<(String, Vec)> = rewrite + .binary_files + .iter() + .filter(|(rel, bytes)| { + !matches!( + planned.project.get(rel.as_str()), + Some(MemoryEntry::Binary(existing)) if existing.as_ref() == bytes.as_slice() + ) + }) + .map(|(rel, bytes)| (roots::join_root(&root, rel), bytes.clone())) + .collect(); + let conflict = text_writes + .iter() + .find_map(|(path, content)| match changed.get(path) { + Some((owner, existing)) if existing != content => Some((path, owner)), + _ => None, + }) + .or_else(|| { + binary_writes + .iter() + .find_map(|(path, bytes)| match changed_binary.get(path) { + Some((owner, existing)) if existing != bytes => Some((path, owner)), + _ => None, + }) + }); + if let Some((path, owner)) = conflict { + let message = format!( + "{path} is rewritten differently by project `{owner}`; none of this project's \ + changes were written" + ); + warnings.push(EngineWarning::new( + "conflicting_write", + message.clone(), + Some(&root), + )); + return ProjectResult { + root, + redirect: serde_json::json!({ "mode": "hosted" }), + summary: state.summary.clone(), + redirected: Vec::new(), + skipped: planned.skipped, + error: Some(ProjectError { + code: "conflicting_write".into(), + message, + }), + }; + } + for (path, content) in text_writes { + changed + .entry(path) + .or_insert_with(|| (root.clone(), content)); + } + for (path, bytes) in binary_writes { + changed_binary + .entry(path) + .or_insert_with(|| (root.clone(), bytes)); + } + + let mut redirect_warnings: Vec = rewrite + .warnings + .iter() + .map(|w| serde_json::json!({ "code": w.code, "detail": w.detail })) + .collect(); + redirect_warnings.extend(record_warnings); + redirect_warnings.extend(rush_warnings); + redirect_warnings.extend(pnpm_warnings); + redirect_warnings.extend(npm_warnings); + redirect_warnings.extend(planned.pre_warnings.iter().cloned()); + let skipped_values: Vec = planned + .skipped + .iter() + .map(|s| serde_json::to_value(s).unwrap_or(serde_json::Value::Null)) + .collect(); + let redirect = crate::commands::scan::hosted::redirect_json_block( + confirmed.len(), + rewritten, + skipped_values, + redirect_warnings, + dry_run, + ); + ProjectResult { + root, + redirect, + summary: state.summary.clone(), + redirected: confirmed + .into_iter() + .map(|(purl, uuid)| RedirectedPatch { purl, uuid }) + .collect(), + skipped: planned.skipped, + error: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use socket_patch_core::patch::redirect::{FileEdit, RewriteResult}; + + fn state(root: &str, project: MemoryProject) -> RootState { + RootState { + root: root.to_string(), + project: Some(project), + unreadable: BTreeSet::new(), + ledger: None, + purls: Vec::new(), + summary: ProjectSummary::default(), + packages: Vec::new(), + selected: Vec::new(), + skipped: Vec::new(), + error: None, + } + } + + fn rewritten(files: &[(&str, &str)]) -> Rewritten { + let planned = redirect::plan(MemoryProject::new(), BTreeSet::new(), &[], &HashMap::new()) + .unwrap_or_else(|r| panic!("{:?}", r.error)); + let mut rewrite = RewriteResult::default(); + for (rel, content) in files { + rewrite + .files + .insert((*rel).to_string(), (*content).to_string()); + rewrite.edits.push(FileEdit { + path: (*rel).to_string(), + kind: "cargo_registry_redirect".into(), + action: "set".into(), + key: None, + original: None, + new: None, + }); + } + Rewritten { + planned, + rewrite, + rewritten: files.iter().map(|(rel, _)| (*rel).to_string()).collect(), + confirmed: vec![("pkg:cargo/serde@1.0.190".into(), "u".into())], + rush_warnings: Vec::new(), + pnpm_warnings: Vec::new(), + npm_warnings: Vec::new(), + } + } + + #[test] + fn a_conflicting_project_writes_nothing() { + let mut changed = BTreeMap::new(); + let mut changed_binary = BTreeMap::new(); + let mut warnings = Vec::new(); + let records = BTreeMap::new(); + let first = finish_root( + &mut state("", MemoryProject::new()), + rewritten(&[("crates/b/Cargo.toml", "A")]), + &records, + false, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + assert!(first.error.is_none()); + let before: Vec = changed.keys().cloned().collect(); + let second = finish_root( + &mut state("crates/b", MemoryProject::new()), + rewritten(&[("Cargo.toml", "B"), ("Cargo.lock", "L")]), + &records, + false, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + assert_eq!(second.error.as_ref().unwrap().code, "conflicting_write"); + assert!(second.redirected.is_empty()); + assert_eq!(second.redirect, serde_json::json!({ "mode": "hosted" })); + assert_eq!(changed.keys().cloned().collect::>(), before); + assert_eq!(changed["crates/b/Cargo.toml"].1, "A"); + assert!(warnings.iter().any( + |w| w.code == "conflicting_write" && w.project_root.as_deref() == Some("crates/b") + )); + + let same = finish_root( + &mut state("crates/b", MemoryProject::new()), + rewritten(&[("Cargo.toml", "A")]), + &records, + true, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + assert!(same.error.is_none()); + } + + #[test] + fn a_conflicting_binary_write_refuses_the_project() { + let mut changed = BTreeMap::new(); + let mut changed_binary = BTreeMap::new(); + changed_binary.insert("web/bun.lockb".to_string(), ("".to_string(), vec![1u8])); + let mut warnings = Vec::new(); + let mut done = rewritten(&[]); + done.rewrite + .binary_files + .insert("bun.lockb".to_string(), vec![2u8]); + let result = finish_root( + &mut state("web", MemoryProject::new()), + done, + &BTreeMap::new(), + true, + &mut changed, + &mut changed_binary, + &mut warnings, + ); + assert_eq!(result.error.unwrap().code, "conflicting_write"); + assert_eq!(changed_binary["web/bun.lockb"].1, vec![1u8]); + assert!(changed.is_empty()); + } + + #[test] + fn workspace_member_roots_lose_their_own_cargo_lock() { + let mut ws = MemoryProject::new(); + ws.insert_text( + "Cargo.toml", + "[workspace]\nmembers = [\"crates/*\"]\nexclude = [\"crates/fuzz\"]\n", + ); + ws.insert_text("Cargo.lock", "version = 3\n"); + ws.insert_text("crates/b/Cargo.toml", "[package]\nname = \"b\"\n"); + ws.insert_text("crates/b/Cargo.lock", "version = 3\n"); + ws.insert_text("crates/b/package-lock.json", "{}"); + ws.insert_text("crates/fuzz/Cargo.toml", "[package]\nname = \"fuzz\"\n"); + ws.insert_text("crates/fuzz/Cargo.lock", "version = 3\n"); + ws.insert_text("crates/fuzz/package-lock.json", "{}"); + let sub = |dir: &str| { + let mut p = MemoryProject::new(); + for (path, entry) in ws.entries() { + if let Some(rel) = roots::strip_root(dir, path) { + p.insert(rel, entry.clone()); + } + } + p + }; + let mut states = vec![ + state("", ws.clone()), + state("crates/b", sub("crates/b")), + state("crates/fuzz", sub("crates/fuzz")), + ]; + let mut warnings = Vec::new(); + demote_cargo_members(&mut states, &mut warnings); + let has_lock = |i: usize| states[i].project.as_ref().unwrap().contains("Cargo.lock"); + assert!(has_lock(0)); + assert!(!has_lock(1), "a member's lock is the workspace's"); + assert!(has_lock(2), "an excluded crate is its own workspace"); + assert_eq!(warnings.len(), 1); + assert_eq!(warnings[0].code, "cargo_member_lock_ignored"); + assert_eq!(warnings[0].project_root.as_deref(), Some("crates/b")); + } + + #[test] + fn nested_roots_share_one_copy_of_each_file() { + let mut files: BTreeMap = BTreeMap::new(); + files.insert( + "a/b/package-lock.json".into(), + share(InputFile::Text("{}".repeat(1024))), + ); + files.insert( + "a/big.lock".into(), + share(InputFile::Present(PresentKind::Oversize)), + ); + files.insert( + "a/odd.txt".into(), + share(InputFile::Present(PresentKind::BinarySkipped)), + ); + let (outer, outer_unreadable) = project_for("", &files); + let (inner, _) = project_for("a/b", &files); + match ( + outer.get("a/b/package-lock.json"), + inner.get("package-lock.json"), + ) { + (Some(MemoryEntry::Text(x)), Some(MemoryEntry::Text(y))) => { + assert!(Arc::ptr_eq(x, y), "each root must share the input's bytes") + } + other => panic!("{other:?}"), + } + assert_eq!( + outer_unreadable, + BTreeSet::from(["a/big.lock".to_string()]), + "a non-UTF-8 file is absent to disk too" + ); + } + + #[test] + fn unsupported_markers_outside_roots_warn_once_per_ecosystem() { + let paths = [ + "java/a/pom.xml", + "java/b/pom.xml", + "web/pom.xml", + "dotnet/nuget.config", + "tests/pom.xml", + "src/Main.java", + ]; + let mut out = Vec::new(); + unrooted_unsupported_warnings(paths.into_iter(), &["web".to_string()], None, &mut out); + assert_eq!(out.len(), 2); + assert!(out + .iter() + .all(|w| w.code == "ecosystem_unsupported_in_memory" && w.project_root.is_none())); + assert!( + out[0].detail.starts_with("java/a/pom.xml (and 1 more)"), + "{}", + out[0].detail + ); + assert!( + out[1].detail.starts_with("dotnet/nuget.config is"), + "{}", + out[1].detail + ); + let mut filtered = Vec::new(); + unrooted_unsupported_warnings( + paths.into_iter(), + &[], + Some(&["npm".to_string()]), + &mut filtered, + ); + assert!(filtered.is_empty()); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/redirect.rs b/crates/socket-patch-cli/src/hosted_memory/redirect.rs new file mode 100644 index 000000000..4b3849455 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/redirect.rs @@ -0,0 +1,1154 @@ +//! One project root's hosted redirect over an in-memory file set: the +//! disk flow's `run_redirect_selected` stages as pure functions +//! (reference → `DepOverride` candidates, candidate-file reads, the +//! rewrite, the pnpm `trustLockfile` and npm `allow-remote` auto-configs, +//! per-ecosystem confirmation, the symlink guard). Everything that needs +//! the host machine (the apply lock, vendored takeover reverts, stale +//! install probes, VEX, telemetry, subprocesses) is left out; a vendored +//! takeover is refused instead of performed. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use socket_patch_core::api::types::PackageVendorResult; +use socket_patch_core::constants::npm_family::{RUSH_COMMON_LOCK_REL, RUSH_SUBSPACES_DIR}; +use socket_patch_core::patch::redirect::npmrc::{ + plan_npmrc_allow_remote_with, NpmrcPlan, OuterAllowRemote, NPMRC_ALLOW_REMOTE_EDIT_KIND, + NPMRC_REL, +}; +use socket_patch_core::patch::redirect::{ + rewrite_registry_redirect_with_pipenv_version, DepOverride, FileEdit, RewriteResult, + RewriteWarning, +}; +use socket_patch_core::utils::purl::{purl_parts, strip_purl_qualifiers}; +use socket_patch_core::vendor::lock_inventory::{MemoryEntry, MemoryProject}; +use socket_patch_core::vendor::VendorState; + +use super::select::{RUSH_REPO_STATE_REL, VENDOR_STATE_REL}; +use super::types::{ProjectError, SkippedPatch}; +use crate::commands::scan::hosted::{ + npm_allow_remote_already_detail, npm_allow_remote_configured_detail, + npm_allow_remote_env_set_detail, npm_allow_remote_manual_detail, + npm_allow_remote_outer_set_detail, npm_allow_remote_unreadable_detail, + npm_allow_remote_user_set_detail, plan_workspace_trust, pnpm_heal_root, + pnpm_lock_may_need_store_flag, pnpm_lock_version_major, pnpm_trust_configured_detail, + pnpm_trust_legacy_detail, pnpm_trust_manual_guidance, pnpm_trust_policy_preamble, + pnpm_trust_workspace_unreadable_detail, url_host, TrustPlan, NPM_LOCKS, + PNPM_TRUST_TRADEOFF_AND_CAUTION, PNPM_WORKSPACE_REL, REDIRECT_CANDIDATE_FILES, + REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND, +}; + +/// Skip reason / warning code for a candidate the disk flow would migrate +/// from vendored to hosted (the migration reverts committed wiring, which +/// the in-memory engine does not do). +pub(crate) const VENDORED_TAKEOVER_UNSUPPORTED: &str = "vendored_takeover_unsupported_in_memory"; + +/// The disk flow's symlink refusal code. +pub(crate) const SYMLINK_REFUSAL: &str = "redirect_symlinked_file_unsupported"; + +/// A candidate file exists but its content was not provided (oversize, an +/// LFS pointer, presence-only); disk would read and rewrite it. +pub(crate) const UNREADABLE_REFUSAL: &str = "candidate_file_unreadable"; + +/// One granted reference: the purl it was granted for plus its override. +#[derive(Debug, Clone)] +pub(crate) struct Candidate { + pub(crate) purl: String, + pub(crate) dep: DepOverride, +} + +/// The engine-level options the per-root stages read. +#[derive(Debug, Clone, Copy)] +pub(crate) struct StageOptions { + pub(crate) dry_run: bool, + pub(crate) pipenv_major: Option, + pub(crate) trust_lockfile_config: bool, + pub(crate) npm_allow_remote_config: bool, +} + +/// Reference grants → candidates (disk: the loop over `selected` after +/// `fetch_registry_references`). +pub(crate) fn build_candidates( + selected: &[(String, String)], + references: &HashMap, + skipped: &mut Vec, +) -> Vec { + let skip = |purl: &str, uuid: &str, reason: &str| SkippedPatch { + purl: purl.to_string(), + uuid: uuid.to_string(), + reason: reason.to_string(), + detail: None, + }; + let mut candidates = Vec::new(); + for (sel_purl, sel_uuid) in selected { + let Some(reference) = references.get(sel_uuid) else { + skipped.push(skip(sel_purl, sel_uuid, "not_found")); + continue; + }; + if reference.status != "granted" && reference.status != "reused" { + skipped.push(skip(sel_purl, sel_uuid, &reference.status)); + continue; + } + let purl = reference.purl.as_deref().unwrap_or(sel_purl); + let Some((ecosystem, name, version)) = purl_parts(purl) else { + skipped.push(skip(purl, sel_uuid, "bad_purl")); + continue; + }; + let Some(url) = reference.url.clone() else { + skipped.push(skip(purl, sel_uuid, "no_url")); + continue; + }; + let mut integrity = reference + .artifacts + .iter() + .flatten() + .find(|a| a.kind == "tarball") + .map(|a| a.integrity.clone()) + .unwrap_or_default(); + let berry_zip = reference + .artifacts + .iter() + .flatten() + .find(|a| a.kind == "yarn-berry-zip"); + if let Some(c) = berry_zip.and_then(|a| a.integrity.yarn_berry10c0.clone()) { + integrity.yarn_berry10c0 = Some(c); + } + if let Some(ov) = reference + .registry_override + .as_ref() + .filter(|o| o.kind == "goproxy") + { + if let (Some(zip_h1), Some(gomod_h1)) = ( + ov.identifiers.go_zip_dirhash_h1.clone(), + ov.identifiers.go_mod_h1.clone(), + ) { + integrity.dirhash_h1 = Some(zip_h1); + integrity.go_mod_h1 = Some(gomod_h1); + } + } + let token = reference + .registry_override + .as_ref() + .and_then(|o| { + socket_patch_core::patch::redirect::grant_token_path_segment(&o.index_url, sel_uuid) + }) + .or_else(|| { + socket_patch_core::patch::redirect::grant_token_path_segment(&url, sel_uuid) + }) + .unwrap_or_default(); + candidates.push(Candidate { + purl: purl.to_string(), + dep: DepOverride { + ecosystem, + name, + namespace: None, + version, + token, + patch_uuid: sel_uuid.clone(), + artifact_url: url, + berry_zip_url: berry_zip.and_then(|a| a.url.clone()), + registry_override: reference.registry_override.clone(), + integrity, + }, + }); + } + candidates +} + +/// The ecosystem a candidate file's rewriter belongs to (`None` for files +/// no rewriter edits), for the symlinked-read refusal. +fn file_ecosystem(rel: &str) -> Option<&'static str> { + let base = rel.rsplit('/').next().unwrap_or(rel); + Some(match base { + "package-lock.json" + | "npm-shrinkwrap.json" + | "pnpm-lock.yaml" + | "shrinkwrap.yaml" + | ".modules.yaml" + | "yarn.lock" + | ".yarnrc.yml" + | "bun.lock" + | "bun.lockb" => "npm", + "requirements.txt" | "uv.lock" | "poetry.lock" | "pdm.lock" | "Pipfile.lock" + | "pyproject.toml" | "hatch.toml" => "pypi", + "Cargo.toml" | "Cargo.lock" | "config.toml" | "config" => "cargo", + "composer.lock" => "composer", + "nuget.config" | "packages.lock.json" => "nuget", + "Gemfile" | "Gemfile.lock" | "gems.rb" | "gems.locked" => "gem", + "go.mod" | "go.sum" => "golang", + "pom.xml" | "maven.config" | "checksums.sha256" => "maven", + _ if socket_patch_core::utils::python_lock::is_python_lock_name(base) + || base.ends_with(".py") => + { + "pypi" + } + _ => return None, + }) +} + +/// A project's state between the reference grants and the wheel-metadata +/// fetch. +#[derive(Debug)] +pub(crate) struct Planned { + pub(crate) project: MemoryProject, + pub(crate) candidates: Vec, + pub(crate) skipped: Vec, + pub(crate) pre_warnings: Vec, + pub(crate) files: BTreeMap, + pub(crate) rush_lock_keys: Vec, + pub(crate) bun_lock_present: bool, + /// Candidate files the disk flow reads through a symbolic link: their + /// bytes are unknown here, so a project whose candidates could rewrite + /// one is refused like the disk symlink guard refuses the write. + pub(crate) symlinked_reads: Vec, + /// Candidate files that exist without content: a project whose + /// candidates could rewrite (or whose rewrite depends on) one is + /// refused, since the rewriters would treat it as absent. + pub(crate) unreadable_reads: Vec, + /// `(artifact url, sha256)` of every pypi wheel whose metadata a + /// native lock rewrite needs. + pub(crate) wheels: Vec<(String, String)>, +} + +/// A refused project: its error and whatever was skipped before it. +#[derive(Debug)] +pub(crate) struct Refused { + pub(crate) error: ProjectError, +} + +fn refusal(code: &str, message: String) -> Refused { + Refused { + error: ProjectError { + code: code.to_string(), + message, + }, + } +} + +fn unreadable_refusal(rel: &str) -> Refused { + refusal( + UNREADABLE_REFUSAL, + format!( + "{rel} exists but its content was not provided (too large, an LFS pointer, or \ + not fetched), so it cannot be rewritten alongside the other lockfiles; nothing \ + was written" + ), + ) +} + +fn symlink_refusal(linked: &str) -> Refused { + refusal( + SYMLINK_REFUSAL, + format!( + "{linked} is a symbolic link; socket-patch rewrites files in place with an atomic \ + rename, which would replace the link — replace the link with a regular file (or \ + run socket-patch in the directory it points to) and re-run; nothing was written" + ), + ) +} + +/// The vendored ledger's entries (the disk `vendor::load_state` parse, +/// including its legacy `{mode}`-only shape); `None` when absent or +/// unreadable. +fn vendored_entries(project: &MemoryProject) -> Option { + let bytes: Vec = match project.get(VENDOR_STATE_REL)? { + MemoryEntry::Text(text) => text.as_bytes().to_vec(), + MemoryEntry::Binary(bytes) => bytes.to_vec(), + _ => return None, + }; + match serde_json::from_slice::(&bytes) { + Ok(state) => Some(state), + Err(_) => { + let value: serde_json::Value = serde_json::from_slice(&bytes).ok()?; + (value.get("mode").is_some() && value.get("entries").is_none()).then(VendorState::new) + } + } +} + +/// Whether Socket-owned vendored `[patch.crates-io]` wiring for exactly +/// `name@version` is committed in the root manifest or (legacy) the +/// project's cargo config — the disk `socket_wiring_present` probe over +/// the in-memory files. The config cargo reads is `.cargo/config` when it +/// exists, else `.cargo/config.toml`. +fn cargo_vendored_wiring(files: &MemoryProject, name: &str, version: &str) -> bool { + use socket_patch_core::vendor::cargo_manifest::{ + crates_io_patch_entries, entry_wires, parse_manifest, + }; + let manifest_wired = files + .text("Cargo.toml") + .and_then(|text| parse_manifest(text).ok()) + .is_some_and(|doc| { + crates_io_patch_entries(&doc) + .iter() + .any(|e| entry_wires(e, name, version)) + }); + let config_rel = if files.contains(".cargo/config") { + ".cargo/config" + } else { + ".cargo/config.toml" + }; + let config_wired = files + .text(config_rel) + .and_then(|text| parse_manifest(text).ok()) + .is_some_and(|doc| { + crates_io_patch_entries(&doc) + .iter() + .any(|e| e.source == "crates-io" && entry_wires(e, name, version)) + }); + manifest_wired || config_wired +} + +/// Everything up to the wheel-metadata fetch. +pub(crate) fn plan( + project: MemoryProject, + unreadable: BTreeSet, + selected: &[(String, String)], + references: &HashMap, +) -> Result { + let mut skipped: Vec = Vec::new(); + let mut candidates = if selected.is_empty() { + Vec::new() + } else { + build_candidates(selected, references, &mut skipped) + }; + + let bun_lock_present = project.contains("bun.lock"); + if candidates.iter().any(|c| c.dep.ecosystem == "npm") + && !bun_lock_present + && project.is_symlink("bun.lockb") + { + return Err(refusal( + SYMLINK_REFUSAL, + "bun.lockb is a symbolic link; replace it with a regular file (or run \ + socket-patch in the directory it points to) before patching; nothing was written" + .to_string(), + )); + } + + let mut pre_warnings: Vec = Vec::new(); + let takeover_capable = |p: &str| { + p.starts_with("pkg:cargo/") || p.starts_with("pkg:npm/") || p.starts_with("pkg:golang/") + }; + if candidates.iter().any(|c| takeover_capable(&c.purl)) { + let vendored = vendored_entries(&project); + let mut refused: BTreeSet = BTreeSet::new(); + for candidate in candidates.iter().filter(|c| takeover_capable(&c.purl)) { + let has_entry = vendored.as_ref().is_some_and(|s| { + socket_patch_core::vendor::lookup_entry( + &s.entries, + strip_purl_qualifiers(&candidate.purl), + ) + .is_some() + }); + let cargo_wired = !has_entry + && candidate.purl.starts_with("pkg:cargo/") + && cargo_vendored_wiring(&project, &candidate.dep.name, &candidate.dep.version); + if has_entry || cargo_wired { + refused.insert(candidate.purl.clone()); + } + } + if !refused.is_empty() { + pre_warnings.push(serde_json::json!({ + "code": VENDORED_TAKEOVER_UNSUPPORTED, + "detail": format!( + "{} currently vendored ({}); migrating a vendored package to hosted \ + reverts its committed vendored wiring, which the in-memory hosted scan \ + does not do — run `socket-patch scan --mode hosted` in a checkout to \ + migrate, then re-run", + if refused.len() == 1 { "1 package is" } else { "packages are" }, + refused.iter().cloned().collect::>().join(", ") + ), + })); + for c in candidates.iter().filter(|c| refused.contains(&c.purl)) { + skipped.push(SkippedPatch { + purl: c.purl.clone(), + uuid: c.dep.patch_uuid.clone(), + reason: VENDORED_TAKEOVER_UNSUPPORTED.to_string(), + detail: None, + }); + } + candidates.retain(|c| !refused.contains(&c.purl)); + } + } + + let mut files: BTreeMap = BTreeMap::new(); + let mut rush_lock_keys: Vec = Vec::new(); + let mut symlinked_reads: Vec = Vec::new(); + let mut unreadable_reads: Vec = Vec::new(); + if !candidates.is_empty() { + let mut read = |rel: &str, files: &mut BTreeMap| -> bool { + if project.is_symlink(rel) { + symlinked_reads.push(rel.to_string()); + return false; + } + if unreadable.contains(rel) { + unreadable_reads.push(rel.to_string()); + return false; + } + // Disk reads any UTF-8 regular file; a non-UTF-8 one is absent + // to it as well. + let text = match project.get(rel) { + Some(MemoryEntry::Text(text)) => Some(text.to_string()), + Some(MemoryEntry::Binary(bytes)) => { + std::str::from_utf8(bytes).ok().map(str::to_string) + } + _ => None, + }; + match text { + Some(text) => { + files.insert(rel.to_string(), text); + true + } + None => false, + } + }; + for name in REDIRECT_CANDIDATE_FILES { + if *name == "bun.lockb" { + continue; + } + read(name, &mut files); + } + if files.contains_key("Cargo.toml") && candidates.iter().any(|c| c.dep.ecosystem == "cargo") + { + let view = socket_patch_core::vendor::lock_inventory::ProjectView::Memory(&project); + for rel in socket_patch_core::utils::cargo_workspace::member_manifests_in(&view) { + read(&rel, &mut files); + } + } + let python_locks: Vec = project + .children("") + .into_iter() + .filter(|(name, is_dir)| { + !is_dir && socket_patch_core::utils::python_lock::is_python_lock_name(name) + }) + .map(|(name, _)| name) + .collect(); + for path in python_locks { + if let Some(script) = socket_patch_core::utils::python_lock::script_of_lock(&path) { + read(script, &mut files); + } + read(&path, &mut files); + } + if project.contains("rush.json") { + if read(RUSH_COMMON_LOCK_REL, &mut files) { + rush_lock_keys.push(RUSH_COMMON_LOCK_REL.to_string()); + } + for (name, is_dir) in project.children(RUSH_SUBSPACES_DIR) { + if !is_dir { + continue; + } + let key = format!("{RUSH_SUBSPACES_DIR}/{name}/pnpm-lock.yaml"); + if read(&key, &mut files) { + rush_lock_keys.push(key); + } + } + } + } + symlinked_reads.sort(); + symlinked_reads.dedup(); + unreadable_reads.sort(); + unreadable_reads.dedup(); + + let mut wheels: Vec<(String, String)> = Vec::new(); + for dep in candidates + .iter() + .map(|c| &c.dep) + .filter(|dep| dep.ecosystem == "pypi") + { + let Some(sha256) = dep.integrity.sha256.as_deref() else { + continue; + }; + if !dep + .artifact_url + .split(['?', '#']) + .next() + .is_some_and(|path| path.ends_with(".whl")) + { + continue; + } + let native_target = files + .iter() + .filter(|(path, _)| { + *path == "uv.lock" + || socket_patch_core::utils::python_lock::is_script_lock_name(path) + }) + .any(|(_, text)| { + socket_patch_core::utils::python_lock::rewrite_python_lock( + text, + &dep.name, + &dep.version, + socket_patch_core::utils::python_lock::ArtifactSource::Url(&dep.artifact_url), + sha256, + ) + .ok() + .flatten() + .is_some() + }); + if native_target { + wheels.push((dep.artifact_url.clone(), sha256.to_string())); + } + } + + Ok(Planned { + project, + candidates, + skipped, + pre_warnings, + files, + rush_lock_keys, + bun_lock_present, + symlinked_reads, + unreadable_reads, + wheels, + }) +} + +/// A project's rewrite, ready for the record fetch and the ledger merge. +#[derive(Debug)] +pub(crate) struct Rewritten { + pub(crate) planned: Planned, + pub(crate) rewrite: RewriteResult, + pub(crate) rewritten: Vec, + pub(crate) confirmed: Vec<(String, String)>, + pub(crate) rush_warnings: Vec, + pub(crate) pnpm_warnings: Vec, + pub(crate) npm_warnings: Vec, +} + +/// The `.npmrc` read the allow-remote planner classifies (disk: +/// `read_npmrc_for_allow_remote`). +fn read_npmrc(project: &MemoryProject) -> Result, String> { + match project.get(NPMRC_REL) { + None => Ok(None), + Some(MemoryEntry::Symlink) => { + Err("is a symbolic link (socket-patch never writes through one)".into()) + } + Some(MemoryEntry::Text(text)) => Ok(Some(text.to_string())), + Some(MemoryEntry::Binary(_)) => { + Err("could not be read (stream did not contain valid UTF-8)".into()) + } + Some(MemoryEntry::Present) => { + Err("could not be read (file content was not provided)".into()) + } + } +} + +/// Wheel metadata → rewrite → install-policy configs → confirmation → +/// symlink guard. +pub(crate) fn rewrite( + mut planned: Planned, + wheel_metadata: &BTreeMap, String>>, + options: StageOptions, +) -> Result { + let project = &planned.project; + let files = &planned.files; + + let mut python_metadata: BTreeMap = BTreeMap::new(); + let mut unavailable: BTreeSet = BTreeSet::new(); + for (url, _) in &planned.wheels { + match wheel_metadata.get(url) { + Some(Ok(Some(metadata))) => { + python_metadata.insert(url.clone(), metadata.clone()); + } + Some(Ok(None)) => {} + Some(Err(detail)) => { + if unavailable.insert(url.clone()) { + for dep in planned + .candidates + .iter() + .map(|c| &c.dep) + .filter(|d| &d.artifact_url == url) + { + planned.skipped.push(SkippedPatch { + purl: format!("pkg:pypi/{}@{}", dep.name, dep.version), + uuid: dep.patch_uuid.clone(), + reason: "python_metadata_unavailable".to_string(), + detail: Some(detail.replace(&dep.artifact_url, "")), + }); + } + } + } + None => { + unavailable.insert(url.clone()); + } + } + } + planned + .candidates + .retain(|c| !unavailable.contains(&c.dep.artifact_url)); + let candidates = &planned.candidates; + let overrides: Vec = candidates.iter().map(|c| c.dep.clone()).collect(); + + let targets_pipenv_lock = + socket_patch_core::patch::redirect::pipenv_lock_targets(files, &overrides); + let pipenv_major = if targets_pipenv_lock { + options.pipenv_major + } else { + None + }; + let binary_bun = !planned.bun_lock_present && project.contains("bun.lockb"); + let binary_content: Option, RewriteWarning>> = + if binary_bun && overrides.iter().any(|o| o.ecosystem == "npm") { + let read = match project.get("bun.lockb") { + Some(MemoryEntry::Binary(bytes)) => Ok(bytes.to_vec()), + Some(MemoryEntry::Text(text)) => Ok(text.as_bytes().to_vec()), + _ => Err("file content was not provided".to_string()), + }; + Some( + read.map_err(|e| RewriteWarning { + code: "redirect_bun_lockb_invalid".into(), + detail: format!("cannot read bun.lockb: {e}"), + }) + .and_then(|bytes| { + socket_patch_core::patch::redirect::preflight_bun_binary(&bytes)?; + Ok(bytes) + }), + ) + } else { + None + }; + let rewrite_overrides: Vec = overrides + .iter() + .filter(|o| !(binary_content.as_ref().is_some_and(Result::is_err) && o.ecosystem == "npm")) + .cloned() + .collect(); + let mut rewrite = rewrite_registry_redirect_with_pipenv_version( + files, + &rewrite_overrides, + &python_metadata, + pipenv_major, + ); + if let Some(content) = binary_content { + rewrite + .warnings + .retain(|w| w.code != "redirect_npm_no_lockfile"); + match content { + Ok(bytes) => socket_patch_core::patch::redirect::rewrite_bun_binary( + &bytes, + &overrides, + &mut rewrite, + ), + Err(warning) => rewrite.warnings.push(warning), + } + } + + if targets_pipenv_lock && pipenv_major.is_none() && rewrite.files.contains_key("Pipfile.lock") { + rewrite.warnings.push(RewriteWarning { + code: "redirect_pipenv_installer_unknown".into(), + detail: format!( + "Pipenv was not found on PATH, so the Pipfile.lock references use the modern `file` form (Pipenv 2018 and later). A project installed with Pipenv 7–11 needs `path` references instead: put that pipenv on PATH or set {}= and re-run `scan --mode hosted`.", + socket_patch_core::utils::pipenv::MAJOR_OVERRIDE_ENV + ), + }); + } + + let mut rush_warnings: Vec = Vec::new(); + if planned + .rush_lock_keys + .iter() + .any(|key| rewrite.files.contains_key(key)) + && (project.contains(RUSH_REPO_STATE_REL)) + { + rush_warnings.push(serde_json::json!({ + "code": "redirect_rush_repo_state_stale", + "detail": + "pnpm-lock.yaml was edited outside `rush update`; if \ + preventManualShrinkwrapChanges is enabled, `rush install` fails until \ + `rush update` refreshes repo-state.json (the redirect survives `rush \ + update`)", + })); + } + + let mut pnpm_warnings: Vec = Vec::new(); + let mut trust_config_write: Option<(String, FileEdit)> = None; + let mut workspace_symlink_write = false; + { + let mut pnpm_lock_texts: Vec<&String> = rewrite + .files + .iter() + .filter(|(key, _)| { + std::path::Path::new(key) + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|name| matches!(name, "pnpm-lock.yaml" | "shrinkwrap.yaml")) + }) + .map(|(_, content)| content) + .collect(); + let heal_root: Option<&String> = pnpm_heal_root( + rewrite.files.contains_key("pnpm-lock.yaml"), + files.get("pnpm-lock.yaml"), + &overrides, + ); + if let Some(text) = heal_root { + pnpm_lock_texts.push(text); + } + if !pnpm_lock_texts.is_empty() { + let mut hosts: Vec<&str> = overrides + .iter() + .filter(|o| o.ecosystem == "npm") + .filter(|o| { + let encoded = + socket_patch_core::utils::uri::encode_uri_component(&o.artifact_url); + pnpm_lock_texts.iter().any(|text| { + socket_patch_core::patch::redirect::artifact_url_present( + text, + &o.artifact_url, + ) || text.contains(encoded.as_str()) + }) + }) + .filter_map(|o| url_host(&o.artifact_url)) + .collect(); + hosts.sort_unstable(); + hosts.dedup(); + let server = if hosts.is_empty() { + "the hosted patch server".to_string() + } else { + format!("the hosted patch server ({})", hosts.join(", ")) + }; + let root_lock_v9 = heal_root + .and_then(|text| pnpm_lock_version_major(text)) + .is_some_and(|major| major >= 9) + || rewrite + .files + .get("pnpm-lock.yaml") + .and_then(|text| pnpm_lock_version_major(text)) + .is_some_and(|major| major >= 9); + let all_locks_legacy = pnpm_lock_texts.iter().all(|text| { + pnpm_lock_version_major(text).is_some_and(|major| major < 9) + || text + .lines() + .any(|line| line.starts_with("shrinkwrapVersion:")) + }); + let workspace: Result, std::io::Error> = + match project.get(PNPM_WORKSPACE_REL) { + None => Ok(None), + Some(MemoryEntry::Text(text)) => Ok(Some(text.to_string())), + Some(MemoryEntry::Symlink) => { + workspace_symlink_write = true; + Ok(None) + } + Some(MemoryEntry::Binary(_)) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + )), + Some(MemoryEntry::Present) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "file content was not provided", + )), + }; + let detail = if all_locks_legacy { + workspace_symlink_write = false; + pnpm_trust_legacy_detail(&server) + } else if !root_lock_v9 || !options.trust_lockfile_config { + workspace_symlink_write = false; + pnpm_trust_manual_guidance(&server) + } else { + match workspace { + Err(e) => pnpm_trust_workspace_unreadable_detail(&server, &e), + Ok(ws_existing) => match plan_workspace_trust(ws_existing.as_deref()) { + TrustPlan::Create(text) => { + trust_config_write = Some(( + text, + FileEdit { + path: PNPM_WORKSPACE_REL.into(), + kind: REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND.into(), + action: "created".into(), + key: Some("trustLockfile".into()), + original: None, + new: Some(serde_json::json!("true")), + }, + )); + pnpm_trust_configured_detail(&server, true, options.dry_run) + } + TrustPlan::Append(text) => { + trust_config_write = Some(( + text, + FileEdit { + path: PNPM_WORKSPACE_REL.into(), + kind: REDIRECT_PNPM_WORKSPACE_TRUST_EDIT_KIND.into(), + action: "added".into(), + key: Some("trustLockfile".into()), + original: None, + new: Some(serde_json::json!("true")), + }, + )); + pnpm_trust_configured_detail(&server, false, options.dry_run) + } + TrustPlan::AlreadyTrue => format!( + "{}, and {PNPM_WORKSPACE_REL} already carries `trustLockfile: \ + true` — keep it committed alongside the lock; installs need \ + no extra flags. {PNPM_TRUST_TRADEOFF_AND_CAUTION}", + pnpm_trust_policy_preamble(&server), + ), + TrustPlan::UserSet(value) => format!( + "{}. {PNPM_WORKSPACE_REL} explicitly sets `trustLockfile: \ + {value}`, which was respected and left untouched — install \ + with `pnpm install --trust-lockfile`, or set `trustLockfile: \ + true` yourself so every install accepts the patched \ + artifacts. {PNPM_TRUST_TRADEOFF_AND_CAUTION}", + pnpm_trust_policy_preamble(&server), + ), + }, + } + }; + let store_note = if pnpm_lock_texts + .iter() + .any(|text| pnpm_lock_may_need_store_flag(text)) + { + " (pnpm 1–4 spell the option `--store`)" + } else { + "" + }; + pnpm_warnings.push(serde_json::json!({ + "code": "redirect_pnpm_trust_lockfile", + "detail": format!( + "{}. After a lock-only change, existing node_modules or a warm pnpm store \ + can still contain upstream files. For a reliable reinstall, use a clean \ + node_modules tree and an empty store with \ + `pnpm install --frozen-lockfile --store-dir `\ + {store_note}. Do not rely on `--force`: some versions re-resolve the \ + upstream artifact. Run `socket-patch vex` after installation to verify \ + the patched files.", + detail.trim_end_matches('.') + ), + })); + } + } + if workspace_symlink_write { + return Err(symlink_refusal(PNPM_WORKSPACE_REL)); + } + + let mut npm_warnings: Vec = Vec::new(); + let mut npmrc_config_write: Option<(String, FileEdit)> = None; + { + let npm_hosts: Vec<&str> = { + let mut hosts: Vec<&str> = overrides + .iter() + .filter(|o| o.ecosystem == "npm") + .filter(|o| { + NPM_LOCKS.iter().any(|lock| { + rewrite + .files + .get(*lock) + .or_else(|| files.get(*lock)) + .is_some_and(|text| { + socket_patch_core::patch::redirect::artifact_url_present( + text, + &o.artifact_url, + ) + }) + }) + }) + .filter_map(|o| url_host(&o.artifact_url)) + .collect(); + hosts.sort_unstable(); + hosts.dedup(); + hosts + }; + if !npm_hosts.is_empty() { + let edit = |action: &str| FileEdit { + path: NPMRC_REL.into(), + kind: NPMRC_ALLOW_REMOTE_EDIT_KIND.into(), + action: action.into(), + key: Some("allow-remote".into()), + original: None, + new: Some(serde_json::json!("all")), + }; + let outer = OuterAllowRemote::default(); + let detail = match read_npmrc(project) { + Ok(existing) => match plan_npmrc_allow_remote_with(existing.as_deref(), &outer) { + NpmrcPlan::AlreadyAll => npm_allow_remote_already_detail(&npm_hosts), + NpmrcPlan::UserSet(value) => { + npm_allow_remote_user_set_detail(&npm_hosts, &value) + } + NpmrcPlan::EnvSet { var, value } => { + npm_allow_remote_env_set_detail(&npm_hosts, &var, &value) + } + NpmrcPlan::OuterSet { layer, path, value } => { + npm_allow_remote_outer_set_detail(&npm_hosts, layer, &path, &value) + } + NpmrcPlan::Unsupported(why) => { + npm_allow_remote_unreadable_detail(&npm_hosts, &why) + } + _ if !options.npm_allow_remote_config => { + npm_allow_remote_manual_detail(&npm_hosts) + } + NpmrcPlan::Create(text) => { + npmrc_config_write = Some((text, edit("created"))); + npm_allow_remote_configured_detail(&npm_hosts, true, options.dry_run) + } + NpmrcPlan::Append(text) => { + npmrc_config_write = Some((text, edit("added"))); + npm_allow_remote_configured_detail(&npm_hosts, false, options.dry_run) + } + }, + Err(why) => npm_allow_remote_unreadable_detail(&npm_hosts, &why), + }; + npm_warnings.push(serde_json::json!({ + "code": "redirect_npm_allow_remote", + "detail": detail, + })); + } + } + if let Some((text, edit)) = trust_config_write { + rewrite.files.insert(PNPM_WORKSPACE_REL.to_string(), text); + rewrite.edits.push(edit); + } + if let Some((text, edit)) = npmrc_config_write { + rewrite.files.insert(NPMRC_REL.to_string(), text); + rewrite.edits.push(edit); + } + let rewritten: Vec = rewrite + .files + .keys() + .chain(rewrite.binary_files.keys()) + .cloned() + .collect(); + + let pdm_inactive = + files.contains_key("pdm.lock") && !socket_patch_core::patch::redirect::pdm_drives(files); + let final_texts: Vec<&String> = files + .iter() + .filter(|(name, _)| !(pdm_inactive && name.as_str() == "pdm.lock")) + .map(|(name, content)| rewrite.files.get(name).unwrap_or(content)) + .chain( + rewrite + .files + .iter() + .filter(|(name, _)| !files.contains_key(*name)) + .map(|(_, content)| content), + ) + .collect(); + let confirmed: Vec<(String, String)> = candidates + .iter() + .filter(|c| { + let purl = c.purl.as_str(); + let uuid = c.dep.patch_uuid.as_str(); + if binary_bun && purl.starts_with("pkg:npm/") { + return rewrite.confirmed_bun_binary_uuids.contains(uuid); + } + if rewrite.refused_pipenv_uuids.contains(uuid) { + return false; + } + if rewrite.refused_pdm_uuids.contains(uuid) { + return false; + } + if purl.starts_with("pkg:pypi/") + && socket_patch_core::patch::redirect::pdm_drives(files) + { + return rewrite.confirmed_pdm_uuids.contains(uuid); + } + if rewrite.python_lock_uuids.contains(uuid) { + return rewrite.confirmed_python_lock_uuids.contains(uuid) + && !rewrite.refused_python_lock_uuids.contains(uuid); + } + if rewrite.hatch_uuids.contains(uuid) { + return rewrite.confirmed_hatch_uuids.contains(uuid); + } + if purl.starts_with("pkg:pypi/") { + return rewrite.confirmed_pipenv_uuids.contains(uuid) + || rewrite.confirmed_requirements_uuids.contains(uuid); + } + if rewrite.refused_pnpm_uuids.contains(uuid) { + return false; + } + if purl.starts_with("pkg:cargo/") { + return rewrite.confirmed_cargo_uuids.contains(uuid); + } + if purl.starts_with("pkg:golang/") { + return rewrite.confirmed_golang_uuids.contains(uuid); + } + let artifact_url = c.dep.artifact_url.as_str(); + let registry = c.dep.registry_override.as_ref(); + let index_url = registry.map(|o| o.index_url.as_str()); + let suffixed_version = + registry.and_then(|o| o.identifiers.maven_suffixed_version.as_deref()); + let encoded = socket_patch_core::utils::uri::encode_uri_component(artifact_url); + final_texts.iter().any(|text| { + socket_patch_core::patch::redirect::artifact_url_present(text, artifact_url) + || text.contains(encoded.as_str()) + || index_url.is_some_and(|iu| text.contains(iu)) + || suffixed_version.is_some_and(|sv| text.contains(sv)) + }) + }) + .map(|c| (c.purl.clone(), c.dep.patch_uuid.clone())) + .collect(); + + if let Some(linked) = rewrite + .files + .keys() + .chain(rewrite.binary_files.keys()) + .find(|k| project.is_symlink(k)) + { + return Err(symlink_refusal(linked)); + } + let candidate_ecosystems: BTreeSet<&str> = candidates + .iter() + .map(|c| c.dep.ecosystem.as_str()) + .collect(); + if let Some(linked) = planned + .symlinked_reads + .iter() + .find(|rel| file_ecosystem(rel).is_some_and(|eco| candidate_ecosystems.contains(eco))) + { + return Err(symlink_refusal(linked)); + } + if let Some(rel) = planned + .unreadable_reads + .iter() + .find(|rel| { + rewrite.files.contains_key(rel.as_str()) + || file_ecosystem(rel).is_some_and(|eco| candidate_ecosystems.contains(eco)) + }) + .or_else(|| { + rewrite + .files + .keys() + .find(|k| matches!(project.get(k), Some(MemoryEntry::Present))) + }) + { + return Err(unreadable_refusal(rel)); + } + + Ok(Rewritten { + planned, + rewrite, + rewritten, + confirmed, + rush_warnings, + pnpm_warnings, + npm_warnings, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reference(value: serde_json::Value) -> PackageVendorResult { + serde_json::from_value(value).unwrap() + } + + #[test] + fn candidates_skip_every_unusable_reference() { + let mut refs: HashMap = HashMap::new(); + refs.insert( + "u-pending".into(), + reference(serde_json::json!({"status": "pending_build"})), + ); + refs.insert( + "u-nourl".into(), + reference(serde_json::json!({"status": "granted", "purl": "pkg:npm/b@1"})), + ); + refs.insert( + "u-ok".into(), + reference(serde_json::json!({ + "status": "reused", + "url": "https://patch.example/patch/npm/c/1/tok/u-ok/c-1.tgz", + "purl": "pkg:npm/c@1", + "artifacts": [{"kind": "tarball", "url": null, "integrity": {"sha512": "sha512-x"}}], + "registryOverride": null + })), + ); + let selected = vec![ + ("pkg:npm/a@1".to_string(), "u-missing".to_string()), + ("pkg:npm/p@1".to_string(), "u-pending".to_string()), + ("pkg:npm/b@1".to_string(), "u-nourl".to_string()), + ("pkg:npm/c@1".to_string(), "u-ok".to_string()), + ]; + let mut skipped = Vec::new(); + let candidates = build_candidates(&selected, &refs, &mut skipped); + let reasons: Vec<&str> = skipped.iter().map(|s| s.reason.as_str()).collect(); + assert_eq!(reasons, vec!["not_found", "pending_build", "no_url"]); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].dep.token, "tok"); + assert_eq!( + candidates[0].dep.integrity.sha512.as_deref(), + Some("sha512-x") + ); + } + + #[test] + fn cargo_wiring_probe_is_scoped_to_the_crate_and_version() { + let mut p = MemoryProject::new(); + p.insert_text( + "Cargo.toml", + "[package]\nname = \"app\"\n\n[dependencies]\nlog = \"0.4\"\ncc = \"1\"\n\n\ + [patch.crates-io]\nopenssl-socket-0123abcd = { package = \"openssl\", path = \ + \".socket/vendor/cargo/0123abcd-0000-4000-8000-000000000000/openssl-0.10.66\" }\n", + ); + assert!(cargo_vendored_wiring(&p, "openssl", "0.10.66")); + assert!(!cargo_vendored_wiring(&p, "openssl", "0.10.65")); + assert!(!cargo_vendored_wiring(&p, "log", "0.4.22")); + assert!(!cargo_vendored_wiring(&p, "cc", "1.1.0")); + + let mut legacy = MemoryProject::new(); + legacy.insert_text("Cargo.toml", "[package]\nname = \"app\"\n"); + let config = "[patch.crates-io]\ncc = { path = \ + \".socket/vendor/cargo/0123abcd-0000-4000-8000-000000000000/cc-1.1.0\" }\n"; + legacy.insert_text(".cargo/config.toml", config); + assert!(cargo_vendored_wiring(&legacy, "cc", "1.1.0")); + // cargo reads the legacy spelling when it exists. + legacy.insert_text(".cargo/config", ""); + assert!(!cargo_vendored_wiring(&legacy, "cc", "1.1.0")); + } + + fn cargo_reference( + uuid: &str, + ) -> (Vec<(String, String)>, HashMap) { + let purl = "pkg:cargo/serde@1.0.190"; + let mut refs = HashMap::new(); + refs.insert( + uuid.to_string(), + reference(serde_json::json!({ + "status": "granted", + "url": format!("https://patch.example/patch/cargo/serde/1.0.190/tok/{uuid}/serde-1.0.190.crate"), + "purl": purl, + "artifacts": [{"kind": "tarball", "url": null, "integrity": {"sha256": "ab"}}], + "registryOverride": null + })), + ); + (vec![(purl.to_string(), uuid.to_string())], refs) + } + + #[test] + fn an_unreadable_candidate_file_refuses_its_ecosystem() { + let (selected, refs) = cargo_reference("u-1"); + let mut p = MemoryProject::new(); + p.insert_text("Cargo.toml", "[dependencies]\nserde = \"1\"\n"); + p.insert_text( + "Cargo.lock", + "version = 3\n\n[[package]]\nname = \"serde\"\nversion = \"1.0.190\"\n\ + source = \"registry+https://github.com/rust-lang/crates.io-index\"\n", + ); + p.insert_present(".cargo/config"); + let options = StageOptions { + dry_run: false, + pipenv_major: None, + trust_lockfile_config: true, + npm_allow_remote_config: true, + }; + let unreadable = BTreeSet::from([".cargo/config".to_string()]); + let planned = plan(p.clone(), unreadable, &selected, &refs) + .unwrap_or_else(|r| panic!("{:?}", r.error)); + assert_eq!(planned.unreadable_reads, vec![".cargo/config"]); + let err = rewrite(planned, &BTreeMap::new(), options).unwrap_err(); + assert_eq!(err.error.code, UNREADABLE_REFUSAL); + + // A non-UTF-8 file is absent to disk too: not a refusal. + let planned = + plan(p, BTreeSet::new(), &selected, &refs).unwrap_or_else(|r| panic!("{:?}", r.error)); + assert!(planned.unreadable_reads.is_empty()); + } + + #[test] + fn file_ecosystems_cover_the_rewrite_targets() { + assert_eq!(file_ecosystem("package-lock.json"), Some("npm")); + assert_eq!( + file_ecosystem("common/config/rush/pnpm-lock.yaml"), + Some("npm") + ); + assert_eq!(file_ecosystem("tool.py.lock"), Some("pypi")); + assert_eq!(file_ecosystem("crates/a/Cargo.toml"), Some("cargo")); + assert_eq!(file_ecosystem("build.gradle"), None); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/roots.rs b/crates/socket-patch-cli/src/hosted_memory/roots.rs new file mode 100644 index 000000000..a2a48fd23 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/roots.rs @@ -0,0 +1,263 @@ +//! Project-root detection over a repository path list. A root is a +//! directory holding a root LOCK marker (manifests alone never make one), +//! outside vendored / test-fixture trees, and not an internal directory of +//! an enclosing Rush monorepo or Cargo workspace. + +use std::collections::{BTreeMap, BTreeSet}; + +use socket_patch_core::utils::python_lock::is_python_lock_name; + +use super::types::IgnoredPath; + +/// Lock markers that make their directory a project root, with the +/// ecosystem each belongs to. +pub(crate) const ROOT_LOCK_MARKERS: [(&str, &str); 18] = [ + ("package-lock.json", "npm"), + ("npm-shrinkwrap.json", "npm"), + ("pnpm-lock.yaml", "npm"), + ("yarn.lock", "npm"), + ("bun.lock", "npm"), + ("bun.lockb", "npm"), + ("rush.json", "npm"), + ("uv.lock", "pypi"), + ("poetry.lock", "pypi"), + ("pdm.lock", "pypi"), + ("Pipfile.lock", "pypi"), + ("requirements.txt", "pypi"), + ("Cargo.lock", "cargo"), + ("go.mod", "golang"), + ("go.sum", "golang"), + ("composer.lock", "composer"), + ("Gemfile.lock", "gem"), + ("gems.locked", "gem"), +]; + +/// Marker files of the ecosystems the in-memory engine cannot inventory +/// (disk discovers them only through installed-tree crawlers). +pub(crate) const UNSUPPORTED_MARKERS: [(&str, &[&str]); 2] = [ + ( + "maven", + &[ + "pom.xml", + "build.gradle", + "build.gradle.kts", + "settings.gradle", + "settings.gradle.kts", + ], + ), + ("nuget", &["packages.lock.json", "nuget.config"]), +]; + +/// Directory names whose subtrees never hold a project root: installed +/// trees, VCS and tool state, vendored dependencies, and test fixtures. +pub(crate) const EXCLUDED_ROOT_SEGMENTS: [&str; 10] = [ + "node_modules", + ".git", + ".socket", + ".yarn", + "vendor", + "test", + "tests", + "fixtures", + "__fixtures__", + "testdata", +]; + +/// The ecosystem a root marker basename belongs to. +pub(crate) fn marker_ecosystem(base: &str) -> Option<&'static str> { + if let Some((_, eco)) = ROOT_LOCK_MARKERS.iter().find(|(name, _)| *name == base) { + return Some(eco); + } + is_python_lock_name(base).then_some("pypi") +} + +/// `(dir, basename)` of a `/`-separated path. +pub(crate) fn split_path(path: &str) -> (&str, &str) { + match path.rsplit_once('/') { + Some((dir, base)) => (dir, base), + None => ("", path), + } +} + +/// `root`-relative form of `path`, or `None` when `path` is not under it. +pub(crate) fn strip_root<'a>(root: &str, path: &'a str) -> Option<&'a str> { + if root.is_empty() { + return Some(path); + } + path.strip_prefix(root)?.strip_prefix('/') +} + +/// `root/rel` (`rel` alone for the repo root). +pub(crate) fn join_root(root: &str, rel: &str) -> String { + if root.is_empty() { + rel.to_string() + } else { + format!("{root}/{rel}") + } +} + +fn is_proper_ancestor(ancestor: &str, dir: &str) -> bool { + if ancestor == dir { + return false; + } + ancestor.is_empty() || dir.starts_with(&format!("{ancestor}/")) +} + +fn allowed(ecosystems: Option<&[String]>, eco: &str) -> bool { + ecosystems.is_none_or(|list| list.iter().any(|e| e == eco)) +} + +/// The detected roots (sorted) and the marker paths that did not make one. +pub(crate) fn detect_roots<'a>( + paths: impl IntoIterator, + ecosystems: Option<&[String]>, +) -> (Vec, Vec) { + let mut ignored: Vec = Vec::new(); + let mut markers: BTreeMap> = BTreeMap::new(); + let mut marker_paths: BTreeMap> = BTreeMap::new(); + for path in paths { + let (dir, base) = split_path(path); + let Some(eco) = marker_ecosystem(base) else { + continue; + }; + let ignore = |reason: &str, ignored: &mut Vec| { + ignored.push(IgnoredPath { + path: path.to_string(), + reason: reason.to_string(), + }); + }; + if dir + .split('/') + .any(|seg| EXCLUDED_ROOT_SEGMENTS.contains(&seg)) + { + ignore("excluded_dir", &mut ignored); + continue; + } + if !allowed(ecosystems, eco) { + ignore("ecosystem_filtered", &mut ignored); + continue; + } + let key: &'static str = ROOT_LOCK_MARKERS + .iter() + .find(|(name, _)| *name == base) + .map_or("python-lock", |(name, _)| name); + markers.entry(dir.to_string()).or_default().insert(key); + marker_paths + .entry(dir.to_string()) + .or_default() + .push(path.to_string()); + } + + let rush_roots: Vec = markers + .iter() + .filter(|(_, m)| m.contains("rush.json")) + .map(|(d, _)| d.clone()) + .collect(); + let cargo_lock_dirs: Vec = markers + .iter() + .filter(|(_, m)| m.contains("Cargo.lock")) + .map(|(d, _)| d.clone()) + .collect(); + + let mut roots: Vec = Vec::new(); + for (dir, set) in &markers { + let rush_internal = rush_roots.iter().any(|r| { + let internal = |sub: &str| join_root(r, sub); + *dir == internal("common/config/rush") + || dir.starts_with(&format!("{}/", internal("common/config/subspaces"))) + || *dir == internal("common/temp") + || dir.starts_with(&format!("{}/", internal("common/temp"))) + }); + let mut remaining: BTreeSet<&str> = set.clone(); + if cargo_lock_dirs.iter().any(|a| is_proper_ancestor(a, dir)) { + remaining.remove("Cargo.lock"); + } + let reason = if rush_internal { + Some("rush_internal") + } else if remaining.is_empty() { + Some("cargo_member") + } else { + None + }; + match reason { + Some(reason) => { + for path in marker_paths.get(dir).into_iter().flatten() { + ignored.push(IgnoredPath { + path: path.clone(), + reason: reason.to_string(), + }); + } + } + None => roots.push(dir.clone()), + } + } + roots.sort(); + ignored.sort_by(|a, b| a.path.cmp(&b.path)); + (roots, ignored) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn roots(paths: &[&str]) -> Vec { + detect_roots(paths.iter().copied(), None).0 + } + + #[test] + fn lock_markers_make_roots_and_manifests_do_not() { + assert_eq!( + roots(&[ + "package-lock.json", + "apps/web/pnpm-lock.yaml", + "libs/a/package.json", + "svc/Cargo.toml", + "py/tool.py.lock", + "py2/pylock.toml" + ]), + vec!["", "apps/web", "py", "py2"] + ); + } + + #[test] + fn excluded_trees_never_hold_a_root() { + let (found, ignored) = detect_roots( + [ + "node_modules/x/package-lock.json", + "test/fixtures/yarn.lock", + "a/vendor/b/composer.lock", + ".socket/vendor/npm/package-lock.json", + "docs/requirements.txt", + ], + None, + ); + assert_eq!(found, vec!["docs"]); + assert_eq!(ignored.len(), 4); + assert!(ignored.iter().all(|i| i.reason == "excluded_dir")); + } + + #[test] + fn rush_and_cargo_internals_are_not_roots() { + assert_eq!( + roots(&[ + "rush.json", + "common/config/rush/pnpm-lock.yaml", + "common/config/subspaces/web/pnpm-lock.yaml", + "ws/Cargo.lock", + "ws/crates/a/Cargo.lock", + "ws/crates/b/Cargo.lock", + "ws/crates/b/package-lock.json", + ]), + vec!["", "ws", "ws/crates/b"] + ); + } + + #[test] + fn ecosystem_filter_limits_markers() { + let only_npm = vec!["npm".to_string()]; + let (found, ignored) = + detect_roots(["a/package-lock.json", "b/Cargo.lock"], Some(&only_npm)); + assert_eq!(found, vec!["a"]); + assert_eq!(ignored[0].reason, "ecosystem_filtered"); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/select.rs b/crates/socket-patch-cli/src/hosted_memory/select.rs new file mode 100644 index 000000000..67a4b4b74 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/select.rs @@ -0,0 +1,478 @@ +//! Which repository files the in-memory engine needs: root detection plus, +//! per root, the same root-relative candidate set the disk hosted flow +//! reads (`REDIRECT_CANDIDATE_FILES`, Python lock / script pairs, Cargo +//! member manifests, Rush locks, the install-policy configs, the +//! Plug'n'Play markers and the two `.socket/vendor/` ledgers), plus one +//! presence-only Maven / NuGet marker per ecosystem so a repo holding only +//! those still gets its `ecosystem_unsupported_in_memory` warning. + +use std::collections::{BTreeMap, BTreeSet}; + +use socket_patch_core::constants::npm_family::{ + BUN_LOCKB, PNP_MARKERS, RUSH_COMMON_LOCK_REL, RUSH_SUBSPACES_DIR, +}; +use socket_patch_core::patch::redirect::npmrc::NPMRC_REL; +use socket_patch_core::patch::redirect::REDIRECT_STATE_REL; +use socket_patch_core::utils::python_lock::is_python_lock_name; + +use super::roots::{ + detect_roots, split_path, strip_root, EXCLUDED_ROOT_SEGMENTS, UNSUPPORTED_MARKERS, +}; +use super::types::{IgnoredPath, PathSelection, SelectOptions, TreeEntryInput}; +use crate::commands::scan::hosted::{PNPM_WORKSPACE_REL, REDIRECT_CANDIDATE_FILES}; + +/// Most entries [`PathSelection::ignored_sample`] carries. +pub const IGNORED_SAMPLE_MAX: usize = 100; + +/// Longest path accepted, in bytes. +const MAX_PATH_LEN: usize = 1024; + +/// Deepest path accepted, in segments. +const MAX_PATH_DEPTH: usize = 64; + +/// The vendored-mode ledger (its presence refuses a vendored takeover). +pub(crate) const VENDOR_STATE_REL: &str = ".socket/vendor/state.json"; + +/// Rush's repo-state file (presence feeds the stale-hash warning). +pub(crate) const RUSH_REPO_STATE_REL: &str = "common/config/rush/repo-state.json"; + +/// Root-relative text files read beyond `REDIRECT_CANDIDATE_FILES`. +const EXTRA_TEXT_FILES: [&str; 4] = [ + PNPM_WORKSPACE_REL, + NPMRC_REL, + VENDOR_STATE_REL, + REDIRECT_STATE_REL, +]; + +/// The one directory name the disk Cargo member walk never enters (it +/// follows `members`, `exclude`, path dependencies and `[patch]` paths +/// anywhere else, `vendor/` included); which manifests it reaches depends +/// on their content, so every other `Cargo.toml` under a Cargo root is +/// fetched and the engine repeats the walk over them. +const CARGO_SKIP_SEGMENT: &str = "target"; + +/// The file `cargo vendor` writes into every crate it copies. The disk walk +/// reads vendored crates only when a path dependency or `[patch]` path +/// names one; without their manifests the engine fails closed +/// (`redirect_cargo_transitive_dependents`), which beats streaming +/// thousands of registry manifests into the file limits. +const CARGO_VENDOR_CHECKSUM: &str = ".cargo-checksum.json"; + +/// Whether `manifest_dir` (root-relative) or a directory above it, below +/// the root, holds a `cargo vendor` checksum file. +fn is_cargo_vendored(manifest_dir: &str, root_files: &BTreeSet<&str>) -> bool { + let mut dir = manifest_dir; + loop { + if root_files.contains(format!("{dir}/{CARGO_VENDOR_CHECKSUM}").as_str()) { + return true; + } + match dir.rsplit_once('/') { + Some((parent, _)) => dir = parent, + None => return false, + } + } +} + +/// A normalized repo-relative file path, or `None` when it is unsafe: +/// absolute, `..`/`.`/empty segments, backslashes, control characters, +/// overlong or overdeep. +pub fn safe_repo_path(path: &str) -> Option { + if path.is_empty() + || path.len() > MAX_PATH_LEN + || path.starts_with('/') + || path.contains('\\') + || path.chars().any(char::is_control) + { + return None; + } + let segments: Vec<&str> = path.split('/').collect(); + if segments.len() > MAX_PATH_DEPTH + || segments + .iter() + .any(|s| s.is_empty() || *s == "." || *s == "..") + { + return None; + } + Some(path.to_string()) +} + +/// A normalized repo-relative directory (`""` is the repo root; a trailing +/// `/` is dropped), or `None` when unsafe. +pub fn safe_root_path(root: &str) -> Option { + let trimmed = root.strip_suffix('/').unwrap_or(root); + if trimmed.is_empty() || trimmed == "." { + return Some(String::new()); + } + safe_repo_path(trimmed) +} + +/// Whether the engine keeps `path` as raw bytes rather than UTF-8 text. +pub fn is_binary_candidate(path: &str) -> bool { + split_path(path).1 == BUN_LOCKB +} + +/// How a root-relative file is consumed, when it is a candidate at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Need { + Text, + Binary, + Present, +} + +fn is_rush_subspace_lock(rel: &str) -> bool { + let Some(rest) = rel + .strip_prefix(RUSH_SUBSPACES_DIR) + .and_then(|r| r.strip_prefix('/')) + else { + return false; + }; + matches!(rest.split_once('/'), Some((name, "pnpm-lock.yaml")) if !name.is_empty()) +} + +/// What `rel` (relative to a root whose files are `root_files`) is needed +/// for. +fn classify(rel: &str, root_files: &BTreeSet<&str>) -> Option { + if rel == BUN_LOCKB { + return Some(Need::Binary); + } + if REDIRECT_CANDIDATE_FILES.contains(&rel) || EXTRA_TEXT_FILES.contains(&rel) { + return Some(Need::Text); + } + if PNP_MARKERS.contains(&rel) || rel == "rush.json" { + return Some(Need::Present); + } + if !rel.contains('/') { + if is_python_lock_name(rel) { + return Some(Need::Text); + } + if rel.ends_with(".py") && root_files.contains(format!("{rel}.lock").as_str()) { + return Some(Need::Text); + } + return None; + } + let rush = root_files.contains("rush.json"); + if rush && (rel == RUSH_COMMON_LOCK_REL || is_rush_subspace_lock(rel)) { + return Some(Need::Text); + } + if rush && rel == RUSH_REPO_STATE_REL { + return Some(Need::Present); + } + if let Some(manifest_dir) = rel.strip_suffix("/Cargo.toml") { + if root_files.contains("Cargo.toml") + && !manifest_dir.split('/').any(|seg| seg == CARGO_SKIP_SEGMENT) + && !is_cargo_vendored(manifest_dir, root_files) + { + return Some(Need::Text); + } + } + None +} + +/// `selectHostedScanPaths`: roots (detected, or `options.projectRoots`) +/// plus the files to stream for them. Only `blob` entries are files; mode +/// `120000` is a symbolic link and is reported, never fetched. +pub fn select_paths(entries: &[TreeEntryInput], options: &SelectOptions) -> PathSelection { + let mut ignored: Vec = Vec::new(); + let mut blobs: BTreeMap = BTreeMap::new(); + for entry in entries { + if entry.kind != "blob" { + continue; + } + match safe_repo_path(&entry.path) { + Some(path) => { + blobs.insert(path, entry.mode == "120000"); + } + None => ignored.push(IgnoredPath { + path: entry.path.chars().take(MAX_PATH_LEN).collect(), + reason: "unsafe_path".to_string(), + }), + } + } + + let roots: Vec = match &options.project_roots { + Some(requested) => { + let mut out: BTreeSet = BTreeSet::new(); + for root in requested { + match safe_root_path(root) { + Some(r) => { + out.insert(r); + } + None => ignored.push(IgnoredPath { + path: root.clone(), + reason: "invalid_project_root".to_string(), + }), + } + } + out.into_iter().collect() + } + None => { + let (found, skipped) = detect_roots( + blobs.keys().map(String::as_str), + options.ecosystems.as_deref(), + ); + ignored.extend(skipped); + found + } + }; + let root_set: BTreeSet<&str> = roots.iter().map(String::as_str).collect(); + + let mut per_root: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for path in blobs.keys() { + let mut dir = path.as_str(); + loop { + dir = match dir.rsplit_once('/') { + Some((parent, _)) => parent, + None => "", + }; + if let Some(root) = root_set.get(dir) { + if let Some(rel) = strip_root(root, path) { + per_root.entry(root).or_default().insert(rel); + } + } + if dir.is_empty() { + break; + } + } + } + + let mut needs: BTreeMap = BTreeMap::new(); + for (root, files) in &per_root { + for rel in files { + let Some(need) = classify(rel, files) else { + continue; + }; + let full = super::roots::join_root(root, rel); + let slot = needs.entry(full).or_insert(need); + *slot = (*slot).min(need); + } + } + + for (eco, markers) in UNSUPPORTED_MARKERS { + if !options + .ecosystems + .as_deref() + .is_none_or(|list| list.iter().any(|e| e == eco)) + { + continue; + } + let first = blobs.keys().find(|path| { + let (dir, base) = split_path(path); + markers.contains(&base) + && !dir + .split('/') + .any(|seg| EXCLUDED_ROOT_SEGMENTS.contains(&seg)) + }); + if let Some(path) = first { + needs.entry(path.clone()).or_insert(Need::Present); + } + } + + let mut selection = PathSelection { + roots, + ..PathSelection::default() + }; + for (path, need) in needs { + if blobs.get(&path).copied().unwrap_or(false) { + selection.symlinks.push(path); + continue; + } + match need { + Need::Text => selection.fetch_text.push(path), + Need::Binary => selection.fetch_binary.push(path), + Need::Present => selection.present_only.push(path), + } + } + ignored.sort_by(|a, b| a.path.cmp(&b.path)); + selection.ignored_count = ignored.len() as u64; + ignored.truncate(IGNORED_SAMPLE_MAX); + selection.ignored_sample = ignored; + selection +} + +/// `hostedScanCandidateFiles`: the root-relative candidate names and +/// patterns, for debugging only (selection is [`select_paths`]). +pub fn candidate_files() -> Vec { + let mut out: BTreeSet = REDIRECT_CANDIDATE_FILES + .iter() + .chain(EXTRA_TEXT_FILES.iter()) + .chain(PNP_MARKERS.iter()) + .map(|s| s.to_string()) + .collect(); + for pattern in [ + "rush.json", + RUSH_COMMON_LOCK_REL, + RUSH_REPO_STATE_REL, + "common/config/subspaces/*/pnpm-lock.yaml", + "*.py.lock", + "*.py (beside *.py.lock)", + "pylock.toml", + "pylock.*.toml", + "**/Cargo.toml (Cargo workspaces)", + ] { + out.insert(pattern.to_string()); + } + out.into_iter().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn blob(path: &str) -> TreeEntryInput { + TreeEntryInput { + path: path.into(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(10), + } + } + + #[test] + fn selects_root_candidates_only() { + let mut entries = vec![ + blob("package-lock.json"), + blob("package.json"), + blob("src/index.js"), + blob(".npmrc"), + blob("bun.lockb"), + blob(".pnp.cjs"), + blob("tool.py.lock"), + blob("tool.py"), + blob("other.py"), + blob(".socket/vendor/redirect-state.json"), + blob("web/yarn.lock"), + blob("web/.yarnrc.yml"), + blob("web/node_modules/x/package-lock.json"), + ]; + entries.push(TreeEntryInput { + path: "pnpm-workspace.yaml".into(), + mode: "120000".into(), + kind: "blob".into(), + size: None, + }); + entries.push(TreeEntryInput { + path: "sub".into(), + mode: "160000".into(), + kind: "commit".into(), + size: None, + }); + entries.push(blob("../escape/package-lock.json")); + let s = select_paths(&entries, &SelectOptions::default()); + assert_eq!(s.roots, vec!["", "web"]); + assert_eq!( + s.fetch_text, + vec![ + ".npmrc", + ".socket/vendor/redirect-state.json", + "package-lock.json", + "tool.py", + "tool.py.lock", + "web/.yarnrc.yml", + "web/yarn.lock" + ] + ); + assert_eq!(s.fetch_binary, vec!["bun.lockb"]); + assert_eq!(s.present_only, vec![".pnp.cjs"]); + assert_eq!(s.symlinks, vec!["pnpm-workspace.yaml"]); + assert_eq!(s.ignored_count, 2); + } + + #[test] + fn rush_and_cargo_members_are_fetched_under_their_root() { + let entries = vec![ + blob("rush.json"), + blob("common/config/rush/pnpm-lock.yaml"), + blob("common/config/rush/repo-state.json"), + blob("common/config/subspaces/a/pnpm-lock.yaml"), + blob("rs/Cargo.toml"), + blob("rs/Cargo.lock"), + blob("rs/crates/x/Cargo.toml"), + blob("rs/vendor/foo/Cargo.toml"), + blob("rs/target/debug/Cargo.toml"), + blob("rs/vendor/serde/Cargo.toml"), + blob("rs/vendor/serde/.cargo-checksum.json"), + blob("rs/vendor/serde/tests/ui/Cargo.toml"), + blob("rs/third_party/.cargo-checksum.json"), + blob("rs/third_party/Cargo.toml"), + ]; + let s = select_paths(&entries, &SelectOptions::default()); + assert_eq!(s.roots, vec!["", "rs"]); + assert_eq!( + s.fetch_text, + vec![ + "common/config/rush/pnpm-lock.yaml", + "common/config/subspaces/a/pnpm-lock.yaml", + "rs/Cargo.lock", + "rs/Cargo.toml", + "rs/crates/x/Cargo.toml", + "rs/vendor/foo/Cargo.toml" + ] + ); + assert_eq!( + s.present_only, + vec!["common/config/rush/repo-state.json", "rush.json"] + ); + } + + #[test] + fn a_maven_or_nuget_only_repo_still_names_one_marker_per_ecosystem() { + let entries = vec![ + blob("svc/b/pom.xml"), + blob("svc/a/pom.xml"), + blob("test/pom.xml"), + blob("app/nuget.config"), + blob("src/Main.java"), + ]; + let s = select_paths(&entries, &SelectOptions::default()); + assert!(s.roots.is_empty()); + assert!(s.fetch_text.is_empty()); + assert_eq!(s.present_only, vec!["app/nuget.config", "svc/a/pom.xml"]); + let s = select_paths( + &entries, + &SelectOptions { + project_roots: None, + ecosystems: Some(vec!["npm".into()]), + }, + ); + assert!(s.present_only.is_empty()); + } + + #[test] + fn explicit_roots_override_detection() { + let entries = vec![blob("a/package-lock.json"), blob("b/yarn.lock")]; + let s = select_paths( + &entries, + &SelectOptions { + project_roots: Some(vec!["b/".into(), "../x".into()]), + ecosystems: None, + }, + ); + assert_eq!(s.roots, vec!["b"]); + assert_eq!(s.fetch_text, vec!["b/yarn.lock"]); + assert_eq!(s.ignored_sample[0].reason, "invalid_project_root"); + } + + #[test] + fn path_safety() { + for bad in [ + "", "/abs", "a/../b", "a//b", "./a", "a\\b", "a\u{0}b", "a\nb", + ] { + assert_eq!(safe_repo_path(bad), None, "{bad:?}"); + } + assert_eq!(safe_repo_path("a/b.json").as_deref(), Some("a/b.json")); + assert_eq!(safe_root_path("").as_deref(), Some("")); + assert_eq!(safe_root_path("a/").as_deref(), Some("a")); + } + + #[test] + fn candidate_listing_is_sorted_and_names_the_ledgers() { + let listed = candidate_files(); + let mut sorted = listed.clone(); + sorted.sort(); + assert_eq!(listed, sorted); + assert!(listed + .iter() + .any(|f| f == ".socket/vendor/redirect-state.json")); + assert!(listed.iter().any(|f| f == "package-lock.json")); + } +} diff --git a/crates/socket-patch-cli/src/hosted_memory/types.rs b/crates/socket-patch-cli/src/hosted_memory/types.rs new file mode 100644 index 000000000..ed8e3a848 --- /dev/null +++ b/crates/socket-patch-cli/src/hosted_memory/types.rs @@ -0,0 +1,403 @@ +//! The engine's input and output shapes. Every struct serializes camelCase +//! and maps 1:1 onto the addon's JS contract (`HostedScanSessionOptions`, +//! `HostedScanResult`, `ProjectResult`, `PathSelection`, …), so a binding +//! converts them field by field. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// Ecosystems the addon contract names (`Ecosystem` in index.d.ts). +pub const ECOSYSTEMS: [&str; 8] = [ + "npm", "pypi", "cargo", "golang", "gem", "composer", "maven", "nuget", +]; + +/// `HostedScanLimits`. `None` fields take the defaults in +/// [`ResolvedLimits::DEFAULT`]. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct HostedScanLimits { + #[serde(skip_serializing_if = "Option::is_none")] + pub max_file_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_total_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_files: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_purls: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_projects: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_artifact_bytes: Option, +} + +/// [`HostedScanLimits`] with every default applied. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ResolvedLimits { + pub max_file_bytes: u64, + pub max_total_bytes: u64, + pub max_files: u64, + pub max_purls: u64, + pub max_projects: u64, + pub max_artifact_bytes: u64, +} + +impl ResolvedLimits { + pub const DEFAULT: ResolvedLimits = ResolvedLimits { + max_file_bytes: 20 * 1024 * 1024, + max_total_bytes: 64 * 1024 * 1024, + max_files: 2000, + max_purls: 20_000, + max_projects: 200, + max_artifact_bytes: 32 * 1024 * 1024, + }; +} + +impl HostedScanLimits { + pub fn resolve(&self) -> ResolvedLimits { + let d = ResolvedLimits::DEFAULT; + ResolvedLimits { + max_file_bytes: self.max_file_bytes.unwrap_or(d.max_file_bytes), + max_total_bytes: self.max_total_bytes.unwrap_or(d.max_total_bytes), + max_files: self.max_files.unwrap_or(d.max_files), + max_purls: self.max_purls.unwrap_or(d.max_purls), + max_projects: self.max_projects.unwrap_or(d.max_projects), + max_artifact_bytes: self.max_artifact_bytes.unwrap_or(d.max_artifact_bytes), + } + } +} + +/// `HostedScanSessionOptions`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HostedScanOptions { + pub org_slug: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ecosystems: Option>, + /// 1..=500, default 100. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub batch_size: Option, + #[serde(default)] + pub dry_run: bool, + /// The installing Pipenv's major; `None` is the CLI's "Pipenv not on + /// PATH" default (modern `file` references). Never probed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pipenv_major: Option, + /// Default true. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trust_lockfile_config: Option, + /// Default true. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub npm_allow_remote_config: Option, + /// Must match the `projectRoots` given to path selection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub project_roots: Option>, + /// Default 8. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_concurrency: Option, + /// Per provider call, default 60000. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_timeout_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limits: Option, +} + +pub const DEFAULT_BATCH_SIZE: u32 = 100; +pub const MAX_BATCH_SIZE: u32 = 500; +pub const DEFAULT_PROVIDER_CONCURRENCY: u32 = 8; +pub const DEFAULT_REQUEST_TIMEOUT_MS: u64 = 60_000; +/// The reference endpoint accepts at most this many uuids per request. +pub const MAX_REFERENCE_BATCH: usize = 500; + +/// Why a path was marked present without content (`markPresent` kinds +/// other than `symlink`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PresentKind { + Present, + BinarySkipped, + Oversize, + LfsPointer, +} + +impl PresentKind { + pub fn parse(kind: &str) -> Option { + Some(match kind { + "present" => MarkKind::Present(PresentKind::Present), + "binary_skipped" => MarkKind::Present(PresentKind::BinarySkipped), + "oversize" => MarkKind::Present(PresentKind::Oversize), + "lfs_pointer" => MarkKind::Present(PresentKind::LfsPointer), + "symlink" => MarkKind::Symlink, + _ => return None, + }) + } +} + +/// A `markPresent(path, kind)` kind. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MarkKind { + Present(PresentKind), + Symlink, +} + +/// One repo file handed to the engine. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InputFile { + Text(String), + Binary(Vec), + Present(PresentKind), + Symlink, +} + +/// Everything [`super::run_in_memory`] consumes. Build it with +/// [`super::SessionBuilder`] (which enforces the session limits) or +/// directly for tests. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HostedScanInput { + pub options: HostedScanOptions, + /// Repo-relative `/`-separated paths. + pub files: BTreeMap, + /// Session-level warnings (e.g. a text file that was not UTF-8). + pub warnings: Vec, +} + +/// `EngineWarning`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EngineWarning { + pub code: String, + pub detail: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub project_root: Option, +} + +impl EngineWarning { + pub fn new(code: impl Into, detail: impl Into, root: Option<&str>) -> Self { + Self { + code: code.into(), + detail: detail.into(), + project_root: root.map(str::to_string), + } + } +} + +/// `ProjectResult.summary`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProjectSummary { + pub scanned_packages: u64, + pub packages_with_patches: u64, + pub total_patches: u64, + pub free_patches: u64, + pub paid_patches: u64, + pub can_access_paid_patches: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RedirectedPatch { + pub purl: String, + pub uuid: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SkippedPatch { + pub purl: String, + pub uuid: String, + pub reason: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProjectError { + pub code: String, + pub message: String, +} + +/// `ProjectResult`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProjectResult { + /// Repo-relative project root (`""` is the repo root). + pub root: String, + /// The CLI `--json` `redirect` block, byte-for-byte the same shape. + pub redirect: serde_json::Value, + pub summary: ProjectSummary, + pub redirected: Vec, + pub skipped: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChangedFile { + pub path: String, + pub content: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChangedBinaryFile { + pub path: String, + #[serde(with = "base64_bytes")] + pub content: Vec, +} + +/// `HostedScanResult.stats`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EngineStats { + pub projects: u64, + pub files_input: u64, + pub bytes_input: u64, + pub packages_scanned: u64, + pub packages_with_patches: u64, + pub patches_selected: u64, + pub patches_redirected: u64, + pub files_changed: u64, + /// Provider calls made, by method name (`searchPatchesBatch`, …). + pub provider_calls: BTreeMap, + /// Wall time per engine phase, milliseconds. + pub phase_ms: BTreeMap, +} + +/// `HostedScanResult`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HostedScanOutput { + pub projects: Vec, + /// Repo-relative, sorted, byte-changed only; ledgers included on wet + /// runs. + pub changed_files: Vec, + pub changed_binary_files: Vec, + pub deleted_files: Vec, + pub warnings: Vec, + pub stats: EngineStats, + pub engine_version: String, +} + +/// `TreeEntryInput`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TreeEntryInput { + pub path: String, + pub mode: String, + #[serde(rename = "type")] + pub kind: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, +} + +/// `selectHostedScanPaths` options. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SelectOptions { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub project_roots: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ecosystems: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IgnoredPath { + pub path: String, + pub reason: String, +} + +/// `PathSelection`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PathSelection { + pub roots: Vec, + pub fetch_text: Vec, + pub fetch_binary: Vec, + pub present_only: Vec, + pub symlinks: Vec, + pub ignored_count: u64, + /// At most [`super::select::IGNORED_SAMPLE_MAX`] entries. + pub ignored_sample: Vec, +} + +/// Engine failure (`finish()` rejection codes). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EngineError { + /// A size/count limit was breached (`code` names the limit). + Limit { code: &'static str, message: String }, + /// The input or options are unusable. + InvalidInput { code: &'static str, message: String }, + /// Cancelled through the cancellation token. + Cancelled, + /// An engine bug. + Internal { message: String }, +} + +impl EngineError { + /// The stable machine code (`cancelled`, `limit_exceeded`'s limit name, + /// …). + pub fn code(&self) -> &str { + match self { + EngineError::Limit { code, .. } | EngineError::InvalidInput { code, .. } => code, + EngineError::Cancelled => "cancelled", + EngineError::Internal { .. } => "engine_internal", + } + } + + /// `limit` / `invalid_input` / `cancelled` / `internal`. + pub fn kind(&self) -> &'static str { + match self { + EngineError::Limit { .. } => "limit", + EngineError::InvalidInput { .. } => "invalid_input", + EngineError::Cancelled => "cancelled", + EngineError::Internal { .. } => "internal", + } + } + + pub(crate) fn limit(code: &'static str, message: impl Into) -> Self { + EngineError::Limit { + code, + message: message.into(), + } + } + + pub(crate) fn invalid(code: &'static str, message: impl Into) -> Self { + EngineError::InvalidInput { + code, + message: message.into(), + } + } +} + +impl std::fmt::Display for EngineError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EngineError::Limit { code, message } | EngineError::InvalidInput { code, message } => { + write!(f, "{code}: {message}") + } + EngineError::Cancelled => write!(f, "cancelled"), + EngineError::Internal { message } => write!(f, "engine_internal: {message}"), + } + } +} + +impl std::error::Error for EngineError {} + +mod base64_bytes { + use base64::Engine; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(bytes: &[u8], s: S) -> Result { + s.serialize_str(&base64::engine::general_purpose::STANDARD.encode(bytes)) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let text = String::deserialize(d)?; + base64::engine::general_purpose::STANDARD + .decode(text) + .map_err(serde::de::Error::custom) + } +} diff --git a/crates/socket-patch-cli/src/lib.rs b/crates/socket-patch-cli/src/lib.rs index 6406d9ab1..da5533f04 100644 --- a/crates/socket-patch-cli/src/lib.rs +++ b/crates/socket-patch-cli/src/lib.rs @@ -8,6 +8,7 @@ pub mod args; pub mod commands; pub(crate) mod ecosystem_dispatch; +pub mod hosted_memory; pub mod json_envelope; pub mod path_scope; pub mod ui; @@ -108,6 +109,12 @@ pub enum Commands { override_usage = "socket-patch --update [VERSION] [OPTIONS]" )] SelfUpdate(commands::update::UpdateArgs), + + // Internal parity/debug harness for the in-memory hosted engine: reads a + // JSON file bundle on stdin, prints the engine result. Hidden and + // documented as internal in CLI_CONTRACT.md (no stability guarantee). + #[command(hide = true, name = "hosted-bundle")] + HostedBundle(commands::hosted_bundle::HostedBundleArgs), } impl Commands { @@ -127,6 +134,7 @@ impl Commands { Commands::Remove(a) => &a.common, Commands::Repair(a) => &a.common, Commands::SelfUpdate(a) => &a.common, + Commands::HostedBundle(a) => &a.common, } } } diff --git a/crates/socket-patch-cli/src/main.rs b/crates/socket-patch-cli/src/main.rs index a1434d097..b50bff6b2 100644 --- a/crates/socket-patch-cli/src/main.rs +++ b/crates/socket-patch-cli/src/main.rs @@ -107,6 +107,7 @@ async fn main() { Commands::Remove(args) => commands::remove::run(args).await, Commands::Repair(args) => commands::repair::run(args).await, Commands::SelfUpdate(args) => commands::update::run(args).await, + Commands::HostedBundle(args) => commands::hosted_bundle::run(args).await, }; // Never delays exit beyond its 500 ms grace budget; never changes the diff --git a/crates/socket-patch-cli/tests/cli_global_args.rs b/crates/socket-patch-cli/tests/cli_global_args.rs index b0ca583cf..334e800a1 100644 --- a/crates/socket-patch-cli/tests/cli_global_args.rs +++ b/crates/socket-patch-cli/tests/cli_global_args.rs @@ -42,6 +42,8 @@ const SUBCOMMANDS_NO_POSITIONAL: &[&str] = &[ // Hidden parse target of the root `--update` flag; its VERSION // positional is optional, so the no-positional variant covers it. "self-update", + // Hidden internal harness for the in-memory hosted engine. + "hosted-bundle", ]; /// Subcommands that require a positional identifier. @@ -137,6 +139,7 @@ fn common_of(cli: &Cli) -> &GlobalArgs { Vendor(a) => &a.common, Vex(a) => &a.common, SelfUpdate(a) => &a.common, + HostedBundle(a) => &a.common, } } diff --git a/crates/socket-patch-cli/tests/hosted_memory_common/mod.rs b/crates/socket-patch-cli/tests/hosted_memory_common/mod.rs new file mode 100644 index 000000000..c0e60e47b --- /dev/null +++ b/crates/socket-patch-cli/tests/hosted_memory_common/mod.rs @@ -0,0 +1,413 @@ +//! Shared harness for the in-memory hosted engine tests: a wiremock patch +//! API that serves every patch a fixture's `overrides.json` describes, the +//! engine run over that API, and a disk `scan --mode hosted --json` run of +//! the same files through the real binary under a scrubbed environment. + +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use serde_json::Value; +use socket_patch_cli::hosted_memory::{ + run_in_memory, HostedScanOptions, HostedScanOutput, MarkKind, PresentKind, SessionBuilder, +}; +use socket_patch_core::api::client::{ApiClient, ApiClientOptions, PatchApi}; +use tokio_util::sync::CancellationToken; +use wiremock::matchers::{method, path, path_regex}; +use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate}; + +pub const ORG: &str = "test-org"; + +pub fn fixtures_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../socket-patch-core/tests/fixtures") +} + +/// One patch the fake API serves. +#[derive(Debug, Clone)] +pub struct Patch { + pub purl: String, + pub uuid: String, + pub reference: Value, +} + +fn percent_decode(s: &str) -> String { + socket_patch_core::utils::purl::percent_decode_purl_component(s).into_owned() +} + +fn purl_key(purl: &str) -> String { + let decoded = socket_patch_core::utils::purl::normalize_purl( + socket_patch_core::utils::purl::strip_purl_qualifiers(purl), + ) + .into_owned(); + if decoded.starts_with("pkg:pypi/") { + decoded.to_ascii_lowercase().replace(['_', '.'], "-") + } else { + decoded + } +} + +/// Patches from a golden fixture's `overrides.json` (one `DepOverride` +/// each). `rewrite_host` replaces `https://patch.socket.dev` in the served +/// URLs (so a wheel-metadata download hits the mock, never the network). +pub fn patches_from_overrides(overrides: &Path, rewrite_host: Option<&str>) -> Vec { + let text = std::fs::read_to_string(overrides).expect("read overrides.json"); + let list: Vec = serde_json::from_str(&text).expect("overrides.json is a list"); + list.into_iter() + .map(|o| { + let eco = o["ecosystem"].as_str().unwrap().to_string(); + let name = o["name"].as_str().unwrap().to_string(); + let full = match o["namespace"].as_str() { + Some(ns) if !ns.is_empty() => format!("{ns}/{name}"), + _ => name, + }; + let version = o["version"].as_str().unwrap(); + let purl = format!("pkg:{eco}/{full}@{version}"); + let uuid = o["patchUuid"].as_str().unwrap().to_string(); + let fix = |v: &Value| -> Value { + match (v.as_str(), rewrite_host) { + (Some(s), Some(host)) => { + Value::String(s.replace("https://patch.socket.dev", host)) + } + _ => v.clone(), + } + }; + let url = fix(&o["artifactUrl"]); + let mut artifacts = vec![serde_json::json!({ + "kind": "tarball", + "url": url, + "integrity": o["integrity"].clone(), + })]; + if let Some(zip) = o["berryZipUrl"].as_str() { + artifacts.push(serde_json::json!({ + "kind": "yarn-berry-zip", + "url": fix(&Value::String(zip.to_string())), + "integrity": {"yarnBerry10c0": o["integrity"]["yarnBerry10c0"].clone()}, + })); + } + let mut registry_override = o.get("registryOverride").cloned().unwrap_or(Value::Null); + if let Some(index) = registry_override.get("indexUrl").cloned() { + registry_override["indexUrl"] = fix(&index); + } + Patch { + purl, + uuid, + reference: serde_json::json!({ + "status": "granted", + "url": url, + "purl": null, + "artifacts": artifacts, + "registryOverride": registry_override, + }), + } + }) + .collect() +} + +struct Batch(Vec); +impl Respond for Batch { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let mut packages = Vec::new(); + for component in body["components"].as_array().into_iter().flatten() { + let Some(purl) = component["purl"].as_str() else { + continue; + }; + let patches: Vec = self + .0 + .iter() + .filter(|p| purl_key(&p.purl) == purl_key(purl)) + .map(|p| { + serde_json::json!({ + "uuid": p.uuid, "purl": purl, "tier": "free", "cveIds": [], + "ghsaIds": ["GHSA-test-aaaa-bbbb"], "severity": "high", "title": "fixture" + }) + }) + .collect(); + if !patches.is_empty() { + packages.push(serde_json::json!({ "purl": purl, "patches": patches })); + } + } + ResponseTemplate::new(200).set_body_json( + serde_json::json!({ "packages": packages, "canAccessPaidPatches": false }), + ) + } +} + +struct ByPackage(Vec); +impl Respond for ByPackage { + fn respond(&self, request: &Request) -> ResponseTemplate { + let raw = request + .url + .path() + .rsplit_once("/by-package/") + .map(|(_, p)| p) + .unwrap_or(""); + let purl = percent_decode(raw); + let patches: Vec = self + .0 + .iter() + .filter(|p| purl_key(&p.purl) == purl_key(&purl)) + .map(|p| { + serde_json::json!({ + "uuid": p.uuid, "purl": purl, "publishedAt": "2024-01-01T00:00:00Z", + "description": "fixture", "license": "MIT", "tier": "free", + "vulnerabilities": {"GHSA-test-aaaa-bbbb": { + "cves": ["CVE-2024-0001"], "summary": "s", "severity": "high", "description": "d" + }} + }) + }) + .collect(); + ResponseTemplate::new(200) + .set_body_json(serde_json::json!({ "patches": patches, "canAccessPaidPatches": false })) + } +} + +struct References(Vec); +impl Respond for References { + fn respond(&self, request: &Request) -> ResponseTemplate { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let mut results = serde_json::Map::new(); + for uuid in body["uuids"].as_array().into_iter().flatten() { + let Some(uuid) = uuid.as_str() else { continue }; + if let Some(p) = self.0.iter().find(|p| p.uuid == uuid) { + results.insert(uuid.to_string(), p.reference.clone()); + } + } + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "results": results })) + } +} + +struct View(Vec); +impl Respond for View { + fn respond(&self, request: &Request) -> ResponseTemplate { + let uuid = request.url.path().rsplit('/').next().unwrap_or(""); + match self.0.iter().find(|p| p.uuid == uuid) { + Some(p) => ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "uuid": p.uuid, "purl": p.purl, "publishedAt": "2024-01-01T00:00:00Z", + "files": {"package/index.js": {"beforeHash": "a".repeat(64), "afterHash": "b".repeat(64)}}, + "vulnerabilities": {"GHSA-test-aaaa-bbbb": { + "cves": ["CVE-2024-0001"], "summary": "s", "severity": "high", "description": "d" + }}, + "description": "fixture", "license": "MIT", "tier": "free" + })), + None => ResponseTemplate::new(404), + } + } +} + +/// Mount the fake patch API for `patches`. +pub async fn mount_api(server: &MockServer, patches: &[Patch]) { + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .respond_with(Batch(patches.to_vec())) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path_regex(format!( + "^/v0/orgs/{ORG}/patches/by-package/.+$" + ))) + .respond_with(ByPackage(patches.to_vec())) + .mount(server) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/package"))) + .respond_with(References(patches.to_vec())) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path_regex(format!("^/v0/orgs/{ORG}/patches/view/.+$"))) + .respond_with(View(patches.to_vec())) + .mount(server) + .await; +} + +pub fn client(server: &MockServer) -> Arc { + Arc::new(ApiClient::new(ApiClientOptions { + api_url: server.uri(), + api_token: Some("fake-token".to_string()), + use_public_proxy: false, + org_slug: Some(ORG.to_string()), + })) +} + +/// Every regular file under `dir`, keyed by `/`-separated relative path. +pub fn read_tree(dir: &Path) -> BTreeMap> { + fn walk(base: &Path, at: &Path, out: &mut BTreeMap>) { + let Ok(entries) = std::fs::read_dir(at) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + let meta = std::fs::symlink_metadata(&path).unwrap(); + if meta.is_dir() { + walk(base, &path, out); + } else if meta.is_file() { + let rel = path + .strip_prefix(base) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + out.insert(rel, std::fs::read(&path).unwrap()); + } + } + } + let mut out = BTreeMap::new(); + walk(dir, dir, &mut out); + out +} + +/// `files` added to a session (`bun.lockb` as bytes, the rest as text) +/// plus presence-only markers. +pub fn build_input( + files: &BTreeMap>, + present: &[&str], + options: HostedScanOptions, +) -> socket_patch_cli::hosted_memory::HostedScanInput { + let mut builder = SessionBuilder::new(options).expect("valid options"); + for (path, bytes) in files { + if path.ends_with("bun.lockb") { + builder.add_binary(path, bytes).unwrap(); + } else { + builder.push_chunk(path, bytes).unwrap(); + builder.end_file(path).unwrap(); + } + } + for path in present { + builder + .mark_present(path, MarkKind::Present(PresentKind::Present)) + .unwrap(); + } + builder.finish().unwrap() +} + +pub fn options(dry_run: bool) -> HostedScanOptions { + HostedScanOptions { + org_slug: ORG.to_string(), + dry_run, + ..HostedScanOptions::default() + } +} + +pub async fn run_engine( + server: &MockServer, + input: socket_patch_cli::hosted_memory::HostedScanInput, +) -> HostedScanOutput { + run_in_memory(input, client(server), CancellationToken::new()) + .await + .expect("engine run") +} + +/// The disk run's outcome: the `--json` envelope and the files it changed +/// (new or byte-changed, relative to the input). +pub struct DiskRun { + pub envelope: Value, + pub changed: BTreeMap>, + pub stderr: String, +} + +/// `socket-patch scan --mode hosted --json` over a copy of `files`, through +/// the real binary, with every ambient input scrubbed: an empty `PATH` (no +/// node / pipenv / gem subprocesses), `HOME` and the language caches at +/// empty directories, no socket-cli config, no telemetry. +pub fn run_disk(server: &MockServer, files: &BTreeMap>, dry_run: bool) -> DiskRun { + let project = tempfile::tempdir().unwrap(); + let home = tempfile::tempdir().unwrap(); + for (rel, bytes) in files { + let path = project.path().join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, bytes).unwrap(); + } + let mut cmd = std::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")); + cmd.env_clear(); + for keep in [ + "SYSTEMROOT", + "SystemRoot", + "windir", + "TMPDIR", + "TEMP", + "TMP", + ] { + if let Some(v) = std::env::var_os(keep) { + cmd.env(keep, v); + } + } + let empty = |name: &str| { + let dir = home.path().join(name); + std::fs::create_dir_all(&dir).unwrap(); + dir + }; + cmd.env("PATH", "") + .env("HOME", home.path()) + .env("USERPROFILE", home.path()) + .env("CARGO_HOME", empty("cargo")) + .env("GOPATH", empty("go")) + .env("GOMODCACHE", empty("gomodcache")) + .env("GEM_HOME", empty("gem")) + .env("XDG_CONFIG_HOME", empty("xdg")) + .env("SOCKET_NO_CONFIG", "1") + .env("SOCKET_TELEMETRY_DISABLED", "1") + .env("SOCKET_NO_UPDATE_CHECK", "1"); + cmd.args([ + "scan", + "--mode", + "hosted", + "--json", + "--yes", + "--cwd", + project.path().to_str().unwrap(), + "--org", + ORG, + "--api-token", + "fake-token", + "--api-url", + &server.uri(), + ]); + if dry_run { + cmd.arg("--dry-run"); + } + let output = cmd.output().expect("spawn socket-patch"); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + let envelope: Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("disk --json output is not JSON ({e}):\n{stdout}\n{stderr}")); + let after = read_tree(project.path()); + let changed = after + .into_iter() + .filter(|(rel, bytes)| files.get(rel) != Some(bytes)) + .collect(); + DiskRun { + envelope, + changed, + stderr, + } +} + +/// The engine's changed files (text and binary) as bytes. +pub fn engine_changed(output: &HostedScanOutput) -> BTreeMap> { + output + .changed_files + .iter() + .map(|f| (f.path.clone(), f.content.as_bytes().to_vec())) + .chain( + output + .changed_binary_files + .iter() + .map(|f| (f.path.clone(), f.content.clone())), + ) + .collect() +} + +/// Every file under `dir` (a golden fixture's `input/`). +pub fn fixture_files(dir: &Path) -> BTreeMap> { + read_tree(dir) +} + +/// Show a readable diff of two changed-file maps. +pub fn describe(map: &BTreeMap>) -> String { + map.iter() + .map(|(k, v)| format!("--- {k}\n{}", String::from_utf8_lossy(v))) + .collect::>() + .join("\n") +} diff --git a/crates/socket-patch-cli/tests/hosted_memory_engine.rs b/crates/socket-patch-cli/tests/hosted_memory_engine.rs new file mode 100644 index 000000000..9357c3e0c --- /dev/null +++ b/crates/socket-patch-cli/tests/hosted_memory_engine.rs @@ -0,0 +1,932 @@ +//! Behavior of the in-memory hosted engine beyond disk parity: lookup +//! dedup across roots, isolation from the process environment, hostile +//! and malformed input, limits, cancellation and timeouts, dry runs, and +//! the per-project refusals. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use serde_json::Value; +use serial_test::serial; +use socket_patch_cli::hosted_memory::{ + run_in_memory, select_paths, EngineError, HostedScanLimits, HostedScanOptions, + HostedScanOutput, SelectOptions, SessionBuilder, TreeEntryInput, +}; +use socket_patch_core::api::client::{ApiError, ApiFuture, PatchApi}; +use socket_patch_core::api::types::{ + BatchSearchResponse, PackageVendorResult, PatchResponse, SearchResponse, +}; +use tokio_util::sync::CancellationToken; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +#[path = "hosted_memory_common/mod.rs"] +mod common; + +use common::*; + +const NPM_FIXTURE: &str = "redirect/npm/package-lock-v3/basic"; + +fn npm_files() -> BTreeMap> { + fixture_files(&fixtures_root().join(NPM_FIXTURE).join("input")) +} + +async fn npm_server() -> MockServer { + let server = MockServer::start().await; + let patches = patches_from_overrides( + &fixtures_root().join(NPM_FIXTURE).join("overrides.json"), + None, + ); + mount_api(&server, &patches).await; + server +} + +async fn count(server: &MockServer, method_name: &str, fragment: &str) -> usize { + server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|r| r.method.as_str() == method_name && r.url.path().contains(fragment)) + .count() +} + +fn prefixed(prefix: &str, files: &BTreeMap>) -> BTreeMap> { + files + .iter() + .map(|(k, v)| (format!("{prefix}/{k}"), v.clone())) + .collect() +} + +fn comparable(output: &HostedScanOutput) -> Value { + let mut value = serde_json::to_value(output).unwrap(); + value["stats"]["phaseMs"] = Value::Null; + value +} + +#[tokio::test] +async fn roots_sharing_a_purl_share_every_lookup() { + let server = npm_server().await; + let mut repo = prefixed("a", &npm_files()); + repo.extend(prefixed("b", &npm_files())); + let output = run_engine(&server, build_input(&repo, &[], options(false))).await; + assert_eq!(output.projects.len(), 2); + for project in &output.projects { + assert!(project.error.is_none(), "{:?}", project.error); + assert_eq!(project.redirected.len(), 1, "{}", project.root); + } + assert_eq!(count(&server, "POST", "/patches/batch").await, 1); + assert_eq!(count(&server, "GET", "/patches/by-package/").await, 1); + assert_eq!(count(&server, "POST", "/patches/package").await, 1); + assert_eq!(count(&server, "GET", "/patches/view/").await, 1); + let paths: Vec<&str> = output + .changed_files + .iter() + .map(|f| f.path.as_str()) + .collect(); + assert_eq!( + paths, + vec![ + "a/.npmrc", + "a/.socket/vendor/redirect-state.json", + "a/package-lock.json", + "b/.npmrc", + "b/.socket/vendor/redirect-state.json", + "b/package-lock.json" + ] + ); + assert_eq!(output.stats.projects, 2); + assert_eq!(output.stats.patches_redirected, 2); + assert_eq!( + output.stats.provider_calls.get("searchPatchesBatch"), + Some(&1) + ); +} + +#[tokio::test] +async fn output_is_deterministic() { + let server = npm_server().await; + let mut repo = prefixed("x", &npm_files()); + repo.extend(prefixed("y/z", &npm_files())); + let first = run_engine(&server, build_input(&repo, &[], options(false))).await; + let second = run_engine(&server, build_input(&repo, &[], options(false))).await; + assert_eq!(comparable(&first), comparable(&second)); +} + +const HOSTILE_ENV: &[(&str, &str)] = &[ + ("SOCKET_API_URL", "http://127.0.0.1:9"), + ("SOCKET_API_TOKEN", "sktsec_hostile_api"), + ("SOCKET_ORG_SLUG", "hostile-org"), + ("SOCKET_PROXY_URL", "http://127.0.0.1:9"), + ("SOCKET_OFFLINE", "1"), + ("SOCKET_DRY_RUN", "1"), + ("SOCKET_ECOSYSTEMS", "cargo"), + ("SOCKET_BATCH_SIZE", "1"), + ("SOCKET_PIPENV_MAJOR", "7"), + ("SOCKET_NO_TRUST_LOCKFILE_CONFIG", "1"), + ("SOCKET_NO_NPM_ALLOW_REMOTE_CONFIG", "1"), + ("SOCKET_TELEMETRY_DISABLED", "0"), + ("SOCKET_NPM_REGISTRY", "http://127.0.0.1:9"), + ("SOCKET_PYPI_JSON_API", "http://127.0.0.1:9"), + ("npm_config_allow_remote", "none"), + ("NPM_CONFIG_USERCONFIG", "/nonexistent/.npmrc"), +]; + +#[tokio::test] +#[serial] +async fn hostile_process_environment_changes_nothing() { + let server = npm_server().await; + let mut repo = npm_files(); + repo.insert( + "pnpm-lock.yaml".into(), + std::fs::read(fixtures_root().join("redirect/npm/pnpm/basic/input/pnpm-lock.yaml")) + .unwrap(), + ); + let baseline = run_engine(&server, build_input(&repo, &[], options(false))).await; + let saved: Vec<(&str, Option)> = HOSTILE_ENV + .iter() + .map(|(k, _)| (*k, std::env::var_os(k))) + .collect(); + for (k, v) in HOSTILE_ENV { + std::env::set_var(k, v); + } + let hostile = run_engine(&server, build_input(&repo, &[], options(false))).await; + for (k, v) in saved { + match v { + Some(v) => std::env::set_var(k, v), + None => std::env::remove_var(k), + } + } + assert_eq!(comparable(&baseline), comparable(&hostile)); + assert!(!baseline.projects[0].redirected.is_empty()); +} + +fn deep(open: &str, close: &str, depth: usize) -> Vec { + let mut s = String::with_capacity(depth * (open.len() + close.len())); + for _ in 0..depth { + s.push_str(open); + } + for _ in 0..depth { + s.push_str(close); + } + s.into_bytes() +} + +async fn run_on_small_stack( + server: &MockServer, + input: socket_patch_cli::hosted_memory::HostedScanInput, +) -> Result { + let api = client(server); + let handle = tokio::runtime::Handle::current(); + tokio::task::spawn_blocking(move || { + std::thread::Builder::new() + .stack_size(2 * 1024 * 1024) + .spawn(move || handle.block_on(run_in_memory(input, api, CancellationToken::new()))) + .unwrap() + .join() + .expect("the engine must not panic or overflow on hostile input") + }) + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn malformed_and_deeply_nested_inputs_never_panic() { + let server = MockServer::start().await; + let mut patches = patches_from_overrides( + &fixtures_root().join(NPM_FIXTURE).join("overrides.json"), + None, + ); + patches.extend(patches_from_overrides( + &fixtures_root().join("redirect/cargo/cargo/basic/overrides.json"), + None, + )); + patches.extend(patches_from_overrides( + &fixtures_root().join("redirect/pypi/requirements/basic/overrides.json"), + None, + )); + mount_api(&server, &patches).await; + let depth = 200_000; + let mut repo: BTreeMap> = BTreeMap::new(); + repo.extend(prefixed("npm", &npm_files())); + repo.insert("npm/.yarnrc.yml".into(), deep("- ", "", depth)); + repo.insert("npm/pnpm-workspace.yaml".into(), deep("[", "]", depth)); + repo.insert("npm/.npmrc".into(), deep("[", "]", depth)); + repo.extend(prefixed( + "rs", + &fixture_files(&fixtures_root().join("redirect/cargo/cargo/basic/input")), + )); + repo.insert("rs/.cargo/config.toml".into(), deep("a = [", "]", depth)); + repo.extend(prefixed( + "py", + &fixture_files(&fixtures_root().join("redirect/pypi/requirements/basic/input")), + )); + repo.insert("py/pyproject.toml".into(), deep("x = [", "]", depth)); + repo.insert("py/hatch.toml".into(), deep("{a=", "}", depth)); + repo.insert("deepjson/package-lock.json".into(), deep("[", "]", depth)); + repo.insert("deepjson/composer.lock".into(), deep("{\"a\":", "}", depth)); + repo.insert("deepjson/Pipfile.lock".into(), deep("[", "]", depth)); + repo.insert( + "deepjson/.socket/vendor/redirect-state.json".into(), + deep("[", "]", depth), + ); + repo.insert("deeptoml/uv.lock".into(), deep("a = [", "]", depth)); + repo.insert("deeptoml/Cargo.lock".into(), deep("a = {b=", "}", depth)); + repo.insert("deeptoml/Cargo.toml".into(), deep("[", "]", depth)); + repo.insert("deeptoml/poetry.lock".into(), deep("a = [", "]", depth)); + repo.insert( + "garbage/yarn.lock".into(), + b"\x00\x01 not a lock \"\"\"\n ::".to_vec(), + ); + repo.insert( + "garbage/go.mod".into(), + b"module \nrequire (\n(((\n".to_vec(), + ); + repo.insert("garbage/go.sum".into(), b"x y z\n\n h1:\n".to_vec()); + repo.insert( + "garbage/Gemfile.lock".into(), + b"GEM\n specs:\n (((\n".to_vec(), + ); + repo.insert( + "garbage/bun.lock".into(), + b"{\"lockfileVersion\": 99,".to_vec(), + ); + repo.insert("garbage2/bun.lockb".into(), vec![0xa5; 4096]); + repo.insert("garbage3/pnpm-lock.yaml".into(), deep("- ", "", depth)); + let mut opts = options(false); + opts.limits = Some(HostedScanLimits { + max_total_bytes: Some(64 * 1024 * 1024), + ..HostedScanLimits::default() + }); + let output = run_on_small_stack(&server, build_input(&repo, &[], opts)) + .await + .expect("engine result"); + let roots: Vec<&str> = output.projects.iter().map(|p| p.root.as_str()).collect(); + for expected in [ + "npm", "rs", "py", "deepjson", "deeptoml", "garbage", "garbage2", + ] { + assert!( + roots.contains(&expected), + "{expected} missing from {roots:?}" + ); + } + let deepjson = output + .projects + .iter() + .find(|p| p.root == "deepjson") + .unwrap(); + assert_eq!(deepjson.error.as_ref().unwrap().code, "corrupt_ledger"); +} + +#[test] +fn session_limits_reject_oversized_input() { + let mut opts = options(false); + opts.limits = Some(HostedScanLimits { + max_files: Some(1), + ..HostedScanLimits::default() + }); + let mut builder = SessionBuilder::new(opts).unwrap(); + builder.add_text("a/package-lock.json", "{}").unwrap(); + let err = builder.add_text("b/package-lock.json", "{}").unwrap_err(); + assert_eq!(err.code(), "max_files"); + assert_eq!(err.kind(), "limit"); +} + +#[tokio::test] +async fn project_and_purl_limits_reject_the_run() { + let server = npm_server().await; + let mut repo = prefixed("a", &npm_files()); + repo.extend(prefixed("b", &npm_files())); + let mut opts = options(false); + opts.limits = Some(HostedScanLimits { + max_projects: Some(1), + ..HostedScanLimits::default() + }); + let err = run_in_memory( + build_input(&repo, &[], opts), + client(&server), + CancellationToken::new(), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + EngineError::Limit { + code: "max_projects", + .. + } + ), + "{err}" + ); + + let mut opts = options(false); + opts.limits = Some(HostedScanLimits { + max_purls: Some(0), + ..HostedScanLimits::default() + }); + let err = run_in_memory( + build_input(&npm_files(), &[], opts), + client(&server), + CancellationToken::new(), + ) + .await + .unwrap_err(); + assert_eq!(err.code(), "max_purls"); + assert_eq!(count(&server, "POST", "/patches/batch").await, 0); +} + +/// A provider whose every call pends forever. +struct Stalled; + +impl PatchApi for Stalled { + fn uses_public_proxy(&self) -> bool { + false + } + fn search_patches_batch<'a>(&'a self, _: &'a [String]) -> ApiFuture<'a, BatchSearchResponse> { + Box::pin(std::future::pending()) + } + fn search_patches_by_package<'a>(&'a self, _: &'a str) -> ApiFuture<'a, SearchResponse> { + Box::pin(std::future::pending()) + } + fn fetch_registry_references<'a>( + &'a self, + _: &'a [String], + ) -> ApiFuture<'a, std::collections::HashMap> { + Box::pin(std::future::pending()) + } + fn fetch_patch<'a>(&'a self, _: &'a str) -> ApiFuture<'a, Option> { + Box::pin(std::future::pending()) + } + fn download_artifact<'a>(&'a self, _: &'a str, _: u64) -> ApiFuture<'a, Vec> { + Box::pin(async { Err(ApiError::Other("unused".into())) }) + } +} + +#[tokio::test] +async fn cancellation_rejects_with_cancelled() { + let cancel = CancellationToken::new(); + let trigger = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + trigger.cancel(); + }); + let err = run_in_memory( + build_input(&npm_files(), &[], options(false)), + Arc::new(Stalled), + cancel, + ) + .await + .unwrap_err(); + assert_eq!(err, EngineError::Cancelled); + assert_eq!(err.code(), "cancelled"); + + let cancelled = CancellationToken::new(); + cancelled.cancel(); + let err = run_in_memory( + build_input(&npm_files(), &[], options(false)), + Arc::new(Stalled), + cancelled, + ) + .await + .unwrap_err(); + assert_eq!(err, EngineError::Cancelled); +} + +/// A provider that counts batch searches, each of which pends forever. +#[derive(Default)] +struct CountingStalled(std::sync::atomic::AtomicUsize); + +impl PatchApi for CountingStalled { + fn uses_public_proxy(&self) -> bool { + false + } + fn search_patches_batch<'a>(&'a self, _: &'a [String]) -> ApiFuture<'a, BatchSearchResponse> { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Box::pin(std::future::pending()) + } + fn search_patches_by_package<'a>(&'a self, _: &'a str) -> ApiFuture<'a, SearchResponse> { + Box::pin(std::future::pending()) + } + fn fetch_registry_references<'a>( + &'a self, + _: &'a [String], + ) -> ApiFuture<'a, std::collections::HashMap> { + Box::pin(std::future::pending()) + } + fn fetch_patch<'a>(&'a self, _: &'a str) -> ApiFuture<'a, Option> { + Box::pin(std::future::pending()) + } + fn download_artifact<'a>(&'a self, _: &'a str, _: u64) -> ApiFuture<'a, Vec> { + Box::pin(async { Err(ApiError::Other("unused".into())) }) + } +} + +/// The inventory phase never pends on memory reads; it must still yield +/// between roots so a cancel (and every other task on this single-threaded +/// runtime) runs before the whole phase — and the first provider call — +/// completes. +#[tokio::test(flavor = "current_thread")] +async fn cancellation_is_honored_between_roots_of_the_inventory() { + let mut repo: BTreeMap> = BTreeMap::new(); + for i in 0..20 { + repo.extend(prefixed(&format!("r{i:02}"), &npm_files())); + } + let cancel = CancellationToken::new(); + let trigger = cancel.clone(); + tokio::spawn(async move { trigger.cancel() }); + let provider = Arc::new(CountingStalled::default()); + let err = run_in_memory( + build_input(&repo, &[], options(false)), + provider.clone(), + cancel, + ) + .await + .unwrap_err(); + assert_eq!(err, EngineError::Cancelled); + assert_eq!(provider.0.load(std::sync::atomic::Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn provider_timeouts_become_project_errors() { + let mut opts = options(false); + opts.request_timeout_ms = Some(30); + let output = run_in_memory( + build_input(&npm_files(), &[], opts), + Arc::new(Stalled), + CancellationToken::new(), + ) + .await + .unwrap(); + let error = output.projects[0].error.as_ref().unwrap(); + assert_eq!(error.code, "patch_lookup_failed"); + assert!(error.message.contains("timed out"), "{}", error.message); + assert!(output.changed_files.is_empty()); +} + +#[tokio::test] +async fn unauthorized_is_a_project_error_without_proxy_fallback() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .respond_with(ResponseTemplate::new(401).set_body_string("bad token")) + .mount(&server) + .await; + let output = run_engine(&server, build_input(&npm_files(), &[], options(false))).await; + let error = output.projects[0].error.as_ref().unwrap(); + assert_eq!(error.code, "patch_lookup_failed"); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests + .iter() + .all(|r| !r.url.path().starts_with("/patch/"))); +} + +#[tokio::test] +async fn dry_run_previews_without_records_or_ledger() { + let server = npm_server().await; + let output = run_engine(&server, build_input(&npm_files(), &[], options(true))).await; + let paths: Vec<&str> = output + .changed_files + .iter() + .map(|f| f.path.as_str()) + .collect(); + assert_eq!(paths, vec![".npmrc", "package-lock.json"]); + assert_eq!(output.projects[0].redirect["dryRun"], true); + assert_eq!(count(&server, "GET", "/patches/view/").await, 0); + assert_eq!(count(&server, "POST", "/patches/package").await, 1); +} + +#[tokio::test] +async fn symlinked_workspace_config_refuses_the_project() { + let dir = fixtures_root().join("redirect/npm/pnpm/basic"); + let server = MockServer::start().await; + mount_api( + &server, + &patches_from_overrides(&dir.join("overrides.json"), None), + ) + .await; + let files = fixture_files(&dir.join("input")); + let mut builder = SessionBuilder::new(options(false)).unwrap(); + for (path, bytes) in &files { + builder + .add_text(path, std::str::from_utf8(bytes).unwrap()) + .unwrap(); + } + builder + .mark_present( + "pnpm-workspace.yaml", + socket_patch_cli::hosted_memory::MarkKind::Symlink, + ) + .unwrap(); + let output = run_engine(&server, builder.finish().unwrap()).await; + let error = output.projects[0].error.as_ref().unwrap(); + assert_eq!(error.code, "redirect_symlinked_file_unsupported"); + assert!(output.changed_files.is_empty()); +} + +#[tokio::test] +async fn symlinked_npmrc_is_left_alone_with_a_warning() { + let server = npm_server().await; + let mut builder = SessionBuilder::new(options(false)).unwrap(); + for (path, bytes) in &npm_files() { + builder + .add_text(path, std::str::from_utf8(bytes).unwrap()) + .unwrap(); + } + builder + .mark_present(".npmrc", socket_patch_cli::hosted_memory::MarkKind::Symlink) + .unwrap(); + let output = run_engine(&server, builder.finish().unwrap()).await; + let project = &output.projects[0]; + assert!(project.error.is_none()); + let warnings = project.redirect["warnings"].as_array().unwrap(); + let npm = warnings + .iter() + .find(|w| w["code"] == "redirect_npm_allow_remote") + .unwrap(); + assert!(npm["detail"].as_str().unwrap().contains("symbolic link")); + assert!(output.changed_files.iter().all(|f| f.path != ".npmrc")); +} + +#[tokio::test] +async fn vendored_takeover_is_refused() { + let server = npm_server().await; + let mut files = npm_files(); + files.insert( + ".socket/vendor/state.json".into(), + serde_json::to_vec(&serde_json::json!({ + "version": 1, + "entries": { + "pkg:npm/left-pad@1.3.0": { + "ecosystem": "npm", + "basePurl": "pkg:npm/left-pad@1.3.0", + "uuid": "22222222-2222-2222-2222-222222222222", + "artifact": {"path": ".socket/vendor/npm/22222222-2222-2222-2222-222222222222/left-pad-1.3.0.tgz"}, + "wiring": [] + } + } + })) + .unwrap(), + ); + let output = run_engine(&server, build_input(&files, &[], options(false))).await; + let project = &output.projects[0]; + assert!(project + .skipped + .iter() + .any(|s| s.reason == "vendored_takeover_unsupported_in_memory")); + assert!(project.redirected.is_empty()); + let warnings = project.redirect["warnings"].as_array().unwrap(); + assert!(warnings + .iter() + .any(|w| w["code"] == "vendored_takeover_unsupported_in_memory")); + assert!(output.changed_files.is_empty()); +} + +#[tokio::test] +async fn corrupt_ledger_fails_only_its_project() { + let server = npm_server().await; + let mut repo = prefixed("good", &npm_files()); + repo.extend(prefixed("bad", &npm_files())); + repo.insert( + "bad/.socket/vendor/redirect-state.json".into(), + b"{ torn".to_vec(), + ); + let output = run_engine(&server, build_input(&repo, &[], options(false))).await; + let bad = output.projects.iter().find(|p| p.root == "bad").unwrap(); + let good = output.projects.iter().find(|p| p.root == "good").unwrap(); + assert_eq!(bad.error.as_ref().unwrap().code, "corrupt_ledger"); + assert_eq!(bad.redirect, serde_json::json!({ "mode": "hosted" })); + assert!(good.error.is_none()); + assert_eq!(good.redirected.len(), 1); + assert!(output + .changed_files + .iter() + .all(|f| !f.path.starts_with("bad/"))); +} + +#[tokio::test] +async fn maven_files_warn_that_the_ecosystem_is_unsupported() { + let server = npm_server().await; + let mut files = npm_files(); + files.insert("pom.xml".into(), b"".to_vec()); + let output = run_engine(&server, build_input(&files, &[], options(false))).await; + assert!(output + .warnings + .iter() + .any(|w| w.code == "ecosystem_unsupported_in_memory" + && w.project_root.as_deref() == Some(""))); +} + +#[tokio::test] +async fn a_maven_only_repo_warns_through_selection() { + let server = npm_server().await; + let entries: Vec = ["java/app/pom.xml", "java/app/src/Main.java"] + .iter() + .map(|p| TreeEntryInput { + path: (*p).to_string(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(1), + }) + .collect(); + let selection = select_paths(&entries, &SelectOptions::default()); + assert!(selection.roots.is_empty()); + let present: Vec<&str> = selection.present_only.iter().map(String::as_str).collect(); + assert_eq!(present, vec!["java/app/pom.xml"]); + let output = run_engine( + &server, + build_input(&BTreeMap::new(), &present, options(false)), + ) + .await; + assert!(output.projects.is_empty()); + let warning = output + .warnings + .iter() + .find(|w| w.code == "ecosystem_unsupported_in_memory") + .expect("the unsupported ecosystem is reported"); + assert!(warning.project_root.is_none()); + assert!( + warning.detail.contains("java/app/pom.xml"), + "{}", + warning.detail + ); +} + +#[tokio::test] +async fn an_unreadable_higher_precedence_lock_refuses_the_project() { + let server = npm_server().await; + let output = run_engine( + &server, + build_input(&npm_files(), &["npm-shrinkwrap.json"], options(false)), + ) + .await; + let project = &output.projects[0]; + assert_eq!( + project.error.as_ref().map(|e| e.code.as_str()), + Some("candidate_file_unreadable"), + "{:#}", + project.redirect + ); + assert!(project.redirected.is_empty()); + assert!(output.changed_files.is_empty()); +} + +#[tokio::test] +async fn selection_drives_the_engine_roots() { + let server = npm_server().await; + let mut repo = prefixed("apps/web", &npm_files()); + repo.extend(prefixed("apps/web/test/fixture", &npm_files())); + repo.insert("apps/web/src/index.js".into(), b"x".to_vec()); + let entries: Vec = repo + .keys() + .map(|p| TreeEntryInput { + path: p.clone(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(1), + }) + .collect(); + let selection = select_paths(&entries, &SelectOptions::default()); + assert_eq!(selection.roots, vec!["apps/web"]); + let fetched: BTreeMap> = selection + .fetch_text + .iter() + .map(|p| (p.clone(), repo[p].clone())) + .collect(); + let mut opts = options(false); + opts.project_roots = Some(selection.roots.clone()); + let output = run_engine(&server, build_input(&fetched, &[], opts)).await; + assert_eq!(output.projects.len(), 1); + assert_eq!(output.projects[0].root, "apps/web"); + assert_eq!(output.projects[0].redirected.len(), 1); +} + +#[tokio::test] +async fn ecosystem_filter_skips_other_ecosystems() { + let server = npm_server().await; + let mut opts = options(false); + opts.ecosystems = Some(vec!["cargo".to_string()]); + let output = run_engine(&server, build_input(&npm_files(), &[], opts)).await; + assert!(output.projects.is_empty() || output.projects[0].summary.scanned_packages == 0); + assert_eq!(count(&server, "POST", "/patches/batch").await, 0); +} + +#[tokio::test] +async fn invalid_options_reject() { + let server = npm_server().await; + let err = run_in_memory( + build_input(&npm_files(), &[], options(false)), + client(&server), + CancellationToken::new(), + ) + .await + .map(|_| ()) + .err(); + assert!(err.is_none()); + let err = SessionBuilder::new(HostedScanOptions { + org_slug: String::new(), + ..HostedScanOptions::default() + }) + .unwrap_err(); + assert_eq!(err.code(), "invalid_org_slug"); +} + +#[tokio::test] +async fn hosted_bundle_command_prints_the_engine_result() { + let server = npm_server().await; + let files: BTreeMap = npm_files() + .into_iter() + .map(|(k, v)| (k, String::from_utf8(v).unwrap())) + .collect(); + let bundle = serde_json::json!({ "files": files, "presentOnly": [".pnp.loader.mjs"] }); + let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")) + .args([ + "hosted-bundle", + "--org", + ORG, + "--api-token", + "fake-token", + "--api-url", + &server.uri(), + ]) + .env("SOCKET_NO_CONFIG", "1") + .env("SOCKET_TELEMETRY_DISABLED", "1") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + { + use std::io::Write; + child + .stdin + .take() + .unwrap() + .write_all(bundle.to_string().as_bytes()) + .unwrap(); + } + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let result: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(result["projects"][0]["root"], ""); + let warnings = result["warnings"].as_array().unwrap(); + assert!( + warnings.iter().any(|w| w["code"] == "yarn_pnp_unsupported"), + "{warnings:?}" + ); + assert!(result["engineVersion"] + .as_str() + .unwrap() + .starts_with(env!("CARGO_PKG_VERSION"))); + + let missing = std::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")) + .args(["hosted-bundle"]) + .env_remove("SOCKET_API_TOKEN") + .env_remove("SOCKET_ORG_SLUG") + .env("SOCKET_NO_CONFIG", "1") + .stdin(std::process::Stdio::null()) + .output() + .unwrap(); + assert_eq!(missing.status.code(), Some(2)); +} + +#[tokio::test] +async fn warnings_are_scoped_to_their_project() { + let server = npm_server().await; + let output = run_engine( + &server, + build_input( + &prefixed("pkg", &npm_files()), + &["pkg/.pnp.cjs"], + options(false), + ), + ) + .await; + let pnp = output + .warnings + .iter() + .find(|w| w.code == "yarn_pnp_unsupported") + .expect("pnp refusal"); + assert_eq!(pnp.project_root.as_deref(), Some("pkg")); +} + +#[tokio::test] +async fn stats_count_inputs_and_calls() { + let server = npm_server().await; + let files = npm_files(); + let bytes: usize = files.values().map(Vec::len).sum(); + let output = run_engine(&server, build_input(&files, &[], options(false))).await; + assert_eq!(output.stats.files_input, files.len() as u64); + assert_eq!(output.stats.bytes_input, bytes as u64); + assert_eq!(output.stats.patches_selected, 1); + assert_eq!( + output.stats.files_changed, + output.changed_files.len() as u64 + ); + for method_name in [ + "searchPatchesBatch", + "searchPatchesByPackage", + "fetchRegistryReferences", + "fetchPatch", + ] { + assert_eq!( + output.stats.provider_calls.get(method_name), + Some(&1), + "{method_name}" + ); + } +} + +#[test] +fn engine_future_is_send_and_static() { + fn assert_send_static(_: &T) {} + let input = build_input(&npm_files(), &[], options(false)); + let future = run_in_memory(input, Arc::new(Stalled), CancellationToken::new()); + assert_send_static(&future); +} + +#[test] +fn contract_shapes_are_camel_case() { + let options: HostedScanOptions = serde_json::from_value(serde_json::json!({ + "orgSlug": "org", "ecosystems": ["npm"], "batchSize": 50, "dryRun": true, + "pipenvMajor": 11, "trustLockfileConfig": false, "npmAllowRemoteConfig": false, + "projectRoots": ["a"], "providerConcurrency": 4, "requestTimeoutMs": 1000, + "limits": {"maxFileBytes": 1, "maxTotalBytes": 2, "maxFiles": 3, "maxPurls": 4, + "maxProjects": 5, "maxArtifactBytes": 6} + })) + .unwrap(); + assert_eq!(options.batch_size, Some(50)); + assert_eq!(options.limits.unwrap().max_artifact_bytes, Some(6)); + let entry: TreeEntryInput = serde_json::from_value(serde_json::json!({ + "path": "a/package-lock.json", "mode": "100644", "type": "blob", "size": 10 + })) + .unwrap(); + let selection = + serde_json::to_value(select_paths(&[entry], &SelectOptions::default())).unwrap(); + for key in [ + "roots", + "fetchText", + "fetchBinary", + "presentOnly", + "symlinks", + "ignoredCount", + "ignoredSample", + ] { + assert!( + selection.get(key).is_some(), + "{key} missing from {selection}" + ); + } +} + +#[tokio::test] +async fn result_serializes_with_the_contract_keys() { + let server = npm_server().await; + let output = run_engine(&server, build_input(&npm_files(), &[], options(false))).await; + let value = serde_json::to_value(&output).unwrap(); + for key in [ + "projects", + "changedFiles", + "changedBinaryFiles", + "deletedFiles", + "warnings", + "stats", + "engineVersion", + ] { + assert!(value.get(key).is_some(), "{key}"); + } + let project = &value["projects"][0]; + for key in ["root", "redirect", "summary", "redirected", "skipped"] { + assert!(project.get(key).is_some(), "{key}"); + } + for key in [ + "scannedPackages", + "packagesWithPatches", + "totalPatches", + "freePatches", + "paidPatches", + "canAccessPaidPatches", + ] { + assert!(project["summary"].get(key).is_some(), "{key}"); + } + for key in [ + "projects", + "filesInput", + "bytesInput", + "packagesScanned", + "packagesWithPatches", + "patchesSelected", + "patchesRedirected", + "filesChanged", + "providerCalls", + "phaseMs", + ] { + assert!(value["stats"].get(key).is_some(), "{key}"); + } +} diff --git a/crates/socket-patch-cli/tests/hosted_memory_parity.rs b/crates/socket-patch-cli/tests/hosted_memory_parity.rs new file mode 100644 index 000000000..14219cc0b --- /dev/null +++ b/crates/socket-patch-cli/tests/hosted_memory_parity.rs @@ -0,0 +1,616 @@ +//! Parity: the in-memory hosted engine over a file set produces the same +//! changed bytes, ledger bytes and `redirect` block as `scan --mode hosted +//! --json` over a checkout of the same files (the disk run goes through +//! the real binary under a scrubbed environment, so crawlers, host caches +//! and subprocesses contribute nothing on either side). + +use std::collections::BTreeMap; + +use base64::Engine; +use serde_json::Value; +use wiremock::MockServer; + +#[path = "hosted_memory_common/mod.rs"] +mod common; + +use common::*; + +struct Case { + /// Fixture dir under `crates/socket-patch-core/tests/fixtures/redirect`. + fixture: &'static str, + /// Extra files layered over the fixture input. + extra: Vec<(&'static str, Vec)>, + /// Require at least one redirect (a parity of two no-ops proves + /// nothing for the formats the engine must rewrite). + expect_redirect: bool, + dry_run: bool, +} + +fn case(fixture: &'static str) -> Case { + Case { + fixture, + extra: Vec::new(), + expect_redirect: true, + dry_run: false, + } +} + +async fn assert_parity(case: Case) { + let dir = fixtures_root().join("redirect").join(case.fixture); + let server = MockServer::start().await; + let patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let mut files = fixture_files(&dir.join("input")); + for (rel, bytes) in &case.extra { + files.insert((*rel).to_string(), bytes.clone()); + } + + let disk = run_disk(&server, &files, case.dry_run); + let input = build_input(&files, &[], options(case.dry_run)); + let memory = run_engine(&server, input).await; + + assert_eq!(memory.projects.len(), 1, "{}: one root", case.fixture); + let project = &memory.projects[0]; + assert_eq!(project.root, ""); + assert!( + project.error.is_none(), + "{}: {:?}", + case.fixture, + project.error + ); + let disk_redirect = disk + .envelope + .get("redirect") + .cloned() + .unwrap_or(Value::Null); + assert_eq!( + project.redirect, disk_redirect, + "{}: redirect block differs\nmemory: {:#}\ndisk: {:#}\nstderr: {}", + case.fixture, project.redirect, disk_redirect, disk.stderr + ); + let memory_changed = engine_changed(&memory); + let expected_changed = if case.dry_run { + assert!( + disk.changed.is_empty(), + "{}: a disk dry run wrote files", + case.fixture + ); + let mut wet = run_disk(&server, &files, false).changed; + wet.remove(".socket/vendor/redirect-state.json"); + wet + } else { + disk.changed.clone() + }; + assert_eq!( + memory_changed.keys().collect::>(), + expected_changed.keys().collect::>(), + "{}: changed file sets differ\nmemory:\n{}\ndisk:\n{}", + case.fixture, + describe(&memory_changed), + describe(&expected_changed) + ); + for (rel, bytes) in &expected_changed { + assert_eq!( + String::from_utf8_lossy(&memory_changed[rel]), + String::from_utf8_lossy(bytes), + "{}: {rel} differs", + case.fixture + ); + } + let redirected = disk_redirect["redirected"].as_u64().unwrap_or(0); + assert_eq!(redirected as usize, project.redirected.len()); + if case.expect_redirect { + assert!( + redirected > 0, + "{}: the fixture redirected nothing: {disk_redirect:#}\n{}", + case.fixture, + disk.stderr + ); + } + assert!( + project.summary.scanned_packages >= redirected, + "{}", + case.fixture + ); +} + +#[tokio::test] +async fn parity_package_lock() { + assert_parity(case("npm/package-lock-v3/basic")).await; +} + +#[tokio::test] +async fn parity_package_lock_dry_run() { + assert_parity(Case { + dry_run: true, + ..case("npm/package-lock-v3/basic") + }) + .await; +} + +#[tokio::test] +async fn parity_pnpm_v9_trust_lockfile() { + assert_parity(case("npm/pnpm/basic")).await; +} + +#[tokio::test] +async fn parity_pnpm_existing_workspace() { + assert_parity(Case { + extra: vec![("pnpm-workspace.yaml", b"packages:\n - 'apps/*'\n".to_vec())], + ..case("npm/pnpm/basic") + }) + .await; +} + +#[tokio::test] +async fn parity_yarn_classic() { + assert_parity(case("npm/yarn-classic/basic")).await; +} + +#[tokio::test] +async fn parity_yarn_berry() { + assert_parity(case("npm/yarn-berry/basic")).await; +} + +#[tokio::test] +async fn parity_bun_text_lock() { + assert_parity(case("npm/bun/basic")).await; +} + +#[tokio::test] +async fn parity_rush() { + assert_parity(Case { + extra: vec![ + ("rush.json", b"{}\n".to_vec()), + ("common/config/rush/repo-state.json", b"{}\n".to_vec()), + ], + ..case("npm/pnpm/nested-rush-lock") + }) + .await; +} + +#[tokio::test] +async fn parity_uv() { + assert_parity(Case { + expect_redirect: false, + ..case("pypi/uv/basic") + }) + .await; +} + +#[tokio::test] +async fn parity_requirements() { + assert_parity(case("pypi/requirements/basic")).await; +} + +#[tokio::test] +async fn parity_cargo() { + assert_parity(case("cargo/cargo/basic")).await; +} + +#[tokio::test] +async fn parity_cargo_workspace() { + assert_parity(case("cargo/cargo/workspace-member")).await; +} + +#[tokio::test] +async fn parity_composer() { + assert_parity(case("composer/composer-lock/basic")).await; +} + +#[tokio::test] +async fn parity_gemfile() { + assert_parity(case("gem/bundler/basic")).await; +} + +#[tokio::test] +async fn parity_golang() { + assert_parity(case("golang/gomod/basic")).await; +} + +/// Formats with no golden redirect fixture: a committed native lock plus a +/// synthetic override for one of its packages. +async fn assert_native_parity( + files: BTreeMap>, + overrides: Value, + expect_redirect: bool, +) { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("overrides.json"), overrides.to_string()).unwrap(); + let server = MockServer::start().await; + let patches = patches_from_overrides(&tmp.path().join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let disk = run_disk(&server, &files, false); + let memory = run_engine(&server, build_input(&files, &[], options(false))).await; + let project = &memory.projects[0]; + assert!(project.error.is_none(), "{:?}", project.error); + assert_eq!( + project.redirect, disk.envelope["redirect"], + "{}", + disk.stderr + ); + let memory_changed = engine_changed(&memory); + assert_eq!( + memory_changed, + disk.changed, + "memory:\n{}\ndisk:\n{}", + describe(&memory_changed), + describe(&disk.changed) + ); + if expect_redirect { + assert!(!project.redirected.is_empty(), "{:#}", project.redirect); + } +} + +fn read_fixture(rel: &str) -> Vec { + std::fs::read(fixtures_root().join(rel)).unwrap() +} + +#[tokio::test] +async fn parity_poetry() { + let files = BTreeMap::from([ + ( + "poetry.lock".to_string(), + read_fixture("poetry/2.4.3/poetry.lock"), + ), + ( + "pyproject.toml".to_string(), + read_fixture("poetry/2.4.3/pyproject.toml"), + ), + ]); + let overrides = serde_json::json!([{ + "ecosystem": "pypi", "name": "urllib3", "version": "1.26.18", + "token": "22222222-2222-4222-8222-222222222222", + "patchUuid": "e828efa5-5c6d-43f3-9909-03f5ac232b98", + "artifactUrl": "https://patch.socket.dev/patch/pypi/urllib3/1.26.18/22222222-2222-4222-8222-222222222222/e828efa5-5c6d-43f3-9909-03f5ac232b98/urllib3-1.26.18-py2.py3-none-any.whl", + "integrity": {"sha256": "c".repeat(64)} + }]); + assert_native_parity(files, overrides, true).await; +} + +#[tokio::test] +async fn parity_pipfile() { + let files = BTreeMap::from([ + ( + "Pipfile.lock".to_string(), + read_fixture("pipenv/2026.8.0/Pipfile.lock"), + ), + ( + "Pipfile".to_string(), + read_fixture("pipenv/2026.8.0/Pipfile"), + ), + ]); + let lock: Value = serde_json::from_slice(&files["Pipfile.lock"]).unwrap(); + let (name, entry) = lock["default"] + .as_object() + .and_then(|m| m.iter().next()) + .expect("a default package"); + let version = entry["version"].as_str().unwrap().trim_start_matches("=="); + let overrides = serde_json::json!([{ + "ecosystem": "pypi", "name": name, "version": version, + "token": "22222222-2222-4222-8222-222222222222", + "patchUuid": "e828efa5-5c6d-43f3-9909-03f5ac232b98", + "artifactUrl": format!("https://patch.socket.dev/patch/pypi/{name}/{version}/22222222-2222-4222-8222-222222222222/e828efa5-5c6d-43f3-9909-03f5ac232b98/{name}-{version}-py3-none-any.whl"), + "integrity": {"sha256": "c".repeat(64)} + }]); + assert_native_parity(files, overrides, true).await; +} + +#[tokio::test] +async fn parity_bun_binary_lock() { + let files = BTreeMap::from([ + ( + "bun.lockb".to_string(), + read_fixture("bun-lockb/1.1.45/bun.lockb"), + ), + ( + "package.json".to_string(), + read_fixture("bun-lockb/1.1.45/package.json"), + ), + ]); + let overrides = serde_json::json!([{ + "ecosystem": "npm", "name": "minimist", "version": "1.2.2", + "token": "22222222-2222-4222-8222-222222222222", + "patchUuid": "33333333-3333-4333-8333-333333333333", + "artifactUrl": "https://patch.socket.dev/patch/npm/minimist/1.2.2/22222222-2222-4222-8222-222222222222/33333333-3333-4333-8333-333333333333/minimist-1.2.2.tgz", + "integrity": {"sha512": format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode([0x5au8; 64]) + )} + }]); + assert_native_parity(files, overrides, true).await; +} + +#[tokio::test] +async fn parity_nested_monorepo_roots_match_their_own_disk_runs() { + let npm = fixtures_root().join("redirect/npm/package-lock-v3/basic"); + let cargo = fixtures_root().join("redirect/cargo/cargo/basic"); + let server = MockServer::start().await; + let mut patches = patches_from_overrides(&npm.join("overrides.json"), None); + patches.extend(patches_from_overrides(&cargo.join("overrides.json"), None)); + mount_api(&server, &patches).await; + let web = fixture_files(&npm.join("input")); + let svc = fixture_files(&cargo.join("input")); + let mut repo: BTreeMap> = BTreeMap::new(); + for (rel, bytes) in &web { + repo.insert(format!("apps/web/{rel}"), bytes.clone()); + } + for (rel, bytes) in &svc { + repo.insert(format!("services/api/{rel}"), bytes.clone()); + } + let memory = run_engine(&server, build_input(&repo, &[], options(false))).await; + let roots: Vec<&str> = memory.projects.iter().map(|p| p.root.as_str()).collect(); + assert_eq!(roots, vec!["apps/web", "services/api"]); + let changed = engine_changed(&memory); + for (root, files) in [("apps/web", &web), ("services/api", &svc)] { + let disk = run_disk(&server, files, false); + let project = memory.projects.iter().find(|p| p.root == root).unwrap(); + assert_eq!(project.redirect, disk.envelope["redirect"], "{root}"); + let prefixed: BTreeMap> = changed + .iter() + .filter_map(|(k, v)| { + k.strip_prefix(&format!("{root}/")) + .map(|rel| (rel.to_string(), v.clone())) + }) + .collect(); + assert_eq!(prefixed, disk.changed, "{root}"); + } +} + +fn wheel(name: &str, version: &str) -> Vec { + use std::io::Write; + let mut buf = std::io::Cursor::new(Vec::new()); + { + let mut zip = zip::ZipWriter::new(&mut buf); + let options = zip::write::SimpleFileOptions::default(); + zip.start_file(format!("{name}-{version}.dist-info/METADATA"), options) + .unwrap(); + write!( + zip, + "Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n" + ) + .unwrap(); + zip.finish().unwrap(); + } + buf.into_inner() +} + +#[tokio::test] +async fn parity_uv_with_hosted_wheel_metadata() { + use sha2::Digest; + let dir = fixtures_root().join("redirect/pypi/uv/basic"); + let server = MockServer::start().await; + let bytes = wheel("click", "8.1.7"); + let sha = hex::encode(sha2::Sha256::digest(&bytes)); + let tmp = tempfile::tempdir().unwrap(); + let mut overrides: Value = + serde_json::from_str(&std::fs::read_to_string(dir.join("overrides.json")).unwrap()) + .unwrap(); + overrides[0]["integrity"]["sha256"] = Value::String(sha); + std::fs::write(tmp.path().join("overrides.json"), overrides.to_string()).unwrap(); + let patches = patches_from_overrides(&tmp.path().join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path_regex("^/patch/pypi/.+\\.whl$")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_bytes(bytes)) + .mount(&server) + .await; + let files = fixture_files(&dir.join("input")); + let disk = run_disk(&server, &files, false); + let memory = run_engine(&server, build_input(&files, &[], options(false))).await; + let project = &memory.projects[0]; + assert_eq!( + project.redirect, disk.envelope["redirect"], + "{}", + disk.stderr + ); + assert_eq!(engine_changed(&memory), disk.changed); + assert_eq!(project.redirected.len(), 1, "{:#}", project.redirect); + assert_eq!( + memory.stats.provider_calls.get("downloadArtifact"), + Some(&1) + ); +} + +/// `files` narrowed to what [`select_paths`] asks the host to stream +/// (presence-only paths marked present), so the parity covers selection. +fn selected_input( + files: &BTreeMap>, +) -> socket_patch_cli::hosted_memory::HostedScanInput { + use socket_patch_cli::hosted_memory::{select_paths, SelectOptions, TreeEntryInput}; + let entries: Vec = files + .iter() + .map(|(p, bytes)| TreeEntryInput { + path: p.clone(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(bytes.len() as u64), + }) + .collect(); + let selection = select_paths(&entries, &SelectOptions::default()); + let fetched: BTreeMap> = selection + .fetch_text + .iter() + .chain(selection.fetch_binary.iter()) + .map(|p| (p.clone(), files[p].clone())) + .collect(); + let present: Vec<&str> = selection.present_only.iter().map(String::as_str).collect(); + let mut opts = options(false); + opts.project_roots = Some(selection.roots.clone()); + build_input(&fetched, &present, opts) +} + +#[tokio::test] +async fn parity_cargo_patch_path_under_vendor_through_selection() { + let dir = fixtures_root().join("redirect/cargo/cargo/workspace-member"); + let server = MockServer::start().await; + let patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let mut files = fixture_files(&dir.join("input")); + let mut manifest = String::from_utf8(files["Cargo.toml"].clone()).unwrap(); + manifest.push_str("\n[patch.crates-io]\nfoo = { path = \"vendor/foo\" }\n"); + files.insert("Cargo.toml".into(), manifest.into_bytes()); + let mut lock = String::from_utf8(files["Cargo.lock"].clone()).unwrap(); + lock.push_str( + "\n[[package]]\nname = \"foo\"\nversion = \"0.1.0\"\ndependencies = [\n \"serde\",\n]\n", + ); + files.insert("Cargo.lock".into(), lock.into_bytes()); + files.insert( + "vendor/foo/Cargo.toml".into(), + b"[package]\nname = \"foo\"\nversion = \"0.1.0\"\n\n[dependencies]\nserde = \"1.0.190\"\n" + .to_vec(), + ); + files.insert( + "tests/fixtures/other/Cargo.toml".into(), + b"[package]\nname = \"other\"\nversion = \"0.1.0\"\n\n[dependencies]\nserde = \"1\"\n" + .to_vec(), + ); + + let disk = run_disk(&server, &files, false); + assert!( + disk.changed.contains_key("vendor/foo/Cargo.toml"), + "disk pins the [patch] path crate: {}\n{}", + describe(&disk.changed), + disk.stderr + ); + let memory = run_engine(&server, selected_input(&files)).await; + let project = &memory.projects[0]; + assert!(project.error.is_none(), "{:?}", project.error); + assert_eq!( + project.redirect, disk.envelope["redirect"], + "{}", + disk.stderr + ); + let memory_changed = engine_changed(&memory); + assert_eq!( + memory_changed, + disk.changed, + "memory:\n{}\ndisk:\n{}", + describe(&memory_changed), + describe(&disk.changed) + ); + assert_eq!(project.redirected.len(), 1, "{:#}", project.redirect); +} + +/// `cargo vendor` output (`.cargo-checksum.json` in every crate) sits under +/// `vendor/` with a `[patch]`-free workspace: disk never reads it, and +/// selection must not fetch it either. +fn with_cargo_vendor_tree(files: &mut BTreeMap>) { + for krate in ["serde", "itoa"] { + files.insert( + format!("vendor/{krate}/Cargo.toml"), + format!("[package]\nname = \"{krate}\"\nversion = \"1.0.190\"\n").into_bytes(), + ); + files.insert( + format!("vendor/{krate}/.cargo-checksum.json"), + b"{\"files\":{},\"package\":\"00\"}".to_vec(), + ); + files.insert( + format!("vendor/{krate}/tests/ui/Cargo.toml"), + b"[package]\nname = \"ui\"\nversion = \"0.0.0\"\n".to_vec(), + ); + } +} + +fn selected_paths(files: &BTreeMap>) -> Vec { + use socket_patch_cli::hosted_memory::{select_paths, SelectOptions, TreeEntryInput}; + let entries: Vec = files + .keys() + .map(|p| TreeEntryInput { + path: p.clone(), + mode: "100644".into(), + kind: "blob".into(), + size: Some(1), + }) + .collect(); + let selection = select_paths(&entries, &SelectOptions::default()); + selection + .fetch_text + .into_iter() + .chain(selection.fetch_binary) + .chain(selection.present_only) + .collect() +} + +#[tokio::test] +async fn parity_cargo_vendor_tree_is_not_fetched() { + let dir = fixtures_root().join("redirect/cargo/cargo/workspace-member"); + let server = MockServer::start().await; + let patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let mut files = fixture_files(&dir.join("input")); + with_cargo_vendor_tree(&mut files); + + let selected = selected_paths(&files); + assert!( + !selected.iter().any(|p| p.starts_with("vendor/")), + "{selected:?}" + ); + let disk = run_disk(&server, &files, false); + assert!( + !disk.changed.keys().any(|p| p.starts_with("vendor/")), + "{}", + describe(&disk.changed) + ); + let memory = run_engine(&server, selected_input(&files)).await; + let project = &memory.projects[0]; + assert!(project.error.is_none(), "{:?}", project.error); + assert_eq!( + project.redirect, disk.envelope["redirect"], + "{}", + disk.stderr + ); + let memory_changed = engine_changed(&memory); + assert_eq!( + memory_changed, + disk.changed, + "memory:\n{}\ndisk:\n{}", + describe(&memory_changed), + describe(&disk.changed) + ); + assert!(!project.redirected.is_empty(), "{:#}", project.redirect); +} + +#[tokio::test] +async fn cargo_patch_path_into_vendor_tree_fails_closed() { + let dir = fixtures_root().join("redirect/cargo/cargo/workspace-member"); + let server = MockServer::start().await; + let patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let mut files = fixture_files(&dir.join("input")); + let mut manifest = String::from_utf8(files["Cargo.toml"].clone()).unwrap(); + manifest.push_str("\n[patch.crates-io]\nfoo = { path = \"vendor/foo\" }\n"); + files.insert("Cargo.toml".into(), manifest.into_bytes()); + let mut lock = String::from_utf8(files["Cargo.lock"].clone()).unwrap(); + lock.push_str( + "\n[[package]]\nname = \"foo\"\nversion = \"0.1.0\"\ndependencies = [\n \"serde\",\n]\n", + ); + files.insert("Cargo.lock".into(), lock.into_bytes()); + files.insert( + "vendor/foo/Cargo.toml".into(), + b"[package]\nname = \"foo\"\nversion = \"0.1.0\"\n\n[dependencies]\nserde = \"1.0.190\"\n" + .to_vec(), + ); + files.insert( + "vendor/foo/.cargo-checksum.json".into(), + b"{\"files\":{},\"package\":\"00\"}".to_vec(), + ); + + let memory = run_engine(&server, selected_input(&files)).await; + let project = &memory.projects[0]; + assert!(project.error.is_none(), "{:?}", project.error); + assert!(project.redirected.is_empty(), "{:#}", project.redirect); + assert!( + engine_changed(&memory).is_empty(), + "{}", + describe(&engine_changed(&memory)) + ); + assert!( + project + .redirect + .to_string() + .contains("redirect_cargo_transitive_dependents"), + "{:#}", + project.redirect + ); +} diff --git a/crates/socket-patch-core/src/api/client.rs b/crates/socket-patch-core/src/api/client.rs index ba3ff5963..b24592493 100644 --- a/crates/socket-patch-core/src/api/client.rs +++ b/crates/socket-patch-core/src/api/client.rs @@ -99,7 +99,7 @@ pub struct ApiClientOptions { /// Supports both the authenticated Socket API (`api.socket.dev`) and the /// public proxy (`patches-api.socket.dev`) which serves free patches /// without authentication. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct ApiClient { client: reqwest::Client, /// Header-free twin of `client` (User-Agent only, never Authorization) @@ -1917,7 +1917,7 @@ fn convert_search_result_to_batch_info(patch: PatchSearchResult) -> BatchPatchIn /// reaches a caller. `scan` renders `packages[].patches` straight to the /// operator and treats the leading entry as the patch apply will install; /// both only hold because of this. -fn sort_batch_response(response: &mut BatchSearchResponse) { +pub fn sort_batch_response(response: &mut BatchSearchResponse) { for pkg in &mut response.packages { pkg.patches.sort_by(cmp_batch_infos); } @@ -1997,6 +1997,302 @@ pub enum ApiError { Other(String), } +// ── Patch API seam ──────────────────────────────────────────────────── + +/// A boxed, `Send` future returned by [`PatchApi`] methods. +pub type ApiFuture<'a, T> = + std::pin::Pin> + Send + 'a>>; + +/// The patch-data calls the hosted flow makes, as an object-safe trait so an +/// embedding host (the in-memory engine, a napi addon) can serve them +/// in-process. [`ApiClient`] implements it by forwarding to its inherent +/// methods, so existing callers are unaffected. +pub trait PatchApi: Send + Sync { + /// Whether calls go to the public patch proxy (free patches only). + fn uses_public_proxy(&self) -> bool; + + /// `POST …/patches/batch` for up to 500 purls, normalized through + /// [`sort_batch_response`]. + fn search_patches_batch<'a>( + &'a self, + purls: &'a [String], + ) -> ApiFuture<'a, BatchSearchResponse>; + + /// `GET …/patches/by-package/`, results best-first. + fn search_patches_by_package<'a>(&'a self, purl: &'a str) -> ApiFuture<'a, SearchResponse>; + + /// `POST …/patches/package` (hosted reference grants), keyed by uuid. + fn fetch_registry_references<'a>( + &'a self, + uuids: &'a [String], + ) -> ApiFuture<'a, std::collections::HashMap>; + + /// `GET …/patches/view/`; `None` when not found. + fn fetch_patch<'a>(&'a self, uuid: &'a str) -> ApiFuture<'a, Option>; + + /// Download a grant-tokenized artifact URL, refusing bodies over + /// `max_bytes`. + fn download_artifact<'a>(&'a self, url: &'a str, max_bytes: u64) -> ApiFuture<'a, Vec>; +} + +impl PatchApi for ApiClient { + fn uses_public_proxy(&self) -> bool { + self.use_public_proxy + } + + fn search_patches_batch<'a>( + &'a self, + purls: &'a [String], + ) -> ApiFuture<'a, BatchSearchResponse> { + Box::pin(ApiClient::search_patches_batch(self, purls)) + } + + fn search_patches_by_package<'a>(&'a self, purl: &'a str) -> ApiFuture<'a, SearchResponse> { + Box::pin(ApiClient::search_patches_by_package(self, purl)) + } + + fn fetch_registry_references<'a>( + &'a self, + uuids: &'a [String], + ) -> ApiFuture<'a, std::collections::HashMap> { + Box::pin(ApiClient::fetch_registry_references(self, uuids)) + } + + fn fetch_patch<'a>(&'a self, uuid: &'a str) -> ApiFuture<'a, Option> { + Box::pin(ApiClient::fetch_patch(self, uuid)) + } + + fn download_artifact<'a>(&'a self, url: &'a str, max_bytes: u64) -> ApiFuture<'a, Vec> { + Box::pin(self.download_artifact_capped(url, max_bytes)) + } +} + +/// Outcome of one capped artifact GET, and `Some(retry-after)` iff a +/// failure is retryable. +type CappedAttempt = (Result, ApiError>, Option>); + +impl ApiClient { + /// [`ApiClient::download_artifact`] with the caller's byte cap applied + /// while streaming (a declared `Content-Length` over it is refused + /// before the body is read), retrying transient failures on the vendor + /// retry policy. + async fn download_artifact_capped( + &self, + url: &str, + max_bytes: u64, + ) -> Result, ApiError> { + if !(url.starts_with("https://") || url.starts_with("http://")) { + return Err(ApiError::Other(format!( + "refusing non-http(s) artifact URL `{url}`" + ))); + } + let attempts = self.vendor_retry.attempts.max(1); + let mut attempt = 1; + loop { + match self.download_artifact_capped_once(url, max_bytes).await { + (Err(_), Some(retry_after)) if attempt < attempts => { + self.vendor_backoff(attempt, retry_after).await; + attempt += 1; + } + (outcome, _) => return outcome, + } + } + } + + async fn download_artifact_capped_once(&self, url: &str, max_bytes: u64) -> CappedAttempt { + use crate::utils::http::{read_capped_typed, ReadCappedError}; + let sent = tokio::time::timeout( + self.vendor_retry.attempt_timeout, + self.plain + .get(url) + .header(header::ACCEPT, "application/octet-stream") + .send(), + ) + .await; + let resp = match sent { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + return ( + Err(ApiError::Network(format!( + "Network error fetching artifact: {}", + network_error_detail(&e) + ))), + Some(None), + ) + } + Err(_) => { + return ( + Err(ApiError::Network(format!( + "Network error fetching artifact: no response within {:?}", + self.vendor_retry.attempt_timeout + ))), + Some(None), + ) + } + }; + let status = resp.status(); + match status { + StatusCode::OK => {} + StatusCode::NOT_FOUND | StatusCode::GONE => { + return ( + Err(ApiError::Other(format!("artifact not found: {url}"))), + None, + ) + } + StatusCode::REQUEST_TIMEOUT => { + return ( + Err(ApiError::Other(format!("artifact still building: {url}"))), + None, + ) + } + _ => { + let hint = + vendor_status_retryable(status).then(|| retry_after_secs(resp.headers())); + let err = classify_auth_error(status, true).unwrap_or_else(|| { + ApiError::Other(format!( + "artifact download failed with status {}", + status.as_u16() + )) + }); + return (Err(err), hint); + } + } + let body = tokio::time::timeout( + self.vendor_retry.body_timeout, + read_capped_typed(resp, max_bytes, "artifact"), + ) + .await + .unwrap_or_else(|_| { + Err(ReadCappedError::Truncated(format!( + "artifact body not received within {:?}", + self.vendor_retry.body_timeout + ))) + }); + match body { + Ok(bytes) => (Ok(bytes), None), + Err(ReadCappedError::Truncated(e)) => (Err(ApiError::Network(e)), Some(None)), + Err(ReadCappedError::CapExceeded(_)) => ( + Err(ApiError::Other(format!( + "artifact exceeds the {max_bytes}-byte limit" + ))), + None, + ), + } + } +} + +/// Never prints the bearer token: an `ApiClient` can end up in a panic +/// message or an error's `{:?}`. +impl std::fmt::Debug for ApiClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ApiClient") + .field("api_url", &self.api_url) + .field("api_token", &self.api_token.as_ref().map(|_| "")) + .field("use_public_proxy", &self.use_public_proxy) + .field("org_slug", &self.org_slug) + .field("vendor_retry", &self.vendor_retry) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod patch_api_seam_tests { + use super::*; + + #[test] + fn debug_redacts_the_token() { + let client = ApiClient::new(ApiClientOptions { + api_url: "https://api.example".into(), + api_token: Some("sktsec_secret_value_api".into()), + use_public_proxy: false, + org_slug: Some("org".into()), + }); + let rendered = format!("{client:?}"); + assert!(!rendered.contains("sktsec_secret_value_api"), "{rendered}"); + assert!(rendered.contains(""), "{rendered}"); + } + + #[test] + fn trait_reports_the_proxy_flag() { + let client = ApiClient::new(ApiClientOptions { + api_url: "https://api.example".into(), + api_token: None, + use_public_proxy: true, + org_slug: None, + }); + let api: &dyn PatchApi = &client; + assert!(api.uses_public_proxy()); + } + + #[tokio::test] + async fn trait_forwards_to_the_inherent_batch_search() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v0/orgs/org/patches/batch")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "packages": [ + {"purl": "pkg:npm/b@1", "patches": []}, + {"purl": "pkg:npm/a@1", "patches": []} + ], + "canAccessPaidPatches": true + }))) + .mount(&server) + .await; + let client = ApiClient::new(ApiClientOptions { + api_url: server.uri(), + api_token: Some("t".into()), + use_public_proxy: false, + org_slug: Some("org".into()), + }); + let api: &dyn PatchApi = &client; + let purls = vec!["pkg:npm/a@1".to_string(), "pkg:npm/b@1".to_string()]; + let response = api.search_patches_batch(&purls).await.unwrap(); + assert!(response.can_access_paid_patches); + let order: Vec<&str> = response.packages.iter().map(|p| p.purl.as_str()).collect(); + assert_eq!(order, vec!["pkg:npm/a@1", "pkg:npm/b@1"]); + } + + #[tokio::test] + async fn trait_download_enforces_the_callers_byte_cap() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/a.whl")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(vec![7u8; 64])) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/gone.whl")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + let client = ApiClient::new(ApiClientOptions { + api_url: server.uri(), + api_token: Some("t".into()), + use_public_proxy: false, + org_slug: Some("org".into()), + }); + let api: &dyn PatchApi = &client; + let url = format!("{}/a.whl", server.uri()); + assert_eq!( + api.download_artifact(&url, 64).await.unwrap(), + vec![7u8; 64] + ); + let err = api.download_artifact(&url, 63).await.unwrap_err(); + assert!(err.to_string().contains("63-byte limit"), "{err}"); + let gone = format!("{}/gone.whl", server.uri()); + let err = api.download_artifact(&gone, 64).await.unwrap_err(); + assert!(err.to_string().contains("not found"), "{err}"); + assert!(api + .download_artifact("file:///etc/passwd", 64) + .await + .is_err()); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/socket-patch-core/src/utils/cargo_workspace.rs b/crates/socket-patch-core/src/utils/cargo_workspace.rs index 5f30771ae..aabd2a697 100644 --- a/crates/socket-patch-core/src/utils/cargo_workspace.rs +++ b/crates/socket-patch-core/src/utils/cargo_workspace.rs @@ -12,10 +12,12 @@ //! Cargo.lock dependents check refuses a crate one of them depends on. use std::collections::BTreeSet; -use std::path::{Component, Path, PathBuf}; +use std::path::{Component, Path}; use toml_edit::{DocumentMut, Item, Table}; +use crate::vendor::lock_inventory::view::{MemoryProject, ProjectView}; + /// Upper bound on discovered manifests — a runaway glob (or a hostile tree) /// must not turn one scan into an unbounded walk. const MAX_MANIFESTS: usize = 4096; @@ -24,7 +26,87 @@ const MAX_MANIFESTS: usize = 4096; /// in-root path dependencies of the project at `root`, sorted. Empty when /// `root/Cargo.toml` is absent or unparseable. pub fn member_manifests(root: &Path) -> Vec { - let Some(doc) = read_manifest(&root.join("Cargo.toml")) else { + member_manifests_with(&DiskTree(root)) +} + +/// [`member_manifests`] over a [`ProjectView`]: the in-memory variant +/// reads the manifests the host supplied and expands member globs against +/// the supplied paths (a directory exists when some supplied path lives +/// under it; symbolic links are never directories). +pub fn member_manifests_in(view: &ProjectView<'_>) -> Vec { + match view { + ProjectView::Disk(root) => member_manifests(root), + ProjectView::Memory(project) => member_manifests_with(&MemoryTree(project)), + } +} + +/// The three filesystem questions the member walk asks, keyed by +/// `/`-separated root-relative paths (`""` is the root). +trait Tree { + /// A regular (non-symlink) manifest file, parsed. + fn read_manifest(&self, rel: &str) -> Option; + /// A real directory (not a symbolic link). + fn is_real_dir(&self, rel: &str) -> bool; + /// The real sub-directory names of `rel`, or `None` when unreadable. + fn child_dirs(&self, rel: &str) -> Option>; +} + +struct DiskTree<'a>(&'a Path); + +impl Tree for DiskTree<'_> { + fn read_manifest(&self, rel: &str) -> Option { + read_manifest(&self.0.join(rel)) + } + + fn is_real_dir(&self, rel: &str) -> bool { + is_real_dir(&self.0.join(rel)) + } + + fn child_dirs(&self, rel: &str) -> Option> { + let entries = std::fs::read_dir(self.0.join(rel)).ok()?; + Some( + entries + .filter_map(Result::ok) + .filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) + .filter_map(|e| e.file_name().to_str().map(str::to_string)) + .collect(), + ) + } +} + +struct MemoryTree<'a>(&'a MemoryProject); + +impl Tree for MemoryTree<'_> { + fn read_manifest(&self, rel: &str) -> Option { + self.0.text(rel)?.parse().ok() + } + + fn is_real_dir(&self, rel: &str) -> bool { + self.0.is_dir(rel) + } + + fn child_dirs(&self, rel: &str) -> Option> { + self.0.is_dir(rel).then(|| { + self.0 + .children(rel) + .into_iter() + .filter(|(_, is_dir)| *is_dir) + .map(|(name, _)| name) + .collect() + }) + } +} + +fn join_rel(base: &str, seg: &str) -> String { + if base.is_empty() { + seg.to_string() + } else { + format!("{base}/{seg}") + } +} + +fn member_manifests_with(tree: &dyn Tree) -> Vec { + let Some(doc) = tree.read_manifest("Cargo.toml") else { return Vec::new(); }; let mut dirs: BTreeSet = BTreeSet::new(); @@ -43,25 +125,25 @@ pub fn member_manifests(root: &Path) -> Vec { }; let excluded: BTreeSet = patterns("exclude") .iter() - .flat_map(|p| expand_glob(root, p)) + .flat_map(|p| expand_glob(tree, p)) .collect(); for pattern in patterns("members") { - for dir in expand_glob(root, &pattern) { + for dir in expand_glob(tree, &pattern) { if !excluded.contains(&dir) { - enqueue(root, dir, &mut dirs, &mut queue); + enqueue(tree, dir, &mut dirs, &mut queue); } } } } for dep_dir in path_dependencies(&doc) { if let Some(dir) = normalize_rel("", &dep_dir) { - enqueue(root, dir, &mut dirs, &mut queue); + enqueue(tree, dir, &mut dirs, &mut queue); } } while let Some((dir, doc)) = queue.pop() { for dep_dir in path_dependencies(&doc) { if let Some(dep) = normalize_rel(&dir, &dep_dir) { - enqueue(root, dep, &mut dirs, &mut queue); + enqueue(tree, dep, &mut dirs, &mut queue); } } } @@ -83,17 +165,17 @@ fn is_real_dir(path: &Path) -> bool { } /// Every component of repo-relative `dir` is a real directory under -/// `root` — none is a symbolic link (which may lead outside the root). -fn is_real_dir_path(root: &Path, dir: &str) -> bool { - let mut at = root.to_path_buf(); +/// the root — none is a symbolic link (which may lead outside the root). +fn is_real_dir_path(tree: &dyn Tree, dir: &str) -> bool { + let mut at = String::new(); dir.split('/').all(|seg| { - at.push(seg); - is_real_dir(&at) + at = join_rel(&at, seg); + tree.is_real_dir(&at) }) } fn enqueue( - root: &Path, + tree: &dyn Tree, dir: String, dirs: &mut BTreeSet, queue: &mut Vec<(String, DocumentMut)>, @@ -107,11 +189,11 @@ fn enqueue( || dirs.len() >= MAX_MANIFESTS || dirs.contains(&dir) || dir.split('/').any(|seg| seg == "target") - || !is_real_dir_path(root, &dir) + || !is_real_dir_path(tree, &dir) { return; } - let Some(doc) = read_manifest(&root.join(&dir).join("Cargo.toml")) else { + let Some(doc) = tree.read_manifest(&format!("{dir}/Cargo.toml")) else { return; }; dirs.insert(dir.clone()); @@ -191,53 +273,51 @@ pub(crate) fn normalize_rel(base: &str, rel: &str) -> Option { /// Expand a cargo `members` / `exclude` glob (`*`, `?`, `**`) to the /// repo-relative directories it names. -fn expand_glob(root: &Path, pattern: &str) -> Vec { +fn expand_glob(tree: &dyn Tree, pattern: &str) -> Vec { let Some(normalized) = normalize_rel("", pattern.trim_end_matches('/')) else { return Vec::new(); }; let segments: Vec<&str> = normalized.split('/').filter(|s| !s.is_empty()).collect(); let mut out = Vec::new(); - expand_from(root, PathBuf::new(), &segments, &mut out); + expand_from(tree, String::new(), &segments, &mut out); out.sort(); out.dedup(); out } -fn expand_from(root: &Path, at: PathBuf, rest: &[&str], out: &mut Vec) { +fn expand_from(tree: &dyn Tree, at: String, rest: &[&str], out: &mut Vec) { if out.len() >= MAX_MANIFESTS { return; } let Some((seg, tail)) = rest.split_first() else { - out.push(at.to_string_lossy().replace('\\', "/")); + out.push(at); return; }; if !seg.contains(['*', '?']) { - let next = at.join(seg); - if is_real_dir(&root.join(&next)) { - expand_from(root, next, tail, out); + let next = join_rel(&at, seg); + if tree.is_real_dir(&next) { + expand_from(tree, next, tail, out); } return; } - let Ok(entries) = std::fs::read_dir(root.join(&at)) else { + let Some(entries) = tree.child_dirs(&at) else { return; }; let mut children: Vec = entries - .filter_map(Result::ok) - .filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) - .filter_map(|e| e.file_name().to_str().map(str::to_string)) + .into_iter() .filter(|name| !name.starts_with('.') && name != "target") .collect(); children.sort(); if *seg == "**" { - expand_from(root, at.clone(), tail, out); + expand_from(tree, at.clone(), tail, out); for child in children { - expand_from(root, at.join(child), rest, out); + expand_from(tree, join_rel(&at, &child), rest, out); } return; } for child in children { if wildcard_match(seg.as_bytes(), child.as_bytes()) { - expand_from(root, at.join(child), tail, out); + expand_from(tree, join_rel(&at, &child), tail, out); } } } diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/bun.rs b/crates/socket-patch-core/src/vendor/lock_inventory/bun.rs index 85153459d..bbb20df10 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/bun.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/bun.rs @@ -3,10 +3,10 @@ use std::path::Path; use crate::constants::npm_family::{BUN_LOCK, BUN_LOCKB}; -use crate::utils::fs::{read_regular_to_bytes, read_regular_to_string}; use crate::vendor::bun_lock_text::{self, BunEntry}; use crate::vendor::bun_lockb::BunLockb; +use super::view::ProjectView; use super::{http_url, LockIntegrity, LockfileEntry, UnsupportedNpmLayout}; /// Every `packages` entry of a text `bun.lock`, read with the ONE @@ -30,21 +30,25 @@ pub(crate) fn bun_text_entries(text: &str) -> Result, String> { /// `bun.lockb` beside it is not the live lock. Lockfile discovery answers /// the same question with `DiscoverCtx::exists` (the same lstat). pub(crate) async fn bun_text_lock_present(root: &Path) -> bool { - tokio::fs::symlink_metadata(root.join(BUN_LOCK)) - .await - .is_ok() + bun_text_lock_present_in(&ProjectView::Disk(root)).await +} + +/// [`bun_text_lock_present`] over a [`ProjectView`]. +pub(crate) async fn bun_text_lock_present_in(view: &ProjectView<'_>) -> bool { + view.exists_no_follow(BUN_LOCK).await } // ── registry view ── -pub(super) async fn inventory_bun_binary( - root: &Path, +pub(super) async fn inventory_bun_binary_in( + view: &ProjectView<'_>, ) -> Result, UnsupportedNpmLayout> { let invalid = |detail: String| UnsupportedNpmLayout { code: "bun_lockb_invalid", detail: format!("cannot inventory bun.lockb: {detail}"), }; - let bytes = read_regular_to_bytes(&root.join(BUN_LOCKB)) + let bytes = view + .read_bytes(BUN_LOCKB) .await .map_err(|error| invalid(error.to_string()))?; let packages = BunLockb::parse_packages(&bytes).map_err(invalid)?; @@ -71,8 +75,13 @@ pub(super) async fn inventory_bun_binary( .collect()) } +#[cfg(test)] pub(super) async fn inventory_bun(root: &Path) -> Option> { - let text = read_regular_to_string(&root.join(BUN_LOCK)).await.ok()?; + inventory_bun_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_bun_in(view: &ProjectView<'_>) -> Option> { + let text = view.read_text(BUN_LOCK).await.ok()?; let entries = bun_text_entries(&text).ok()?; let mut out = Vec::new(); diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs b/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs index 8cc89d110..ca97eb22c 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/cargo.rs @@ -1,10 +1,12 @@ //! `Cargo.lock`: the registry view. +#[cfg(test)] use std::path::Path; use crate::utils::digest::is_hex; use crate::utils::purl::simple_purl; +use super::view::ProjectView; use super::{dedup_prefer_integrity, LockIntegrity, LockfileEntry, SourceKind}; // ── registry view ── @@ -22,18 +24,32 @@ use super::{dedup_prefer_integrity, LockIntegrity, LockfileEntry, SourceKind}; /// verifier (its checksum pins a tagged version no registry serves under /// the purl's version). A lock that is not TOML yields nothing — cargo /// itself refuses to build from it. +#[cfg(test)] pub(super) async fn inventory_cargo_lock(project_root: &Path) -> Option> { - inventory_cargo_lock_raw(project_root) + inventory_cargo_lock_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_cargo_lock`] over a [`ProjectView`]. +pub(super) async fn inventory_cargo_lock_in(view: &ProjectView<'_>) -> Option> { + inventory_cargo_lock_raw_in(view) .await .map(dedup_prefer_integrity) } /// [`inventory_cargo_lock`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_cargo_lock_raw(project_root: &Path) -> Option> { - let (_, doc, _) = crate::vendor::cargo_lock::read_lock(project_root) - .await - .ok()?; +pub(super) async fn inventory_cargo_lock_raw_in( + view: &ProjectView<'_>, +) -> Option> { + let doc: toml_edit::DocumentMut = match view { + ProjectView::Disk(project_root) => { + crate::vendor::cargo_lock::read_lock(project_root) + .await + .ok()? + .1 + } + ProjectView::Memory(_) => view.read_text("Cargo.lock").await.ok()?.parse().ok()?, + }; let mut out = Vec::new(); for pkg in crate::vendor::cargo_lock::locked_packages(&doc) { let Some(source) = pkg.source else { diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/composer.rs b/crates/socket-patch-core/src/vendor/lock_inventory/composer.rs index 5223de1eb..25227762d 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/composer.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/composer.rs @@ -1,6 +1,7 @@ //! `composer.lock`: the shared entry walk ([`composer_lock_packages`]) and //! its registry view. +#[cfg(test)] use std::path::Path; use serde_json::Value; @@ -8,9 +9,9 @@ use serde_json::Value; use crate::crawlers::composer_crawler::normalize_version; use crate::patch::path_safety; use crate::utils::digest::sha1_hex; -use crate::utils::fs::read_regular_to_bytes; use crate::vendor::path::{parse_vendor_path, VendorPathParts}; +use super::view::ProjectView; use super::{dedup_prefer_integrity, http_url, LockIntegrity, LockfileEntry, SourceKind}; // ── entry model ── @@ -87,18 +88,26 @@ pub(crate) fn composer_lock_packages(doc: &Value) -> Vec /// discovery-only. Names lowercase to the canonical packagist form; /// versions drop the pretty leading `v`/`V` through the crawler's /// [`normalize_version`], so installed and lockfile rows agree. +#[cfg(test)] pub(super) async fn inventory_composer_lock(project_root: &Path) -> Option> { - inventory_composer_lock_raw(project_root) + inventory_composer_lock_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_composer_lock`] over a [`ProjectView`]. +pub(super) async fn inventory_composer_lock_in( + view: &ProjectView<'_>, +) -> Option> { + inventory_composer_lock_raw_in(view) .await .map(dedup_prefer_integrity) } /// [`inventory_composer_lock`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_composer_lock_raw(project_root: &Path) -> Option> { - let bytes = read_regular_to_bytes(&project_root.join("composer.lock")) - .await - .ok()?; +pub(super) async fn inventory_composer_lock_raw_in( + view: &ProjectView<'_>, +) -> Option> { + let bytes = view.read_bytes("composer.lock").await.ok()?; let doc: Value = serde_json::from_slice(&bytes).ok()?; let mut out = Vec::new(); for pkg in composer_lock_packages(&doc) { diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/gem.rs b/crates/socket-patch-core/src/vendor/lock_inventory/gem.rs index 885a0c690..ee6fb95c3 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/gem.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/gem.rs @@ -7,6 +7,7 @@ use crate::utils::fs::read_regular_to_string; use crate::utils::purl::simple_purl; use crate::vendor::gemfile_lock::{self, Section}; +use super::view::ProjectView; use super::{dedup_prefer_integrity, http_url, LockIntegrity, LockfileEntry, SourceKind}; // ── registry view ── @@ -26,18 +27,26 @@ use super::{dedup_prefer_integrity, http_url, LockIntegrity, LockfileEntry, Sour /// distinct `remote:` lines is a legacy bundler 1.x multisource lock whose /// per-spec origin is genuinely ambiguous: its specs stay discovery-only /// (no resolved URL — the fetch layer then refuses), fail-closed. +#[cfg(test)] pub(super) async fn inventory_gemfile_lock(project_root: &Path) -> Option> { - inventory_gemfile_lock_raw(project_root) + inventory_gemfile_lock_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_gemfile_lock`] over a [`ProjectView`]. +pub(super) async fn inventory_gemfile_lock_in( + view: &ProjectView<'_>, +) -> Option> { + inventory_gemfile_lock_raw_in(view) .await .map(dedup_prefer_integrity) } /// [`inventory_gemfile_lock`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_gemfile_lock_raw(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("Gemfile.lock")) - .await - .ok()?; +pub(super) async fn inventory_gemfile_lock_raw_in( + view: &ProjectView<'_>, +) -> Option> { + let text = view.read_text("Gemfile.lock").await.ok()?; // The shared lock model (lockfile discovery reads it too); what bundler // would refuse (`problems`) still inventories whatever parsed — this is // read-only discovery. diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/golang.rs b/crates/socket-patch-core/src/vendor/lock_inventory/golang.rs index 3ef93542b..25636f85d 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/golang.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/golang.rs @@ -1,11 +1,12 @@ //! `go.sum`: the registry view. +#[cfg(test)] use std::path::Path; -use crate::utils::fs::read_regular_to_string; use crate::utils::purl::golang_purl; use crate::vendor::go_sum_edit::go_sum_lines; +use super::view::ProjectView; use super::{dedup_prefer_integrity, LockIntegrity, LockfileEntry, SourceKind}; // ── registry view ── @@ -14,18 +15,22 @@ use super::{dedup_prefer_integrity, LockIntegrity, LockfileEntry, SourceKind}; /// `/go.mod`-suffixed lines hash only the manifest and are skipped. go.sum /// may list more modules than the final build graph — acceptable for /// discovery, and the manifest decides what actually gets vendored. +#[cfg(test)] pub(super) async fn inventory_go_sum(project_root: &Path) -> Option> { - inventory_go_sum_raw(project_root) + inventory_go_sum_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_go_sum`] over a [`ProjectView`]. +pub(super) async fn inventory_go_sum_in(view: &ProjectView<'_>) -> Option> { + inventory_go_sum_raw_in(view) .await .map(dedup_prefer_integrity) } /// [`inventory_go_sum`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_go_sum_raw(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("go.sum")) - .await - .ok()?; +pub(super) async fn inventory_go_sum_raw_in(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("go.sum").await.ok()?; let mut out = Vec::new(); for line in go_sum_lines(&text) { if line.go_mod || !line.hash.starts_with("h1:") { diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs b/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs index 774ad3778..11279bf8a 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/mod.rs @@ -65,23 +65,31 @@ pub(crate) mod npm_family; pub(crate) mod pnpm; pub(crate) mod pypi; pub(crate) mod recover; +pub mod view; pub(crate) mod wired; pub(crate) mod yarn; pub(crate) use self::composer::{composer_lock_packages, ComposerLockPackage}; pub(crate) use self::npm::{npm_lock_nodes, NpmLockNode}; +#[cfg(test)] pub(crate) use self::npm_family::inventory_npm_lock; pub(crate) use self::pnpm::pnpm_registry_key; pub(crate) use self::pypi::pipfile_lock_entries; pub use self::recover::recover_lock_entry; +pub use self::view::{MemoryEntry, MemoryProject, ProjectView}; pub use self::wired::wired_vendor_integrity; // The per-format views `inventory_project_diagnosed` unions (and the test // modules reach through `super::*`). +#[cfg(test)] use self::cargo::inventory_cargo_lock; +#[cfg(test)] use self::composer::inventory_composer_lock; +#[cfg(test)] use self::gem::inventory_gemfile_lock; +#[cfg(test)] use self::golang::inventory_go_sum; +#[cfg(test)] use self::pypi::inventory_pypi_locks; #[cfg(test)] use self::{ @@ -260,18 +268,33 @@ enum Instances { Every, } +/// [`inventory_project_diagnosed`] over a [`ProjectView`]: the same views, +/// precedence and guards, reading from disk or from an in-memory project. +pub async fn inventory_project_diagnosed_in( + view: &ProjectView<'_>, +) -> (Vec, Vec) { + union_views_in(view, Instances::Collapsed).await +} + /// The union of the per-format views, in the one precedence order. async fn union_views( project_root: &Path, instances: Instances, +) -> (Vec, Vec) { + union_views_in(&ProjectView::Disk(project_root), instances).await +} + +async fn union_views_in( + view: &ProjectView<'_>, + instances: Instances, ) -> (Vec, Vec) { let every = instances == Instances::Every; let mut out: Vec = Vec::new(); let mut unsupported: Vec = Vec::new(); let npm = if every { - npm_family::inventory_npm_lock_raw(project_root).await + npm_family::inventory_npm_lock_raw_in(view).await } else { - inventory_npm_lock(project_root).await + npm_family::inventory_npm_lock_in(view).await }; match npm { Ok(Some((_, entries))) => out.extend(entries), @@ -280,29 +303,29 @@ async fn union_views( } let views = [ if every { - cargo::inventory_cargo_lock_raw(project_root).await + cargo::inventory_cargo_lock_raw_in(view).await } else { - inventory_cargo_lock(project_root).await + cargo::inventory_cargo_lock_in(view).await }, if every { - golang::inventory_go_sum_raw(project_root).await + golang::inventory_go_sum_raw_in(view).await } else { - inventory_go_sum(project_root).await + golang::inventory_go_sum_in(view).await }, if every { - composer::inventory_composer_lock_raw(project_root).await + composer::inventory_composer_lock_raw_in(view).await } else { - inventory_composer_lock(project_root).await + composer::inventory_composer_lock_in(view).await }, if every { - gem::inventory_gemfile_lock_raw(project_root).await + gem::inventory_gemfile_lock_raw_in(view).await } else { - inventory_gemfile_lock(project_root).await + gem::inventory_gemfile_lock_in(view).await }, if every { - pypi::inventory_pypi_locks_raw(project_root).await + pypi::inventory_pypi_locks_raw_in(view).await } else { - inventory_pypi_locks(project_root).await + pypi::inventory_pypi_locks_in(view).await }, ]; out.extend(views.into_iter().flatten().flatten()); diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/npm.rs b/crates/socket-patch-core/src/vendor/lock_inventory/npm.rs index 2d892c9c6..0353ec3ca 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/npm.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/npm.rs @@ -1,15 +1,16 @@ //! `package-lock.json` / `npm-shrinkwrap.json`: the shared entry walk //! ([`npm_lock_nodes`]) and its registry view. +#[cfg(test)] use std::path::Path; use serde_json::Value; use crate::constants::npm_family::NPM_LOCKS; use crate::utils::digest::is_sri_pin; -use crate::utils::fs::read_regular_to_bytes; use crate::vendor::path::parse_vendor_path; +use super::view::ProjectView; use super::{http_url, LockIntegrity, LockfileEntry}; // ── entry model ── @@ -105,11 +106,18 @@ fn walk_npm_legacy_dependencies<'a>( // ── registry view ── +#[cfg(test)] pub(super) async fn inventory_package_lock(root: &Path) -> Option> { + inventory_package_lock_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_package_lock_in( + view: &ProjectView<'_>, +) -> Option> { // Shrinkwrap wins, mirroring `npm_lock::select_lockfile`. let mut bytes = None; for lock in NPM_LOCKS { - if let Ok(b) = read_regular_to_bytes(&root.join(lock)).await { + if let Ok(b) = view.read_bytes(lock).await { bytes = Some(b); break; } diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/npm_family.rs b/crates/socket-patch-core/src/vendor/lock_inventory/npm_family.rs index a82a4dda9..d111a6ba8 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/npm_family.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/npm_family.rs @@ -2,16 +2,20 @@ //! installs from ([`inventory_npm_lock`]), the migration-leftover sibling //! probe, and the shared name/version guard + dedup of npm entries. +#[cfg(test)] use std::path::Path; use crate::constants::npm_family::{BUN_LOCK, BUN_LOCKB, NPM_LOCKS, PNPM_SHRINKWRAP_LEGACY}; use crate::utils::purl::npm_purl; -use crate::vendor::npm_flavor::{detect_npm_lock_flavor, NpmLockFlavor}; +use crate::vendor::npm_flavor::NpmLockFlavor; -use super::bun::{bun_text_lock_present, inventory_bun, inventory_bun_binary}; -use super::npm::inventory_package_lock; -use super::pnpm::{inventory_pnpm_lock, inventory_pnpm_lock_at, inventory_rush_pnpm_locks}; -use super::yarn::{inventory_yarn_berry, inventory_yarn_classic}; +use super::bun::{bun_text_lock_present_in, inventory_bun_binary_in, inventory_bun_in}; +use super::npm::inventory_package_lock_in; +use super::pnpm::{ + inventory_pnpm_lock_in, inventory_pnpm_lock_rel_in, inventory_rush_pnpm_locks_in, +}; +use super::view::{detect_npm_lock_flavor_in, ProjectView}; +use super::yarn::{inventory_yarn_berry_in, inventory_yarn_classic_in}; use super::{dedup_prefer_integrity, LockfileEntry, UnsupportedNpmLayout}; // ── registry view ── @@ -27,24 +31,32 @@ use super::{dedup_prefer_integrity, LockfileEntry, UnsupportedNpmLayout}; /// `pnpm-lock.yaml` directly — unless a live sibling lock the router would /// otherwise have chosen sits beside it (a pnpm→yarn/npm migration /// leftover), in which case the SIBLING is inventoried instead -/// ([`inventory_live_sibling_lock`]) — and `vendor_lockfile_missing` reads +/// ([`inventory_live_sibling_lock_in`]) — and `vendor_lockfile_missing` reads /// the pnpm <=2-era `shrinkwrap.yaml` (same v5 grammar, older filename). /// Any remaining probe failure falls back to Rush's common lock when /// `rush.json` is present. +#[cfg(test)] pub(crate) async fn inventory_npm_lock( project_root: &Path, ) -> Result)>, UnsupportedNpmLayout> { - inventory_npm_lock_raw(project_root) + inventory_npm_lock_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_npm_lock`] over a [`ProjectView`]. +pub(super) async fn inventory_npm_lock_in( + view: &ProjectView<'_>, +) -> Result)>, UnsupportedNpmLayout> { + inventory_npm_lock_raw_in(view) .await .map(|found| found.map(|(flavor, raw)| (flavor, dedup_prefer_integrity(raw)))) } /// [`inventory_npm_lock`] before its collapse: every /// guarded instance ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_npm_lock_raw( - project_root: &Path, +pub(super) async fn inventory_npm_lock_raw_in( + view: &ProjectView<'_>, ) -> Result)>, UnsupportedNpmLayout> { - let (flavor, _warnings) = match detect_npm_lock_flavor(project_root).await { + let (flavor, _warnings) = match detect_npm_lock_flavor_in(view).await { Ok(found) => found, Err((code, detail)) => { // The PnP loaders are a refusal, not an absence: propagate the @@ -81,7 +93,7 @@ pub(super) async fn inventory_npm_lock_raw( // have chosen had the pnpm lock not shadowed it; only a // sibling-less project is a genuine old-pnpm project whose // lock the fallback may surface. - match inventory_live_sibling_lock(project_root).await { + match inventory_live_sibling_lock_in(view).await { Some((flavor, entries)) if !entries.is_empty() => { return Ok(Some((flavor, guard_npm(entries)))); } @@ -91,7 +103,7 @@ pub(super) async fn inventory_npm_lock_raw( // blind beats presenting dead resolutions as live. Some(_) => {} None => { - let pnpm = inventory_pnpm_lock(project_root).await.unwrap_or_default(); + let pnpm = inventory_pnpm_lock_in(view).await.unwrap_or_default(); if !pnpm.is_empty() { return Ok(Some((NpmLockFlavor::Pnpm, guard_npm(pnpm)))); } @@ -111,7 +123,7 @@ pub(super) async fn inventory_npm_lock_raw( // long-ago migration whose dead resolutions must not pose as // the live dependency set. if code == "vendor_lockfile_missing" { - let legacy = inventory_pnpm_lock_at(&project_root.join(PNPM_SHRINKWRAP_LEGACY)) + let legacy = inventory_pnpm_lock_rel_in(view, PNPM_SHRINKWRAP_LEGACY) .await .unwrap_or_default(); if !legacy.is_empty() { @@ -123,23 +135,23 @@ pub(super) async fn inventory_npm_lock_raw( // The flavor probe (root-relative) can't see it, so fall back // explicitly when the root lock is absent but rush.json is // present. - let rush = inventory_rush_pnpm_locks(project_root).await; + let rush = inventory_rush_pnpm_locks_in(view).await; return Ok((!rush.is_empty()).then(|| (NpmLockFlavor::Pnpm, guard_npm(rush)))); } }; let raw = match flavor { - NpmLockFlavor::PackageLock => inventory_package_lock(project_root).await, + NpmLockFlavor::PackageLock => inventory_package_lock_in(view).await, // The pnpm reader is grammar-agnostic (it already served legacy // 5.4/6.0 locks through the refusal fallback below before those // grammars had a wiring backend), so both pnpm flavors share it. - NpmLockFlavor::Pnpm | NpmLockFlavor::PnpmLegacy => inventory_pnpm_lock(project_root).await, - NpmLockFlavor::YarnClassic => inventory_yarn_classic(project_root).await, - NpmLockFlavor::YarnBerry => inventory_yarn_berry(project_root).await, + NpmLockFlavor::Pnpm | NpmLockFlavor::PnpmLegacy => inventory_pnpm_lock_in(view).await, + NpmLockFlavor::YarnClassic => inventory_yarn_classic_in(view).await, + NpmLockFlavor::YarnBerry => inventory_yarn_berry_in(view).await, NpmLockFlavor::Bun => { - if bun_text_lock_present(project_root).await { - inventory_bun(project_root).await + if bun_text_lock_present_in(view).await { + inventory_bun_in(view).await } else { - Some(inventory_bun_binary(project_root).await?) + Some(inventory_bun_binary_in(view).await?) } } }; @@ -155,27 +167,23 @@ pub(super) async fn inventory_npm_lock_raw( /// sibling's inventory (possibly empty: presence alone proves the pnpm lock /// is migration debris, so the caller must not fall back to it). Raw /// entries — the caller guards and collapses them. -pub(super) async fn inventory_live_sibling_lock( - root: &Path, +pub(super) async fn inventory_live_sibling_lock_in( + view: &ProjectView<'_>, ) -> Option<(NpmLockFlavor, Vec)> { - let exists = |name: &str| { - let p = root.join(name); - async move { tokio::fs::metadata(&p).await.is_ok() } - }; // bun.lock — router step 2. That step runs BEFORE the pnpm sniff, so // when the version refusal fired no bun.lock can actually be present; // probed anyway to keep this a literal transcription of the router's // order. The binary lock shares the same routing precedence. - if exists(BUN_LOCK).await { + if view.exists(BUN_LOCK).await { return Some(( NpmLockFlavor::Bun, - inventory_bun(root).await.unwrap_or_default(), + inventory_bun_in(view).await.unwrap_or_default(), )); } - if exists(BUN_LOCKB).await { + if view.exists(BUN_LOCKB).await { return Some(( NpmLockFlavor::Bun, - inventory_bun_binary(root).await.unwrap_or_default(), + inventory_bun_binary_in(view).await.unwrap_or_default(), )); } // yarn.lock — router step 4, where classic vs berry is a content @@ -184,22 +192,22 @@ pub(super) async fn inventory_live_sibling_lock( // fields vs berry's `resolution:` lines), so a non-empty result is the // sniff's answer. Berry PnP needs no carve-out: a PnP marker would have // refused at the router's step 1 with a code this fallback ignores. - if exists("yarn.lock").await { - let classic = inventory_yarn_classic(root).await.unwrap_or_default(); + if view.exists("yarn.lock").await { + let classic = inventory_yarn_classic_in(view).await.unwrap_or_default(); if !classic.is_empty() { return Some((NpmLockFlavor::YarnClassic, classic)); } return Some(( NpmLockFlavor::YarnBerry, - inventory_yarn_berry(root).await.unwrap_or_default(), + inventory_yarn_berry_in(view).await.unwrap_or_default(), )); } // npm — router step 5 (`inventory_package_lock` itself prefers the // shrinkwrap when both exist, mirroring npm). - if exists(NPM_LOCKS[0]).await || exists(NPM_LOCKS[1]).await { + if view.exists(NPM_LOCKS[0]).await || view.exists(NPM_LOCKS[1]).await { return Some(( NpmLockFlavor::PackageLock, - inventory_package_lock(root).await.unwrap_or_default(), + inventory_package_lock_in(view).await.unwrap_or_default(), )); } None diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs b/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs index 383adb5d8..9d78c288a 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/pnpm.rs @@ -11,6 +11,7 @@ use crate::patch::redirect::pnpm; use crate::utils::fs::read_regular_to_string; use crate::vendor::path::parse_vendor_path; +use super::view::ProjectView; use super::{http_url, LockIntegrity, LockfileEntry}; // ── entry model ── @@ -190,8 +191,22 @@ pub(crate) async fn rush_lock_rels(root: &Path) -> Vec { // ── registry view ── +#[cfg(test)] pub(super) async fn inventory_pnpm_lock(root: &Path) -> Option> { - inventory_pnpm_lock_at(&root.join(PNPM_LOCK)).await + inventory_pnpm_lock_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_pnpm_lock_in(view: &ProjectView<'_>) -> Option> { + inventory_pnpm_lock_rel_in(view, PNPM_LOCK).await +} + +/// [`inventory_pnpm_lock_at`] for a project-relative lock path. +pub(super) async fn inventory_pnpm_lock_rel_in( + view: &ProjectView<'_>, + rel: &str, +) -> Option> { + let text = view.read_text(rel).await.ok()?; + pnpm_lock_text_inventory(&text) } /// Inventory a specific `pnpm-lock.yaml` (path given explicitly so the Rush @@ -203,6 +218,10 @@ pub(super) async fn inventory_pnpm_lock(root: &Path) -> Option Option> { let text = read_regular_to_string(lock_path).await.ok()?; + pnpm_lock_text_inventory(&text) +} + +fn pnpm_lock_text_inventory(text: &str) -> Option> { if !text .lines() .any(|l| l.trim_end_matches('\r') == "packages:") @@ -210,7 +229,7 @@ pub(super) async fn inventory_pnpm_lock_at(lock_path: &Path) -> Option Option Vec { +pub(super) async fn inventory_rush_pnpm_locks_in(view: &ProjectView<'_>) -> Vec { + let project = match view { + ProjectView::Disk(project_root) => return inventory_rush_pnpm_locks(project_root).await, + ProjectView::Memory(project) => *project, + }; + if !project.contains("rush.json") { + return Vec::new(); + } + let mut out = Vec::new(); + if let Some(entries) = inventory_pnpm_lock_rel_in(view, RUSH_COMMON_LOCK_REL).await { + out.extend(entries); + } + for (name, is_dir) in project.children(RUSH_SUBSPACES_DIR) { + if !is_dir { + continue; + } + let rel = format!("{RUSH_SUBSPACES_DIR}/{name}/{PNPM_LOCK}"); + if let Some(entries) = inventory_pnpm_lock_rel_in(view, &rel).await { + out.extend(entries); + } + } + out +} + +async fn inventory_rush_pnpm_locks(project_root: &Path) -> Vec { if tokio::fs::metadata(project_root.join("rush.json")) .await .is_err() diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs b/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs index 8bbd452ad..b30c4484b 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs @@ -2,19 +2,20 @@ //! requirements): the registry views, with the Pipfile.lock entry walk //! lockfile discovery shares ([`pipfile_lock_entries`]). +#[cfg(test)] use std::path::Path; use serde_json::Value; use toml_edit::{DocumentMut, Item, TableLike}; use crate::crawlers::python_crawler::canonicalize_pypi_name; -use crate::utils::fs::read_regular_to_string; use crate::utils::purl::{percent_decode_purl_component, pypi_purl}; use crate::utils::python_lock::{lock_package_collection, package_artifacts, UvSource}; use crate::utils::requirements::archive_filename_coords; use crate::utils::digest::{sha256_hex, sha256_prefixed}; +use super::view::ProjectView; use super::{dedup_prefer_integrity, http_url, LockIntegrity, LockfileEntry, SourceKind}; // pypi purls and lock entries compare in PEP 503 normalized form @@ -171,21 +172,45 @@ pub(crate) fn hosted_artifact_url(url: &str) -> Result Option> { - inventory_pypi_locks_raw(project_root) + inventory_pypi_locks_in(&ProjectView::Disk(project_root)).await +} + +/// [`inventory_pypi_locks`] over a [`ProjectView`]. +pub(super) async fn inventory_pypi_locks_in(view: &ProjectView<'_>) -> Option> { + inventory_pypi_locks_raw_in(view) .await .map(dedup_prefer_integrity) } +/// The project-root Python lock names ([`crate::utils::python_lock::python_lock_paths`] +/// on disk; the in-memory project's root-level names otherwise), sorted. +pub(crate) fn python_lock_paths_in(view: &ProjectView<'_>) -> std::io::Result> { + match view { + ProjectView::Disk(root) => crate::utils::python_lock::python_lock_paths(root), + ProjectView::Memory(project) => Ok(project + .children("") + .into_iter() + .filter(|(name, is_dir)| { + !is_dir && crate::utils::python_lock::is_python_lock_name(name) + }) + .map(|(name, _)| name) + .collect()), + } +} + /// [`inventory_pypi_locks`] before its collapse: every instance /// ([`super::inventory_project_every_lock`]). -pub(super) async fn inventory_pypi_locks_raw(project_root: &Path) -> Option> { +pub(super) async fn inventory_pypi_locks_raw_in( + view: &ProjectView<'_>, +) -> Option> { let mut out = Vec::new(); let mut found = false; let mut uv_lock = false; - if let Ok(paths) = crate::utils::python_lock::python_lock_paths(project_root) { + if let Ok(paths) = python_lock_paths_in(view) { for path in paths { - let Ok(text) = read_regular_to_string(&project_root.join(&path)).await else { + let Ok(text) = view.read_text(&path).await else { continue; }; if let Some(entries) = python_lock_inventory(&text) { @@ -209,10 +234,10 @@ pub(super) async fn inventory_pypi_locks_raw(project_root: &Path) -> Option Option Option> { /// API; it stays discovery-only otherwise (Poetry 0.12's /// `[metadata.hashes]` lists bare digests without filenames, so no wheel /// can be chosen there). A lock that is not TOML contributes nothing. -async fn inventory_poetry_lock(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("poetry.lock")) - .await - .ok()?; +async fn inventory_poetry_lock(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("poetry.lock").await.ok()?; let document: DocumentMut = text.parse().ok()?; let pure_wheel_sha = |files: Vec<&dyn TableLike>| { files.into_iter().find_map(|entry| { @@ -428,10 +451,8 @@ pub(super) fn socket_reference_coords(reference: &str) -> Option<(String, String /// our own already-wired file references. An unparseable lock contributes /// nothing, so the caller falls through to requirements.txt like an absent /// lock would. -async fn inventory_pipfile_lock(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("Pipfile.lock")) - .await - .ok()?; +async fn inventory_pipfile_lock(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("Pipfile.lock").await.ok()?; let value = parse_pipfile_lock(&text).ok()?; let root = value.as_object()?; // Digests are only fetchable through PyPI's JSON API when the lock @@ -515,10 +536,8 @@ async fn inventory_pipfile_lock(project_root: &Path) -> Option Option> { - let text = read_regular_to_string(&project_root.join("pdm.lock")) - .await - .ok()?; +async fn inventory_pdm_lock(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("pdm.lock").await.ok()?; let document: DocumentMut = text.parse().ok()?; let out: Vec = toml_package_coords(&document) .into_iter() @@ -542,10 +561,8 @@ async fn inventory_pdm_lock(project_root: &Path) -> Option> { /// logical lines with the shared requirements lexer /// ([`crate::utils::requirements`]: continuations joined, comments cut, one /// leading BOM dropped), the same one the planner and discovery use. -async fn inventory_requirements_txt(project_root: &Path) -> Option> { - let text = read_regular_to_string(&project_root.join("requirements.txt")) - .await - .ok()?; +async fn inventory_requirements_txt(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("requirements.txt").await.ok()?; let mut out = Vec::new(); for line in crate::utils::requirements::logical_lines(&text) { let t = crate::utils::requirements::strip_comment(&line.text).trim(); diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/view.rs b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs new file mode 100644 index 000000000..72e4bf315 --- /dev/null +++ b/crates/socket-patch-core/src/vendor/lock_inventory/view.rs @@ -0,0 +1,498 @@ +//! Where the registry views read a project from: the filesystem +//! ([`ProjectView::Disk`]) or an in-memory file map +//! ([`ProjectView::Memory`]) handed in by a host that never materializes +//! the repository (the hosted in-memory engine). The disk variant calls the +//! exact readers the views always used, so on-disk behavior is unchanged. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io; +use std::path::Path; +use std::sync::Arc; + +use crate::constants::npm_family::{BUN_LOCK, BUN_LOCKB, NPM_LOCKS, PNPM_LOCK, PNP_MARKERS}; +use crate::utils::fs::{ + read_regular_to_bytes, read_regular_to_string, read_regular_to_string_sync, +}; +use crate::vendor::npm_flavor::NpmLockFlavor; +use crate::vendor::pnpm_lock_legacy::{sniff_lock_grammar, PnpmLockGrammar}; +use crate::vendor::VendorWarning; + +/// One in-memory file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MemoryEntry { + /// UTF-8 content. + Text(Arc), + /// Raw bytes (e.g. `bun.lockb`). + Binary(Arc<[u8]>), + /// Known to exist, content not provided (presence-only markers, files + /// the host skipped as oversize, binary, or LFS pointers). + Present, + /// A symbolic link: exists, never readable, never writable. + Symlink, +} + +/// A project's files, keyed by `/`-separated project-relative path. +/// Directories are implied by the keys. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct MemoryProject { + entries: BTreeMap, +} + +impl MemoryProject { + pub fn new() -> Self { + Self::default() + } + + pub fn insert(&mut self, rel: impl Into, entry: MemoryEntry) { + self.entries.insert(rel.into(), entry); + } + + pub fn insert_text(&mut self, rel: impl Into, text: impl Into>) { + self.insert(rel, MemoryEntry::Text(text.into())); + } + + pub fn insert_binary(&mut self, rel: impl Into, bytes: impl Into>) { + self.insert(rel, MemoryEntry::Binary(bytes.into())); + } + + pub fn insert_present(&mut self, rel: impl Into) { + self.insert(rel, MemoryEntry::Present); + } + + pub fn insert_symlink(&mut self, rel: impl Into) { + self.insert(rel, MemoryEntry::Symlink); + } + + pub fn remove(&mut self, rel: &str) -> Option { + self.entries.remove(rel) + } + + pub fn get(&self, rel: &str) -> Option<&MemoryEntry> { + self.entries.get(rel) + } + + pub fn contains(&self, rel: &str) -> bool { + self.entries.contains_key(rel) + } + + pub fn text(&self, rel: &str) -> Option<&str> { + match self.entries.get(rel)? { + MemoryEntry::Text(text) => Some(text), + _ => None, + } + } + + pub fn is_symlink(&self, rel: &str) -> bool { + matches!(self.entries.get(rel), Some(MemoryEntry::Symlink)) + } + + pub fn paths(&self) -> impl Iterator { + self.entries.keys().map(String::as_str) + } + + pub fn entries(&self) -> impl Iterator { + self.entries.iter().map(|(k, v)| (k.as_str(), v)) + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Whether `rel` is an implied directory (some key lives under it). + /// The project root (`""`) is always a directory. + pub fn is_dir(&self, rel: &str) -> bool { + if rel.is_empty() { + return true; + } + let prefix = format!("{rel}/"); + self.entries + .range(prefix.clone()..) + .next() + .is_some_and(|(k, _)| k.starts_with(&prefix)) + } + + /// The direct children of directory `rel`, sorted: `(name, is_dir)`. + pub fn children(&self, rel: &str) -> Vec<(String, bool)> { + let prefix = if rel.is_empty() { + String::new() + } else { + format!("{rel}/") + }; + let mut files: BTreeSet = BTreeSet::new(); + let mut dirs: BTreeSet = BTreeSet::new(); + for key in self + .entries + .range(prefix.clone()..) + .map(|(k, _)| k) + .take_while(|k| k.starts_with(&prefix)) + { + let rest = &key[prefix.len()..]; + match rest.split_once('/') { + Some((dir, _)) => { + dirs.insert(dir.to_string()); + } + None => { + files.insert(rest.to_string()); + } + } + } + let mut out: Vec<(String, bool)> = files + .into_iter() + .filter(|f| !dirs.contains(f)) + .map(|f| (f, false)) + .collect(); + out.extend(dirs.into_iter().map(|d| (d, true))); + out.sort(); + out + } + + fn read_bytes(&self, rel: &str) -> io::Result> { + match self.entries.get(rel) { + Some(MemoryEntry::Text(text)) => Ok(text.as_bytes().to_vec()), + Some(MemoryEntry::Binary(bytes)) => Ok(bytes.to_vec()), + Some(MemoryEntry::Present) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "file content was not provided", + )), + Some(MemoryEntry::Symlink) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "is a symbolic link", + )), + None if self.is_dir(rel) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "not a regular file", + )), + None => Err(io::Error::new(io::ErrorKind::NotFound, "not found")), + } + } + + fn read_text(&self, rel: &str) -> io::Result { + match self.entries.get(rel) { + Some(MemoryEntry::Text(text)) => Ok(text.to_string()), + Some(MemoryEntry::Binary(bytes)) => String::from_utf8(bytes.to_vec()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "stream did not contain valid UTF-8", + ) + }), + _ => self.read_bytes(rel).map(|_| String::new()), + } + } +} + +/// A directory entry as the registry views need it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirEntryInfo { + pub name: String, + /// A real directory (a symbolic link to one is not). + pub is_dir: bool, +} + +/// Where the registry views read the project from. +#[derive(Debug, Clone, Copy)] +pub enum ProjectView<'a> { + Disk(&'a Path), + Memory(&'a MemoryProject), +} + +impl ProjectView<'_> { + /// FIFO-safe regular-file text read. + pub async fn read_text(&self, rel: &str) -> io::Result { + match self { + ProjectView::Disk(root) => read_regular_to_string(&root.join(rel)).await, + ProjectView::Memory(project) => project.read_text(rel), + } + } + + /// FIFO-safe regular-file byte read. + pub async fn read_bytes(&self, rel: &str) -> io::Result> { + match self { + ProjectView::Disk(root) => read_regular_to_bytes(&root.join(rel)).await, + ProjectView::Memory(project) => project.read_bytes(rel), + } + } + + /// Synchronous twin of [`Self::read_text`]. + pub fn read_text_sync(&self, rel: &str) -> io::Result { + match self { + ProjectView::Disk(root) => read_regular_to_string_sync(&root.join(rel)), + ProjectView::Memory(project) => project.read_text(rel), + } + } + + /// `metadata` (follows links) succeeds. + pub async fn exists(&self, rel: &str) -> bool { + match self { + ProjectView::Disk(root) => tokio::fs::metadata(root.join(rel)).await.is_ok(), + ProjectView::Memory(project) => project.contains(rel) || project.is_dir(rel), + } + } + + /// `symlink_metadata` (does not follow links) succeeds. + pub async fn exists_no_follow(&self, rel: &str) -> bool { + match self { + ProjectView::Disk(root) => tokio::fs::symlink_metadata(root.join(rel)).await.is_ok(), + ProjectView::Memory(project) => project.contains(rel) || project.is_dir(rel), + } + } + + /// A regular file (following links on disk). + pub fn is_file(&self, rel: &str) -> bool { + match self { + ProjectView::Disk(root) => root.join(rel).is_file(), + ProjectView::Memory(project) => matches!( + project.get(rel), + Some(MemoryEntry::Text(_) | MemoryEntry::Binary(_) | MemoryEntry::Present) + ), + } + } + + /// The path itself is a symbolic link. + pub fn is_symlink(&self, rel: &str) -> bool { + match self { + ProjectView::Disk(root) => { + std::fs::symlink_metadata(root.join(rel)).is_ok_and(|m| m.file_type().is_symlink()) + } + ProjectView::Memory(project) => project.is_symlink(rel), + } + } + + /// The UTF-8-named entries of directory `rel`, sorted by name. + pub async fn list_dir(&self, rel: &str) -> io::Result> { + match self { + ProjectView::Disk(root) => { + let mut dir = tokio::fs::read_dir(root.join(rel)).await?; + let mut out = Vec::new(); + while let Ok(Some(entry)) = dir.next_entry().await { + let Some(name) = entry.file_name().to_str().map(str::to_string) else { + continue; + }; + let is_dir = entry.file_type().await.is_ok_and(|t| t.is_dir()); + out.push(DirEntryInfo { name, is_dir }); + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) + } + ProjectView::Memory(project) => { + if !project.is_dir(rel) { + return Err(io::Error::new(io::ErrorKind::NotFound, "not found")); + } + Ok(project + .children(rel) + .into_iter() + .map(|(name, is_dir)| DirEntryInfo { name, is_dir }) + .collect()) + } + } + } +} + +/// How many head lines the yarn content sniff reads (mirrors the disk +/// probe). +const YARN_SNIFF_HEAD_LINES: usize = 30; + +/// [`crate::vendor::npm_flavor::detect_npm_lock_flavor`] over a +/// [`ProjectView`]. The disk variant IS the disk probe; the memory variant +/// follows the same decision table, with pnpm's own Plug'n'Play layout +/// never detected (there is no installed store in memory). +pub(crate) async fn detect_npm_lock_flavor_in( + view: &ProjectView<'_>, +) -> Result<(NpmLockFlavor, Vec), (&'static str, String)> { + let project = match view { + ProjectView::Disk(root) => { + return crate::vendor::npm_flavor::detect_npm_lock_flavor(root).await + } + ProjectView::Memory(project) => *project, + }; + let exists = |name: &str| project.contains(name); + let read_lock = |name: &str| -> Result { + project.read_text(name).map_err(|e| { + ( + "vendor_lockfile_missing", + format!("cannot read {name}: {e}"), + ) + }) + }; + + if let Some(marker) = PNP_MARKERS.iter().find(|m| exists(m)) { + return Err(( + "vendor_yarn_berry_unsupported", + format!( + "found `{marker}`: this is a yarn berry Plug'n'Play project — packages \ + live inside .yarn/cache/ zips, not node_modules/, so there is nothing \ + vendor could stage or rewire; use `yarn patch ` instead" + ), + )); + } + + let detected = 'flavor: { + if exists(BUN_LOCK) || exists(BUN_LOCKB) { + break 'flavor NpmLockFlavor::Bun; + } + if exists(PNPM_LOCK) { + let text = read_lock(PNPM_LOCK)?; + match sniff_lock_grammar(&text) { + Ok(PnpmLockGrammar::V9) => break 'flavor NpmLockFlavor::Pnpm, + Ok(PnpmLockGrammar::V54 | PnpmLockGrammar::V60) => { + break 'flavor NpmLockFlavor::PnpmLegacy + } + Err(detail) => return Err(("vendor_lockfile_version_unsupported", detail)), + } + } + if exists("yarn.lock") { + let text = read_lock("yarn.lock")?; + let head: Vec<&str> = text + .strip_prefix('\u{feff}') + .unwrap_or(&text) + .lines() + .take(YARN_SNIFF_HEAD_LINES) + .collect(); + if head.iter().any(|l| l.starts_with("__metadata:")) { + break 'flavor NpmLockFlavor::YarnBerry; + } + if head.iter().any(|l| l.trim() == "# yarn lockfile v1") { + break 'flavor NpmLockFlavor::YarnClassic; + } + return Err(( + "vendor_lockfile_version_unsupported", + "yarn.lock carries neither the `# yarn lockfile v1` header nor a berry \ + `__metadata:` key; cannot identify the lockfile version" + .to_string(), + )); + } + if exists(NPM_LOCKS[0]) || exists(NPM_LOCKS[1]) { + break 'flavor NpmLockFlavor::PackageLock; + } + if exists("rush.json") { + return Err(( + "vendor_rush_unsupported", + format!( + "found rush.json: this is a Rush monorepo — its single pnpm lockfile \ + lives at {}; use `socket-patch scan --mode hosted`, which edits it in \ + place", + crate::constants::npm_family::RUSH_COMMON_LOCK_REL + ), + )); + } + return Err(( + "vendor_lockfile_missing", + "no package-lock.json, npm-shrinkwrap.json, yarn.lock, pnpm-lock.yaml, bun.lock, \ + or bun.lockb in the project root" + .to_string(), + )); + }; + Ok((detected, Vec::new())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn project(files: &[(&str, MemoryEntry)]) -> MemoryProject { + let mut p = MemoryProject::new(); + for (k, v) in files { + p.insert(*k, v.clone()); + } + p + } + + fn text(s: &str) -> MemoryEntry { + MemoryEntry::Text(Arc::from(s)) + } + + #[test] + fn children_lists_files_and_implied_dirs_sorted() { + let p = project(&[ + ("b.txt", text("")), + ("a/x.toml", text("")), + ("a/y/z.toml", text("")), + ("c", MemoryEntry::Symlink), + ]); + assert_eq!( + p.children(""), + vec![ + ("a".to_string(), true), + ("b.txt".to_string(), false), + ("c".to_string(), false) + ] + ); + assert_eq!( + p.children("a"), + vec![("x.toml".to_string(), false), ("y".to_string(), true)] + ); + assert!(p.is_dir("a/y")); + assert!(!p.is_dir("a/x.toml")); + assert!(!p.is_dir("ab")); + } + + #[tokio::test] + async fn memory_reads_classify_like_the_disk_reader() { + let p = project(&[ + ("t", text("hello")), + ("b", MemoryEntry::Binary(Arc::from(vec![0xffu8, 0xfe]))), + ("p", MemoryEntry::Present), + ("s", MemoryEntry::Symlink), + ("d/f", text("")), + ]); + let view = ProjectView::Memory(&p); + assert_eq!(view.read_text("t").await.unwrap(), "hello"); + assert_eq!( + view.read_text("b").await.unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + assert_eq!(view.read_bytes("b").await.unwrap(), vec![0xff, 0xfe]); + assert!(view.read_text("p").await.is_err()); + assert!(view.read_text("s").await.is_err()); + assert_eq!( + view.read_text("missing").await.unwrap_err().kind(), + io::ErrorKind::NotFound + ); + assert_eq!( + view.read_text("d").await.unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert!(view.exists("s").await); + assert!(view.is_symlink("s")); + assert!(!view.is_file("s")); + assert!(view.is_file("p")); + } + + #[tokio::test] + async fn memory_flavor_probe_follows_the_disk_decision_table() { + let berry = project(&[("yarn.lock", text("__metadata:\n version: 8\n"))]); + assert_eq!( + detect_npm_lock_flavor_in(&ProjectView::Memory(&berry)) + .await + .unwrap() + .0, + NpmLockFlavor::YarnBerry + ); + let bun_over_npm = project(&[ + ("package-lock.json", text("{}")), + ("bun.lockb", MemoryEntry::Present), + ]); + assert_eq!( + detect_npm_lock_flavor_in(&ProjectView::Memory(&bun_over_npm)) + .await + .unwrap() + .0, + NpmLockFlavor::Bun + ); + let pnp = project(&[(".pnp.cjs", MemoryEntry::Present), ("yarn.lock", text(""))]); + assert_eq!( + detect_npm_lock_flavor_in(&ProjectView::Memory(&pnp)) + .await + .unwrap_err() + .0, + "vendor_yarn_berry_unsupported" + ); + let empty = MemoryProject::new(); + assert_eq!( + detect_npm_lock_flavor_in(&ProjectView::Memory(&empty)) + .await + .unwrap_err() + .0, + "vendor_lockfile_missing" + ); + } +} diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/yarn.rs b/crates/socket-patch-core/src/vendor/lock_inventory/yarn.rs index d7c2413c9..91b489217 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/yarn.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/yarn.rs @@ -1,10 +1,10 @@ //! `yarn.lock`, classic and berry: the entry models lockfile discovery //! shares ([`classic_entries`], [`berry_entries`]) and the registry views. +#[cfg(test)] use std::path::Path; use crate::utils::digest::is_hex; -use crate::utils::fs::read_regular_to_string; use crate::vendor::yarn_berry_lock::{ berry_field, berry_metadata, parse_berry_locator, BerryLocator, }; @@ -13,6 +13,7 @@ use crate::vendor::yarn_classic_lock::{ split_resolved_sha1, LockBlock, }; +use super::view::ProjectView; use super::{http_url, LockIntegrity, LockfileEntry}; // ── entry model ── @@ -105,8 +106,15 @@ pub(crate) fn berry_checksum_pin(value: &str, cache_key: Option<&str>) -> Option // ── registry view ── +#[cfg(test)] pub(super) async fn inventory_yarn_classic(root: &Path) -> Option> { - let text = read_regular_to_string(&root.join("yarn.lock")).await.ok()?; + inventory_yarn_classic_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_yarn_classic_in( + view: &ProjectView<'_>, +) -> Option> { + let text = view.read_text("yarn.lock").await.ok()?; Some(classic_registry_view(&text)) } @@ -147,8 +155,13 @@ fn classic_registry_view(text: &str) -> Vec { out } +#[cfg(test)] pub(super) async fn inventory_yarn_berry(root: &Path) -> Option> { - let text = read_regular_to_string(&root.join("yarn.lock")).await.ok()?; + inventory_yarn_berry_in(&ProjectView::Disk(root)).await +} + +pub(super) async fn inventory_yarn_berry_in(view: &ProjectView<'_>) -> Option> { + let text = view.read_text("yarn.lock").await.ok()?; Some(berry_registry_view(&text)) } diff --git a/crates/socket-patch-core/src/vendor/pypi.rs b/crates/socket-patch-core/src/vendor/pypi.rs index 48a1ba98d..57e587b5e 100644 --- a/crates/socket-patch-core/src/vendor/pypi.rs +++ b/crates/socket-patch-core/src/vendor/pypi.rs @@ -84,7 +84,7 @@ fn validate_hosted_wheel_sha256(sha256: &str) -> Result<(), String> { Ok(()) } -fn decode_hosted_wheel_metadata(bytes: &[u8], sha256: &str) -> Result, String> { +pub fn decode_hosted_wheel_metadata(bytes: &[u8], sha256: &str) -> Result, String> { validate_hosted_wheel_sha256(sha256)?; if !hex::encode(Sha256::digest(bytes)).eq_ignore_ascii_case(sha256) { return Err("hosted wheel sha256 does not match the published artifact".to_string()); From 3ed381887b6cab1a36eead1ca6683a3dd1748ad7 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Sat, 26 Sep 2026 18:14:56 -0400 Subject: [PATCH 2/4] Add napi addon for the hosted engine Expose the in-memory hosted engine to Node as a native addon, so a service can stream repository files into a scan session and serve patch lookups from its own data source instead of the Socket API. The package lives under crates/socket-patch-node/npm and is private, so the npm release tooling does not publish it. Release builds now build only the CLI crate, and a new CI job builds the addon and runs its Node smoke test. Assisted-by: Claude Code:claude-opus-5-5 --- .github/workflows/ci.yml | 32 ++ .github/workflows/release.yml | 4 +- Cargo.lock | 112 +++++ Cargo.toml | 17 + crates/socket-patch-node/Cargo.toml | 31 ++ crates/socket-patch-node/build.rs | 3 + crates/socket-patch-node/npm/.gitignore | 2 + crates/socket-patch-node/npm/index.d.ts | 82 ++++ crates/socket-patch-node/npm/index.js | 355 ++++++++++++++ crates/socket-patch-node/npm/package.json | 19 + .../npm/scripts/build-addon.mjs | 39 ++ crates/socket-patch-node/npm/test/smoke.mjs | 461 ++++++++++++++++++ crates/socket-patch-node/src/lib.rs | 330 +++++++++++++ crates/socket-patch-node/src/provider.rs | 280 +++++++++++ 14 files changed, 1765 insertions(+), 2 deletions(-) create mode 100644 crates/socket-patch-node/Cargo.toml create mode 100644 crates/socket-patch-node/build.rs create mode 100644 crates/socket-patch-node/npm/.gitignore create mode 100644 crates/socket-patch-node/npm/index.d.ts create mode 100644 crates/socket-patch-node/npm/index.js create mode 100644 crates/socket-patch-node/npm/package.json create mode 100644 crates/socket-patch-node/npm/scripts/build-addon.mjs create mode 100644 crates/socket-patch-node/npm/test/smoke.mjs create mode 100644 crates/socket-patch-node/src/lib.rs create mode 100644 crates/socket-patch-node/src/provider.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fea73c77f..1b756adfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,38 @@ jobs: exit 1 fi + # The napi addon is only ever loaded by Node, so cargo's own tests never + # exercise its JS loader or the engine/provider boundary. + node-addon: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Rust + run: rustup show + + - name: Cache cargo + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '20.20.2' + + - name: Build addon + env: + SOCKET_PATCH_NODE_CARGO_PROFILE: dev + run: node crates/socket-patch-node/npm/scripts/build-addon.mjs + + - name: Smoke-test addon + run: node --test crates/socket-patch-node/npm/test/smoke.mjs + # Lint the out-of-workspace packaging artifacts: the RubyGems CLI launcher # gem + the Bundler plugin gem (Ruby), and the curl|sh installer. Ruby is # pre-installed on the ubuntu-latest runner. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ce21c1e31..7c2abcb48 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -175,11 +175,11 @@ jobs: - name: Build (cargo) if: matrix.build-tool == 'cargo' - run: cargo build --release --target ${{ matrix.target }} + run: cargo build --release -p socket-patch-cli --target ${{ matrix.target }} - name: Build (cross) if: matrix.build-tool == 'cross' - run: cross build --release --target ${{ matrix.target }} + run: cross build --release -p socket-patch-cli --target ${{ matrix.target }} - name: Package (unix) if: matrix.archive == 'tar.gz' diff --git a/Cargo.lock b/Cargo.lock index 9dc768837..2f603dcf6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,6 +257,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "convert_case" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -319,6 +328,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" + [[package]] name = "deadpool" version = "0.12.3" @@ -955,6 +970,16 @@ version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1015,6 +1040,65 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "napi" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b12638855cec016784a7e17cc4ecbdee7ca8b0a3456aa50640d0ee5ed1a42f8" +dependencies = [ + "bitflags 2.11.0", + "ctor", + "futures", + "libc", + "napi-build", + "napi-sys", + "nohash-hasher", + "rustc-hash", + "tokio", +] + +[[package]] +name = "napi-build" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0941983559cd62bfa7f04646a16428967a847365adeef27711e4e3927046bbed" + +[[package]] +name = "napi-derive" +version = "3.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1d232c24dc6754979b27efdf2047a5849efc7c92c1245de87e6bedbb9f5c3c" +dependencies = [ + "convert_case", + "ctor", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "napi-derive-backend" +version = "6.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6856fe1686867da90ed0dc292d7c0c87a695eb7c578b913360f33619ee01e3" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "semver", + "syn", +] + +[[package]] +name = "napi-sys" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6801fc65ba4d7035636e93635e281113f00410401a237dcd4f55b1565ead2115" +dependencies = [ + "libloading", +] + [[package]] name = "nix" version = "0.28.0" @@ -1027,6 +1111,12 @@ dependencies = [ "libc", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "num-traits" version = "0.2.19" @@ -1680,6 +1770,7 @@ dependencies = [ "tar", "tempfile", "tokio", + "tokio-util", "uuid", "windows-sys 0.59.0", "wiremock", @@ -1718,6 +1809,21 @@ dependencies = [ "zip", ] +[[package]] +name = "socket-patch-node" +version = "4.0.0" +dependencies = [ + "napi", + "napi-build", + "napi-derive", + "serde", + "serde_json", + "socket-patch-cli", + "socket-patch-core", + "tokio", + "tokio-util", +] + [[package]] name = "socket2" version = "0.6.2" @@ -2051,6 +2157,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index c1e954830..398ef1eeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,13 @@ members = [ "crates/socket-patch-core", "crates/socket-patch-cli", + "crates/socket-patch-node", +] +# Bare `cargo build` (release.yml's per-target builds) skips the Node addon, +# which no release artifact ships; `--workspace` and `-p` still build it. +default-members = [ + "crates/socket-patch-core", + "crates/socket-patch-cli", ] resolver = "2" @@ -21,6 +28,7 @@ sha1 = "=0.10.6" hex = "=0.4.3" reqwest = { version = "=0.12.28", features = ["rustls-tls", "json"], default-features = false } tokio = { version = "=1.50.0", features = ["full"] } +tokio-util = "=0.7.18" thiserror = "=2.0.18" walkdir = "=2.5.0" uuid = { version = "=1.21.0", features = ["v4"] } @@ -45,6 +53,9 @@ wiremock = "=0.6.5" portable-pty = "=0.9.0" base64 = "=0.22.1" serial_test = "=3.4.0" +napi = { version = "=3.13.0", features = ["napi8", "tokio_rt"] } +napi-derive = "=3.6.9" +napi-build = "=2.5.0" [profile.release] strip = true @@ -62,6 +73,12 @@ inherits = "release" lto = false strip = "none" # unstripped test binaries => usable backtraces on failure +# The Node addon (crates/socket-patch-node) for hosts that load it +# in-process: release semantics without the slow full-LTO link. +[profile.addon] +inherits = "release" +lto = "thin" + # Test-execution speed: `cargo test` builds dependencies with the dev # profile; at opt-level 0 the hash/compression/bsdiff hot loops are 10-100x # slower (the self_update fixture family measured 403s debug vs ~2s release diff --git a/crates/socket-patch-node/Cargo.toml b/crates/socket-patch-node/Cargo.toml new file mode 100644 index 000000000..d95cb71a4 --- /dev/null +++ b/crates/socket-patch-node/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "socket-patch-node" +description = "Node addon that runs the socket-patch hosted redirect engine in-process" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[lib] +name = "socket_patch_node" +path = "src/lib.rs" +crate-type = ["cdylib"] +# A cdylib that links against the host's `napi_*` symbols has nothing to +# test outside Node; the engine's tests live in socket-patch-cli and the +# addon's in npm/test/smoke.mjs. +test = false +doctest = false + +[dependencies] +socket-patch-cli = { path = "../socket-patch-cli" } +socket-patch-core = { workspace = true } +napi = { workspace = true } +napi-derive = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } + +[build-dependencies] +napi-build = { workspace = true } diff --git a/crates/socket-patch-node/build.rs b/crates/socket-patch-node/build.rs new file mode 100644 index 000000000..0f1b01002 --- /dev/null +++ b/crates/socket-patch-node/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/crates/socket-patch-node/npm/.gitignore b/crates/socket-patch-node/npm/.gitignore new file mode 100644 index 000000000..5797b59e6 --- /dev/null +++ b/crates/socket-patch-node/npm/.gitignore @@ -0,0 +1,2 @@ +socket_patch_node.node +node_modules/ diff --git a/crates/socket-patch-node/npm/index.d.ts b/crates/socket-patch-node/npm/index.d.ts new file mode 100644 index 000000000..e5fc61dc7 --- /dev/null +++ b/crates/socket-patch-node/npm/index.d.ts @@ -0,0 +1,82 @@ +export type Ecosystem = 'npm' | 'pypi' | 'cargo' | 'golang' | 'gem' | 'composer' | 'maven' | 'nuget' + +export interface TreeEntryInput { path: string; mode: string; type: 'blob' | 'tree' | 'commit'; size?: number } +export interface PathSelection { + roots: string[] // detected project roots, repo-relative ('' = repo root), sorted + fetchText: string[] // stream these as UTF-8 text files + fetchBinary: string[] // stream these as raw bytes (e.g. bun.lockb) + presentOnly: string[] // engine only needs to know they exist (e.g. .pnp.cjs, rush repo-state.json) + symlinks: string[] // candidate paths that are symlinks (mode 120000) — refuse-to-write + ignoredCount: number + ignoredSample: { path: string; reason: string }[] // ≤100 +} +export function selectHostedScanPaths(entries: TreeEntryInput[], options?: { projectRoots?: string[]; ecosystems?: Ecosystem[] }): PathSelection +export function hostedScanCandidateFiles(): string[] // debug listing only +export function engineVersion(): string // "+" + +export type ProviderErrorKind = 'unauthorized' | 'forbidden' | 'rate_limited' | 'network' | 'parse' | 'not_found' | 'other' +export type ProviderResult = { ok: true; value: T } | { ok: false; error: { kind: ProviderErrorKind; message: string } } + +// Request/response bodies are EXACTLY the api-v0 HTTP JSON bodies (camelCase), so the Rust serde types are reused unchanged. +export interface BatchPatchInfo { uuid: string; purl: string; tier: string; cveIds: string[]; ghsaIds: string[]; severity: string | null; title: string; publishedAt?: string } +export interface BatchSearchResponse { packages: { purl: string; patches: BatchPatchInfo[] }[]; canAccessPaidPatches: boolean } +export interface PatchSearchResult { uuid: string; purl: string; publishedAt: string; description: string; license: string; tier: string; vulnerabilities: Record } +export interface SearchResponse { patches: PatchSearchResult[]; canAccessPaidPatches: boolean } +export interface PackageVendorResult { status: string; url: string | null; purl: string | null; artifacts: unknown[] | null; registryOverride: unknown | null } // exact api-v0 package.ts serialization (null-filled) +export interface PatchResponse { uuid: string; purl: string; publishedAt: string; files: Record; vulnerabilities: Record; description: string; license: string; tier: string } + +export interface PatchProvider { + searchPatchesBatch(request: { components: { purl: string }[] }): Promise> + searchPatchesByPackage(request: { purl: string }): Promise> + fetchRegistryReferences(request: { uuids: string[] }): Promise }>> + fetchPatch(request: { uuid: string }): Promise> + downloadArtifact(request: { url: string; maxBytes: number }): Promise> +} +// Providers MUST resolve (never reject); the JS loader wraps provider methods so a thrown/rejected call becomes {ok:false, kind:'other'}. +// A 'not_found' failure mirrors the HTTP API's 404 per method: searchPatchesByPackage => no patches, fetchRegistryReferences => no references, fetchPatch => null, searchPatchesBatch and downloadArtifact => error. + +export interface HostedScanLimits { maxFileBytes?: number /*20 MiB*/; maxTotalBytes?: number /*64 MiB*/; maxFiles?: number /*2000*/; maxPurls?: number /*20000*/; maxProjects?: number /*200*/; maxArtifactBytes?: number /*32 MiB*/ } +export interface HostedScanSessionOptions { + orgSlug: string + ecosystems?: Ecosystem[] + batchSize?: number // 1..500, default 100 + dryRun?: boolean + pipenvMajor?: number // never spawns pipenv; absent => same default as CLI when pipenv unavailable + trustLockfileConfig?: boolean // default true + npmAllowRemoteConfig?: boolean // default true + projectRoots?: string[] // must match selectHostedScanPaths input + providerConcurrency?: number // default 8 + requestTimeoutMs?: number // per provider call, default 60000 + limits?: HostedScanLimits +} +export class HostedScanSession { + constructor(options: HostedScanSessionOptions, provider: PatchProvider) + pushChunk(path: string, chunk: Buffer): void // throws on limit breach, unknown state, or after finish + endFile(path: string): void + markPresent(path: string, kind: 'present' | 'symlink' | 'binary_skipped' | 'oversize' | 'lfs_pointer'): void + finish(): Promise // runs off the JS thread; rejects only on engine bug/limit/cancel + cancel(): void // cooperative; finish() rejects with code 'cancelled' +} +export interface EngineWarning { code: string; detail: string; projectRoot?: string } +export interface ProjectResult { + root: string + redirect: Record // same shape as CLI `--json` `redirect` block + summary: { scannedPackages: number; packagesWithPatches: number; totalPatches: number; freePatches: number; paidPatches: number; canAccessPaidPatches: boolean } + redirected: { purl: string; uuid: string }[] + skipped: { purl: string; uuid: string; reason: string; detail?: string }[] + error?: { code: string; message: string } // project-level failure (e.g. corrupt_ledger, patch_lookup_failed) +} +export interface HostedScanResult { + projects: ProjectResult[] + changedFiles: { path: string; content: string }[] // repo-relative, sorted, only byte-changed, includes ledgers (wet runs only) + changedBinaryFiles: { path: string; content: Buffer }[] + deletedFiles: string[] + warnings: EngineWarning[] + stats: { projects: number; filesInput: number; bytesInput: number; packagesScanned: number; packagesWithPatches: number; patchesSelected: number; patchesRedirected: number; filesChanged: number; providerCalls: Record; phaseMs: Record } + engineVersion: string +} + +export class SocketPatchAddonUnavailableError extends Error { + readonly code: 'addon_unavailable' + readonly attempted: string[] +} diff --git a/crates/socket-patch-node/npm/index.js b/crates/socket-patch-node/npm/index.js new file mode 100644 index 000000000..381d33bbc --- /dev/null +++ b/crates/socket-patch-node/npm/index.js @@ -0,0 +1,355 @@ +'use strict' + +const fs = require('node:fs') +const path = require('node:path') + +const ADDON_ENV = 'SOCKET_PATCH_NODE_ADDON_PATH' +const PROVIDER_ERROR_KINDS = new Set([ + 'unauthorized', + 'forbidden', + 'rate_limited', + 'network', + 'parse', + 'not_found', + 'other', +]) +const JSON_PROVIDER_METHODS = [ + 'searchPatchesBatch', + 'searchPatchesByPackage', + 'fetchRegistryReferences', + 'fetchPatch', +] +const MAX_PROVIDER_MESSAGE = 2000 +const REQUIRED_EXPORTS = [ + 'selectHostedScanPathsJson', + 'hostedScanCandidateFiles', + 'engineVersion', + 'NativeHostedScanSession', +] + +class SocketPatchAddonUnavailableError extends Error { + constructor(message, attempted, options) { + super(message, options) + this.name = 'SocketPatchAddonUnavailableError' + this.code = 'addon_unavailable' + this.attempted = attempted + } +} + +function libraryNames() { + if (process.platform === 'win32') { + return ['socket_patch_node.dll'] + } + if (process.platform === 'darwin') { + return ['libsocket_patch_node.dylib'] + } + return ['libsocket_patch_node.so'] +} + +function candidatePaths() { + const explicit = process.env[ADDON_ENV] + if (explicit) { + return [path.resolve(explicit)] + } + const candidates = [path.join(__dirname, 'socket_patch_node.node')] + if (process.env.NODE_ENV !== 'production') { + const targetDir = path.resolve(__dirname, '..', '..', '..', 'target') + for (const profile of ['release', 'debug']) { + for (const name of libraryNames()) { + candidates.push(path.join(targetDir, profile, name)) + } + } + } + return candidates +} + +function loadFile(file) { + if (path.extname(file) === '.node') { + return require(file) + } + const addonModule = { exports: {} } + process.dlopen(addonModule, file) + return addonModule.exports +} + +let cachedBinding = null + +function binding() { + if (cachedBinding) { + return cachedBinding + } + const attempted = candidatePaths() + const found = attempted.find((file) => fs.existsSync(file)) + if (!found) { + const hint = process.env[ADDON_ENV] + ? `${ADDON_ENV} points at a missing file` + : `build it with \`pnpm run build:addon\` or set ${ADDON_ENV}` + throw new SocketPatchAddonUnavailableError( + `socket-patch-node addon not found (${hint}); tried: ${attempted.join(', ')}`, + attempted, + ) + } + let loaded + try { + loaded = loadFile(found) + } catch (error) { + throw new SocketPatchAddonUnavailableError( + `socket-patch-node addon at ${found} failed to load: ${describe(error)}`, + attempted, + { cause: error }, + ) + } + const missing = REQUIRED_EXPORTS.filter((name) => loaded[name] === undefined) + if (missing.length > 0) { + throw new SocketPatchAddonUnavailableError( + `socket-patch-node addon at ${found} is missing exports: ${missing.join(', ')}`, + attempted, + ) + } + cachedBinding = loaded + return loaded +} + +function describe(error) { + let text + if (error instanceof Error) { + text = error.message + } else { + try { + text = String(error) + } catch { + text = 'unprintable error' + } + } + return text.length > MAX_PROVIDER_MESSAGE + ? `${text.slice(0, MAX_PROVIDER_MESSAGE)}…` + : text +} + +function engineError(code, kind, message) { + const error = new Error(message) + error.name = 'HostedScanError' + error.code = code + error.kind = kind + return error +} + +function fromNativeError(error) { + const reason = error instanceof Error ? error.message : undefined + if (typeof reason === 'string') { + try { + const parsed = JSON.parse(reason) + if ( + parsed && + typeof parsed.code === 'string' && + typeof parsed.kind === 'string' && + typeof parsed.message === 'string' + ) { + return engineError(parsed.code, parsed.kind, parsed.message) + } + } catch {} + } + return engineError('addon_internal', 'internal', describe(error)) +} + +function callNative(call) { + try { + return call() + } catch (error) { + throw fromNativeError(error) + } +} + +function providerFailure(kind, message) { + return { + ok: false, + error: { kind: PROVIDER_ERROR_KINDS.has(kind) ? kind : 'other', message }, + } +} + +function normalizeJsonResult(result) { + if (result === null || typeof result !== 'object' || typeof result.ok !== 'boolean') { + return providerFailure('other', 'provider returned a malformed result') + } + if (result.ok) { + return { ok: true, value: result.value === undefined ? null : result.value } + } + const error = result.error !== null && typeof result.error === 'object' ? result.error : {} + return providerFailure( + typeof error.kind === 'string' ? error.kind : 'other', + typeof error.message === 'string' ? describe(error.message) : '', + ) +} + +function serializeResult(result) { + try { + return JSON.stringify(result) + } catch (error) { + return JSON.stringify( + providerFailure('parse', `provider result is not serializable: ${describe(error)}`), + ) + } +} + +function invokeProvider(provider, method, requestJson) { + return Promise.resolve().then(() => { + const request = JSON.parse(requestJson) + const fn = provider[method] + if (typeof fn !== 'function') { + throw new TypeError(`provider.${method} is not a function`) + } + return fn.call(provider, request) + }) +} + +function wrapJsonMethod(provider, method) { + return (requestJson) => + invokeProvider(provider, method, requestJson) + .then(normalizeJsonResult, (error) => providerFailure('other', describe(error))) + .then(serializeResult) + .catch(() => serializeResult(providerFailure('other', `provider.${method} failed`))) +} + +function toBuffer(value) { + if (Buffer.isBuffer(value)) { + return value + } + if (value instanceof Uint8Array) { + return Buffer.from(value.buffer, value.byteOffset, value.byteLength) + } + return null +} + +function flatDownloadFailure(kind, message) { + return { + ok: false, + value: undefined, + kind: PROVIDER_ERROR_KINDS.has(kind) ? kind : 'other', + message, + } +} + +function normalizeDownloadResult(result) { + if (result === null || typeof result !== 'object' || typeof result.ok !== 'boolean') { + return flatDownloadFailure('other', 'provider returned a malformed result') + } + if (result.ok) { + const bytes = toBuffer(result.value) + if (!bytes) { + return flatDownloadFailure('parse', 'downloadArtifact must resolve a Buffer') + } + return { ok: true, value: bytes, kind: undefined, message: undefined } + } + const error = result.error !== null && typeof result.error === 'object' ? result.error : {} + return flatDownloadFailure( + typeof error.kind === 'string' ? error.kind : 'other', + typeof error.message === 'string' ? describe(error.message) : '', + ) +} + +function wrapDownloadMethod(provider) { + return (requestJson) => + invokeProvider(provider, 'downloadArtifact', requestJson) + .then(normalizeDownloadResult, (error) => flatDownloadFailure('other', describe(error))) + .catch(() => flatDownloadFailure('other', 'provider.downloadArtifact failed')) +} + +function wrapProvider(provider) { + if (provider === null || (typeof provider !== 'object' && typeof provider !== 'function')) { + throw new TypeError('provider must be an object implementing PatchProvider') + } + const wrapped = {} + for (const method of JSON_PROVIDER_METHODS) { + wrapped[method] = wrapJsonMethod(provider, method) + } + wrapped.downloadArtifact = wrapDownloadMethod(provider) + return wrapped +} + +function requirePath(value) { + if (typeof value !== 'string') { + throw new TypeError('path must be a string') + } + return value +} + +function selectHostedScanPaths(entries, options) { + if (!Array.isArray(entries)) { + throw new TypeError('entries must be an array of tree entries') + } + const native = binding() + const optionsJson = options === undefined || options === null ? null : JSON.stringify(options) + return JSON.parse( + callNative(() => native.selectHostedScanPathsJson(JSON.stringify(entries), optionsJson)), + ) +} + +function hostedScanCandidateFiles() { + return binding().hostedScanCandidateFiles() +} + +function engineVersion() { + return binding().engineVersion() +} + +class HostedScanSession { + #native + + constructor(options, provider) { + if (options === null || typeof options !== 'object') { + throw new TypeError('options must be a HostedScanSessionOptions object') + } + const native = binding() + const wrapped = wrapProvider(provider) + const optionsJson = JSON.stringify(options) + this.#native = callNative(() => new native.NativeHostedScanSession(optionsJson, wrapped)) + } + + pushChunk(path, chunk) { + const bytes = toBuffer(chunk) + if (!bytes) { + throw new TypeError('chunk must be a Buffer or Uint8Array') + } + callNative(() => this.#native.pushChunk(requirePath(path), bytes)) + } + + endFile(path) { + callNative(() => this.#native.endFile(requirePath(path))) + } + + markPresent(path, kind) { + if (typeof kind !== 'string') { + throw new TypeError('kind must be a string') + } + callNative(() => this.#native.markPresent(requirePath(path), kind)) + } + + async finish() { + const outcome = await callNative(() => this.#native.finish()) + if (!outcome.ok) { + throw engineError( + outcome.errorCode ?? 'engine_internal', + outcome.errorKind ?? 'internal', + outcome.errorMessage ?? 'the hosted scan failed', + ) + } + const result = JSON.parse(outcome.resultJson) + result.changedBinaryFiles = (outcome.binaryFiles ?? []).map((file) => ({ + path: file.path, + content: file.content, + })) + return result + } + + cancel() { + callNative(() => this.#native.cancel()) + } +} + +module.exports = { + HostedScanSession, + SocketPatchAddonUnavailableError, + engineVersion, + hostedScanCandidateFiles, + selectHostedScanPaths, +} diff --git a/crates/socket-patch-node/npm/package.json b/crates/socket-patch-node/npm/package.json new file mode 100644 index 000000000..1bd82062d --- /dev/null +++ b/crates/socket-patch-node/npm/package.json @@ -0,0 +1,19 @@ +{ + "name": "@socketsecurity/socket-patch-node", + "version": "4.0.0", + "private": true, + "description": "In-process Node addon for the socket-patch hosted redirect engine", + "license": "MIT", + "main": "index.js", + "types": "index.d.ts", + "files": [ + "index.js", + "index.d.ts" + ], + "scripts": { + "build:addon": "node scripts/build-addon.mjs" + }, + "engines": { + "node": ">=18" + } +} diff --git a/crates/socket-patch-node/npm/scripts/build-addon.mjs b/crates/socket-patch-node/npm/scripts/build-addon.mjs new file mode 100644 index 000000000..dc8d74e72 --- /dev/null +++ b/crates/socket-patch-node/npm/scripts/build-addon.mjs @@ -0,0 +1,39 @@ +import { spawnSync } from 'node:child_process' +import { copyFileSync, existsSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const packageDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const repoRoot = path.resolve(packageDir, '..', '..', '..') +const profile = process.env.SOCKET_PATCH_NODE_CARGO_PROFILE || 'release' +const targetDir = process.env.CARGO_TARGET_DIR + ? path.resolve(process.env.CARGO_TARGET_DIR) + : path.join(repoRoot, 'target') + +const cargo = spawnSync( + 'cargo', + ['build', '--locked', '--profile', profile, '-p', 'socket-patch-node'], + { cwd: repoRoot, stdio: 'inherit' }, +) +if (cargo.error) { + throw cargo.error +} +if (cargo.status !== 0) { + process.exit(cargo.status ?? 1) +} + +const libraryName = + process.platform === 'win32' + ? 'socket_patch_node.dll' + : process.platform === 'darwin' + ? 'libsocket_patch_node.dylib' + : 'libsocket_patch_node.so' +const profileDir = profile === 'dev' ? 'debug' : profile +const built = path.join(targetDir, profileDir, libraryName) +if (!existsSync(built)) { + console.error(`build:addon: ${built} was not produced`) + process.exit(1) +} +const destination = path.join(packageDir, 'socket_patch_node.node') +copyFileSync(built, destination) +console.log(`build:addon: copied ${built} -> ${destination}`) diff --git a/crates/socket-patch-node/npm/test/smoke.mjs b/crates/socket-patch-node/npm/test/smoke.mjs new file mode 100644 index 000000000..a333ade1d --- /dev/null +++ b/crates/socket-patch-node/npm/test/smoke.mjs @@ -0,0 +1,461 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import path from 'node:path' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' + +const require = createRequire(import.meta.url) +const addon = require('../index.js') + +const here = path.dirname(fileURLToPath(import.meta.url)) +const fixtureDir = path.resolve( + here, + '../../../socket-patch-core/tests/fixtures/redirect/npm/package-lock-v3/basic', +) +const inputLock = readFileSync(path.join(fixtureDir, 'input/package-lock.json')) +const expectedLock = readFileSync(path.join(fixtureDir, 'expected/package-lock.json'), 'utf8') +const overrides = JSON.parse(readFileSync(path.join(fixtureDir, 'overrides.json'), 'utf8')) + +const patches = overrides.map((o) => ({ + purl: `pkg:${o.ecosystem}/${o.namespace ? `${o.namespace}/` : ''}${o.name}@${o.version}`, + uuid: o.patchUuid, + reference: { + status: 'granted', + url: o.artifactUrl, + purl: null, + artifacts: [{ kind: 'tarball', url: o.artifactUrl, integrity: o.integrity }], + registryOverride: o.registryOverride ?? null, + }, +})) + +const vulnerabilities = { + 'GHSA-test-aaaa-bbbb': { + cves: ['CVE-2024-0001'], + summary: 's', + severity: 'high', + description: 'd', + }, +} + +function fakeProvider(overridesByMethod = {}) { + const calls = { + searchPatchesBatch: 0, + searchPatchesByPackage: 0, + fetchRegistryReferences: 0, + fetchPatch: 0, + downloadArtifact: 0, + } + const base = { + async searchPatchesBatch({ components }) { + const packages = [] + for (const { purl } of components) { + const matches = patches.filter((p) => p.purl === purl) + if (matches.length > 0) { + packages.push({ + purl, + patches: matches.map((p) => ({ + uuid: p.uuid, + purl, + tier: 'free', + cveIds: [], + ghsaIds: ['GHSA-test-aaaa-bbbb'], + severity: 'high', + title: 'fixture', + })), + }) + } + } + return { ok: true, value: { packages, canAccessPaidPatches: false } } + }, + async searchPatchesByPackage({ purl }) { + return { + ok: true, + value: { + patches: patches + .filter((p) => p.purl === purl) + .map((p) => ({ + uuid: p.uuid, + purl, + publishedAt: '2024-01-01T00:00:00Z', + description: 'fixture', + license: 'MIT', + tier: 'free', + vulnerabilities, + })), + canAccessPaidPatches: false, + }, + } + }, + async fetchRegistryReferences({ uuids }) { + const results = {} + for (const uuid of uuids) { + const patch = patches.find((p) => p.uuid === uuid) + if (patch) { + results[uuid] = patch.reference + } + } + return { ok: true, value: { results } } + }, + async fetchPatch({ uuid }) { + const patch = patches.find((p) => p.uuid === uuid) + if (!patch) { + return { ok: true, value: null } + } + return { + ok: true, + value: { + uuid: patch.uuid, + purl: patch.purl, + publishedAt: '2024-01-01T00:00:00Z', + files: { + 'package/index.js': { beforeHash: 'a'.repeat(64), afterHash: 'b'.repeat(64) }, + }, + vulnerabilities, + description: 'fixture', + license: 'MIT', + tier: 'free', + }, + } + }, + async downloadArtifact() { + return { ok: false, error: { kind: 'not_found', message: 'no artifacts in this fixture' } } + }, + } + const provider = {} + for (const method of Object.keys(calls)) { + const impl = overridesByMethod[method] ?? base[method] + provider[method] = (request) => { + calls[method] += 1 + return impl(request) + } + } + return { provider, calls } +} + +const tree = [ + { path: 'package.json', mode: '100644', type: 'blob', size: 40 }, + { path: 'package-lock.json', mode: '100644', type: 'blob', size: inputLock.length }, + { path: 'src', mode: '040000', type: 'tree' }, + { path: 'src/index.js', mode: '100644', type: 'blob', size: 10 }, + { path: 'node_modules/left-pad/package-lock.json', mode: '100644', type: 'blob', size: 10 }, + { path: 'test/fixtures/app/package-lock.json', mode: '100644', type: 'blob', size: 10 }, +] + +const files = { + 'package-lock.json': inputLock, + 'package.json': Buffer.from('{"name":"consumer","version":"1.0.0"}\n'), +} + +function streamSelection(session, selection, chunkSize = 7) { + for (const file of [...selection.fetchText, ...selection.fetchBinary]) { + const bytes = files[file] + if (bytes === undefined) { + session.markPresent(file, 'present') + continue + } + for (let offset = 0; offset < bytes.length; offset += chunkSize) { + session.pushChunk(file, bytes.subarray(offset, offset + chunkSize)) + } + session.endFile(file) + } + for (const file of selection.presentOnly) { + session.markPresent(file, 'present') + } + for (const file of selection.symlinks) { + session.markPresent(file, 'symlink') + } +} + +test('engineVersion and hostedScanCandidateFiles', () => { + assert.match(addon.engineVersion(), /^\d+\.\d+\.\d+\+.+$/) + const candidates = addon.hostedScanCandidateFiles() + assert.ok(Array.isArray(candidates)) + assert.ok(candidates.includes('package-lock.json')) +}) + +test('selectHostedScanPaths picks the root lockfile and ignores vendored trees', () => { + const selection = addon.selectHostedScanPaths(tree) + assert.deepEqual(selection.roots, ['']) + assert.ok(selection.fetchText.includes('package-lock.json')) + assert.ok(!selection.fetchText.includes('node_modules/left-pad/package-lock.json')) + assert.ok(!selection.fetchText.includes('test/fixtures/app/package-lock.json')) + assert.deepEqual(selection.fetchBinary, []) + assert.equal(typeof selection.ignoredCount, 'number') + assert.ok(selection.ignoredSample.length <= 100) +}) + +test('streamed session redirects the package-lock fixture', async () => { + const { provider, calls } = fakeProvider() + const selection = addon.selectHostedScanPaths(tree, { ecosystems: ['npm'] }) + const session = new addon.HostedScanSession( + { orgSlug: 'test-org', ecosystems: ['npm'] }, + provider, + ) + streamSelection(session, selection) + const result = await session.finish() + + assert.equal(result.projects.length, 1) + const [project] = result.projects + assert.equal(project.root, '') + assert.equal(project.error, undefined) + assert.deepEqual(project.redirected, [ + { purl: 'pkg:npm/left-pad@1.3.0', uuid: '22222222-2222-2222-2222-222222222222' }, + ]) + assert.equal(typeof project.redirect, 'object') + + const lock = result.changedFiles.find((f) => f.path === 'package-lock.json') + assert.ok(lock, 'package-lock.json changed') + assert.equal(lock.content, expectedLock) + const paths = result.changedFiles.map((f) => f.path) + assert.deepEqual(paths, [...paths].sort()) + assert.ok( + paths.some((p) => p.startsWith('.socket/')), + `a wet run writes the ledger (changed: ${paths.join(', ')})`, + ) + assert.deepEqual(result.changedBinaryFiles, []) + assert.deepEqual(result.deletedFiles, []) + assert.equal(result.engineVersion, addon.engineVersion()) + assert.equal(result.stats.patchesRedirected, 1) + assert.equal(calls.searchPatchesBatch, 1) + assert.equal(calls.fetchRegistryReferences, 1) + assert.ok(calls.fetchPatch >= 1) +}) + +test('dry run previews the lockfile without a ledger or patch fetches', async () => { + const { provider, calls } = fakeProvider() + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession({ orgSlug: 'test-org', dryRun: true }, provider) + streamSelection(session, selection, 4096) + const result = await session.finish() + const paths = result.changedFiles.map((f) => f.path) + assert.ok(paths.includes('package-lock.json')) + assert.ok(!paths.some((p) => p.startsWith('.socket/'))) + assert.equal(calls.fetchPatch, 0) +}) + +test('provider failures become project errors, never rejections', async () => { + for (const searchPatchesBatch of [ + async () => ({ ok: false, error: { kind: 'unauthorized', message: 'token revoked' } }), + async () => { + throw new Error('database unavailable') + }, + () => { + throw new Error('synchronous provider bug') + }, + async () => ({ nonsense: true }), + ]) { + const { provider } = fakeProvider({ searchPatchesBatch }) + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + streamSelection(session, selection) + const result = await session.finish() + assert.equal(result.projects.length, 1) + assert.equal(result.projects[0].error?.code, 'patch_lookup_failed') + assert.deepEqual(result.changedFiles, []) + } +}) + +test('cancel rejects a running finish with code cancelled', async () => { + let entered + const reached = new Promise((resolve) => { + entered = resolve + }) + const { provider } = fakeProvider({ + searchPatchesBatch: () => { + entered() + return new Promise(() => {}) + }, + }) + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + streamSelection(session, selection) + const pending = session.finish() + await reached + session.cancel() + await assert.rejects(pending, (error) => { + assert.equal(error.code, 'cancelled') + assert.equal(error.kind, 'cancelled') + return true + }) + await assert.rejects(session.finish(), (error) => error.code === 'cancelled') +}) + +test('a provider call that never settles times out into a project error', async () => { + const { provider } = fakeProvider({ searchPatchesBatch: () => new Promise(() => {}) }) + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession( + { orgSlug: 'test-org', requestTimeoutMs: 50 }, + provider, + ) + streamSelection(session, selection) + const result = await session.finish() + assert.equal(result.projects[0].error?.code, 'patch_lookup_failed') +}) + +test('cancel before finish rejects and blocks further input', async () => { + const { provider, calls } = fakeProvider() + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + session.cancel() + assert.throws( + () => session.pushChunk('package-lock.json', inputLock), + (error) => error.code === 'cancelled', + ) + await assert.rejects(session.finish(), (error) => error.code === 'cancelled') + assert.equal(calls.searchPatchesBatch, 0) +}) + +test('cancel before finish frees the buffered input right away', async () => { + const { provider } = fakeProvider() + // One whole-file chunk each: one allocation per file, which every + // allocator returns to the OS on free, so RSS reflects what is retained. + const fileMiB = 16 + const chunk = Buffer.alloc(fileMiB * 1024 * 1024, 0x61) + const roots = ['a', 'b', 'c'] + const sessions = [] + const before = process.memoryUsage().rss + for (let i = 0; i < 10; i += 1) { + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + for (const root of roots) { + session.pushChunk(`${root}/package-lock.json`, chunk) + } + session.cancel() + sessions.push(session) + } + const grownMiB = (process.memoryUsage().rss - before) / (1024 * 1024) + const bufferedMiB = sessions.length * roots.length * fileMiB + assert.ok( + grownMiB < bufferedMiB / 3, + `RSS grew ${grownMiB.toFixed(0)} MiB while ${bufferedMiB} MiB was buffered and cancelled`, + ) + for (const session of sessions) { + await assert.rejects(session.finish(), (error) => error.code === 'cancelled') + } +}) + +function bigLock(count) { + const packages = { '': { name: 'big', version: '1.0.0', dependencies: {} } } + for (let i = 0; i < count; i += 1) { + const name = `pkg-${i}` + packages[''].dependencies[name] = '1.0.0' + packages[`node_modules/${name}`] = { + version: '1.0.0', + resolved: `https://registry.npmjs.org/${name}/-/${name}-1.0.0.tgz`, + integrity: `sha512-${'A'.repeat(86)}==`, + } + } + return Buffer.from( + JSON.stringify({ name: 'big', version: '1.0.0', lockfileVersion: 3, requires: true, packages }), + ) +} + +test('cancel settles finish without waiting out a synchronous engine phase', async () => { + const lock = bigLock(60000) + const roots = ['a', 'b', 'c'] + const { provider } = fakeProvider({ + searchPatchesBatch: async () => ({ + ok: true, + value: { packages: [], canAccessPaidPatches: false }, + }), + }) + const start = () => { + const session = new addon.HostedScanSession( + { orgSlug: 'test-org', dryRun: true, projectRoots: roots, limits: { maxPurls: 200000 } }, + provider, + ) + for (const root of roots) { + session.pushChunk(`${root}/package-lock.json`, lock) + session.endFile(`${root}/package-lock.json`) + session.markPresent(`${root}/package.json`, 'present') + } + return session + } + + const baselineStart = performance.now() + const baseline = await start().finish() + const baselineMs = performance.now() - baselineStart + assert.equal(baseline.projects.length, roots.length) + const inventoryMs = baseline.stats.phaseMs.inventory + if (inventoryMs < 150) { + return + } + + const session = start() + const pending = session.finish() + await new Promise((resolve) => setTimeout(resolve, 20)) + const cancelledAt = performance.now() + session.cancel() + await assert.rejects(pending, (error) => error.code === 'cancelled') + const latencyMs = performance.now() - cancelledAt + assert.ok( + latencyMs < inventoryMs / 3, + `finish() settled ${latencyMs.toFixed(0)} ms after cancel (inventory ${inventoryMs} ms, run ${baselineMs.toFixed(0)} ms)`, + ) +}) + +test('not_found failures map per method like the HTTP API 404', async () => { + const notFound = async () => ({ ok: false, error: { kind: 'not_found', message: 'missing' } }) + const run = async (overrides) => { + const { provider, calls } = fakeProvider(overrides) + const selection = addon.selectHostedScanPaths(tree) + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + streamSelection(session, selection) + const result = await session.finish() + assert.equal(result.projects.length, 1) + return { project: result.projects[0], result, calls } + } + + const byPackage = await run({ searchPatchesByPackage: notFound }) + assert.equal(byPackage.calls.searchPatchesByPackage, 1) + assert.equal(byPackage.project.error, undefined) + assert.deepEqual(byPackage.project.redirected, []) + + const references = await run({ fetchRegistryReferences: notFound }) + assert.equal(references.calls.fetchRegistryReferences, 1) + assert.equal(references.project.error, undefined) + assert.deepEqual(references.project.redirected, []) + + const patch = await run({ fetchPatch: notFound }) + assert.ok(patch.calls.fetchPatch >= 1) + assert.equal(patch.project.error, undefined) + + const batch = await run({ searchPatchesBatch: notFound }) + assert.equal(batch.project.error?.code, 'patch_lookup_failed') + assert.match(batch.project.error.message, /not_found/) + assert.deepEqual(batch.result.changedFiles, []) +}) + +test('limits are enforced while streaming and poison the session', async () => { + const { provider } = fakeProvider() + const session = new addon.HostedScanSession( + { orgSlug: 'test-org', limits: { maxFileBytes: 16 } }, + provider, + ) + session.pushChunk('package-lock.json', inputLock.subarray(0, 10)) + assert.throws( + () => session.pushChunk('package-lock.json', inputLock.subarray(10, 30)), + (error) => error.code === 'max_file_bytes' && error.kind === 'limit', + ) + assert.throws( + () => session.endFile('package-lock.json'), + (error) => error.code === 'max_file_bytes', + ) + await assert.rejects(session.finish(), (error) => error.code === 'max_file_bytes') +}) + +test('invalid options and inputs throw typed errors', () => { + const { provider } = fakeProvider() + assert.throws( + () => new addon.HostedScanSession({ orgSlug: 'test-org', batchSize: 0 }, provider), + (error) => error.code === 'invalid_batch_size' && error.kind === 'invalid_input', + ) + assert.throws( + () => new addon.HostedScanSession({ orgSlug: 'test-org' }, null), + TypeError, + ) + const session = new addon.HostedScanSession({ orgSlug: 'test-org' }, provider) + assert.throws( + () => session.markPresent('.pnp.cjs', 'bogus'), + (error) => error.code === 'invalid_mark_kind', + ) +}) diff --git a/crates/socket-patch-node/src/lib.rs b/crates/socket-patch-node/src/lib.rs new file mode 100644 index 000000000..5f1b33fea --- /dev/null +++ b/crates/socket-patch-node/src/lib.rs @@ -0,0 +1,330 @@ +//! Node addon for the in-memory hosted redirect engine +//! (`socket_patch_cli::hosted_memory`). +//! +//! This is the private native half of `@socketsecurity/socket-patch-node`; +//! npm/index.js is the public surface (npm/index.d.ts). Options, tree +//! entries and results cross as JSON strings so the engine's serde types +//! are the single definition of every shape; bytes cross as `Buffer`s. + +mod provider; + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use napi::bindgen_prelude::{Buffer, Function, JsObjectValue, Object, PromiseRaw}; +use napi::{Env, Status}; +use napi_derive::napi; +use socket_patch_cli::hosted_memory::{ + self, EngineError, HostedScanOptions, HostedScanOutput, PresentKind, SelectOptions, + SessionBuilder, TreeEntryInput, +}; +use socket_patch_core::api::client::PatchApi; +use tokio_util::sync::CancellationToken; + +use provider::{JsPatchApi, ProviderRefs}; + +/// Every error the addon throws carries `{code, kind, message}` as a JSON +/// reason; the loader rethrows it as an `Error` with those properties. +fn js_error(code: &str, kind: &str, message: impl Into) -> napi::Error { + let reason = serde_json::json!({ "code": code, "kind": kind, "message": message.into() }); + napi::Error::new(Status::GenericFailure, reason.to_string()) +} + +fn engine_js_error(error: &EngineError) -> napi::Error { + js_error(error.code(), error.kind(), error.to_string()) +} + +fn invalid_input(code: &str, message: impl Into) -> napi::Error { + js_error(code, "invalid_input", message) +} + +fn napi_js_error(error: napi::Error) -> napi::Error { + js_error("addon_internal", "internal", error.reason.clone()) +} + +#[napi(js_name = "selectHostedScanPathsJson")] +pub fn select_hosted_scan_paths_json( + entries_json: String, + options_json: Option, +) -> napi::Result { + let entries: Vec = serde_json::from_str(&entries_json) + .map_err(|e| invalid_input("invalid_entries", format!("tree entries: {e}")))?; + let options: SelectOptions = match options_json.as_deref() { + None => SelectOptions::default(), + Some(text) => serde_json::from_str(text) + .map_err(|e| invalid_input("invalid_options", format!("selection options: {e}")))?, + }; + let selection = hosted_memory::select_paths(&entries, &options); + serde_json::to_string(&selection) + .map_err(|e| js_error("addon_internal", "internal", e.to_string())) +} + +#[napi(js_name = "hostedScanCandidateFiles")] +pub fn hosted_scan_candidate_files() -> Vec { + hosted_memory::candidate_files() +} + +#[napi(js_name = "engineVersion")] +pub fn engine_version() -> String { + hosted_memory::engine_version() +} + +#[napi(object)] +pub struct NativeBinaryFile { + pub path: String, + pub content: Buffer, +} + +/// `finish()`'s settlement. The loader turns a failure into a rejected +/// promise whose error carries `code` and `kind`, so the native promise +/// itself always resolves. +#[napi(object)] +pub struct NativeFinishOutcome { + pub ok: bool, + /// `HostedScanResult` with `changedBinaryFiles` left empty. + pub result_json: Option, + pub binary_files: Option>, + pub error_code: Option, + pub error_kind: Option, + pub error_message: Option, +} + +impl NativeFinishOutcome { + fn failure(code: &str, kind: &str, message: impl Into) -> Self { + Self { + ok: false, + result_json: None, + binary_files: None, + error_code: Some(code.to_string()), + error_kind: Some(kind.to_string()), + error_message: Some(message.into()), + } + } + + fn engine_failure(error: &EngineError) -> Self { + Self::failure(error.code(), error.kind(), error.to_string()) + } + + fn success(mut output: HostedScanOutput) -> Self { + let binaries = std::mem::take(&mut output.changed_binary_files); + match serde_json::to_string(&output) { + Ok(json) => Self { + ok: true, + result_json: Some(json), + binary_files: Some( + binaries + .into_iter() + .map(|file| NativeBinaryFile { + path: file.path, + content: Buffer::from(file.content), + }) + .collect(), + ), + error_code: None, + error_kind: None, + error_message: None, + }, + Err(e) => Self::failure( + "engine_internal", + "internal", + format!("serializing the result failed: {e}"), + ), + } + } +} + +enum SessionState { + Open(Box), + Failed(EngineError), + Finished, +} + +type OutcomeFuture = Pin + Send>>; + +async fn run_engine( + input: hosted_memory::HostedScanInput, + provider: Arc, + cancel: CancellationToken, +) -> NativeFinishOutcome { + let engine_cancel = cancel.child_token(); + let _stop_engine_if_dropped = engine_cancel.clone().drop_guard(); + let mut task = tokio::spawn(hosted_memory::run_in_memory(input, provider, engine_cancel)); + // The engine only observes cancellation between synchronous phases, so + // racing the join settles finish() without waiting out a long parse. + let joined = tokio::select! { + biased; + _ = cancel.cancelled() => { + return NativeFinishOutcome::engine_failure(&EngineError::Cancelled); + } + joined = &mut task => joined, + }; + match joined { + Ok(Ok(output)) => NativeFinishOutcome::success(output), + Ok(Err(error)) => NativeFinishOutcome::engine_failure(&error), + Err(join) if join.is_panic() => NativeFinishOutcome::failure( + "engine_panic", + "internal", + "the hosted scan engine panicked", + ), + Err(_) => NativeFinishOutcome::failure( + "engine_internal", + "internal", + "the hosted scan engine task was aborted", + ), + } +} + +fn ready(outcome: NativeFinishOutcome) -> OutcomeFuture { + Box::pin(async move { outcome }) +} + +#[napi] +pub struct NativeHostedScanSession { + state: SessionState, + provider: Option, + cancel: CancellationToken, +} + +fn provider_function<'a, Return>( + provider: &Object<'a>, + name: &str, +) -> napi::Result> +where + Return: napi::bindgen_prelude::FromNapiValue, +{ + provider + .get_named_property::>(name) + .map_err(|_| { + invalid_input( + "invalid_provider", + format!("provider.{name} must be a function"), + ) + }) +} + +#[napi] +impl NativeHostedScanSession { + #[napi(constructor)] + pub fn new(options_json: String, provider: Object<'_>) -> napi::Result { + let options: HostedScanOptions = serde_json::from_str(&options_json) + .map_err(|e| invalid_input("invalid_options", format!("session options: {e}")))?; + let builder = SessionBuilder::new(options).map_err(|e| engine_js_error(&e))?; + let refs = ProviderRefs { + search_patches_batch: provider_function(&provider, "searchPatchesBatch")? + .create_ref() + .map_err(napi_js_error)?, + search_patches_by_package: provider_function(&provider, "searchPatchesByPackage")? + .create_ref() + .map_err(napi_js_error)?, + fetch_registry_references: provider_function(&provider, "fetchRegistryReferences")? + .create_ref() + .map_err(napi_js_error)?, + fetch_patch: provider_function(&provider, "fetchPatch")? + .create_ref() + .map_err(napi_js_error)?, + download_artifact: provider_function(&provider, "downloadArtifact")? + .create_ref() + .map_err(napi_js_error)?, + }; + Ok(Self { + state: SessionState::Open(Box::new(builder)), + provider: Some(refs), + cancel: CancellationToken::new(), + }) + } + + fn with_builder( + &mut self, + step: impl FnOnce(&mut SessionBuilder) -> Result<(), EngineError>, + ) -> napi::Result<()> { + if self.cancel.is_cancelled() { + return Err(engine_js_error(&EngineError::Cancelled)); + } + let builder = match &mut self.state { + SessionState::Open(builder) => builder, + SessionState::Failed(error) => return Err(engine_js_error(error)), + SessionState::Finished => { + return Err(invalid_input( + "session_finished", + "the session has already finished", + )) + } + }; + match step(builder) { + Ok(()) => Ok(()), + Err(error) => { + let thrown = engine_js_error(&error); + self.state = SessionState::Failed(error); + self.provider = None; + Err(thrown) + } + } + } + + #[napi] + pub fn push_chunk(&mut self, path: String, chunk: Buffer) -> napi::Result<()> { + self.with_builder(|builder| builder.push_chunk(&path, &chunk)) + } + + #[napi] + pub fn end_file(&mut self, path: String) -> napi::Result<()> { + self.with_builder(|builder| builder.end_file(&path)) + } + + #[napi] + pub fn mark_present(&mut self, path: String, kind: String) -> napi::Result<()> { + let Some(mark) = PresentKind::parse(&kind) else { + return Err(invalid_input( + "invalid_mark_kind", + format!("`{kind}` is not a markPresent kind"), + )); + }; + self.with_builder(|builder| builder.mark_present(&path, mark)) + } + + #[napi] + pub fn finish<'env>( + &mut self, + env: &'env Env, + ) -> napi::Result> { + let state = std::mem::replace(&mut self.state, SessionState::Finished); + let refs = self.provider.take(); + let outcome: OutcomeFuture = match state { + _ if self.cancel.is_cancelled() => { + ready(NativeFinishOutcome::engine_failure(&EngineError::Cancelled)) + } + SessionState::Finished => ready(NativeFinishOutcome::failure( + "session_finished", + "invalid_input", + "the session has already finished", + )), + SessionState::Failed(error) => ready(NativeFinishOutcome::engine_failure(&error)), + SessionState::Open(builder) => match (builder.finish(), refs) { + (Err(error), _) => ready(NativeFinishOutcome::engine_failure(&error)), + (Ok(_), None) => ready(NativeFinishOutcome::failure( + "engine_internal", + "internal", + "the session has no provider", + )), + (Ok(input), Some(refs)) => { + let api = JsPatchApi::new(env, &refs).map_err(napi_js_error)?; + let provider: Arc = Arc::new(api); + Box::pin(run_engine(input, provider, self.cancel.clone())) + } + }, + }; + env.spawn_future(async move { Ok(outcome.await) }) + } + + #[napi] + pub fn cancel(&mut self) { + self.cancel.cancel(); + // Buffered chunks are native memory V8 does not see, so waiting for + // the wrapper's finalizer could hold up to maxTotalBytes per session. + if matches!(self.state, SessionState::Open(_)) { + self.state = SessionState::Failed(EngineError::Cancelled); + } + self.provider = None; + } +} diff --git a/crates/socket-patch-node/src/provider.rs b/crates/socket-patch-node/src/provider.rs new file mode 100644 index 000000000..13b132ce7 --- /dev/null +++ b/crates/socket-patch-node/src/provider.rs @@ -0,0 +1,280 @@ +//! [`PatchApi`] served by the host's JavaScript `PatchProvider`. +//! +//! Each provider method crosses as a threadsafe function. The JSON methods +//! take the api-v0 request body as a JSON string and resolve with a +//! `ProviderResult` envelope as a JSON string; `downloadArtifact` resolves +//! with a [`NativeDownload`] so the bytes stay a `Buffer`. The loader +//! (npm/index.js) wraps every provider method so these promises always +//! resolve: a rejected promise would carry a reference to a JS value into +//! Rust, which is exactly what the envelopes avoid. + +use std::collections::HashMap; + +use napi::bindgen_prelude::{Buffer, FunctionRef, Promise}; +use napi::threadsafe_function::ThreadsafeFunction; +use napi::{Env, Status}; +use napi_derive::napi; +use serde::de::DeserializeOwned; +use serde::Deserialize; +use socket_patch_core::api::client::{sort_batch_response, ApiError, ApiFuture, PatchApi}; +use socket_patch_core::api::types::{ + BatchSearchResponse, PackageVendorResult, PatchResponse, SearchResponse, +}; + +type JsonCall = ThreadsafeFunction, String, Status, false>; +type DownloadCall = ThreadsafeFunction, String, Status, false>; + +pub(crate) type JsonRef = FunctionRef>; +pub(crate) type DownloadRef = FunctionRef>; + +/// `downloadArtifact`'s reply as the loader flattens it. +#[napi(object)] +pub struct NativeDownload { + pub ok: bool, + pub value: Option, + pub kind: Option, + pub message: Option, +} + +/// The provider's functions, held on the JS thread until `finish()` turns +/// them into threadsafe functions. +pub(crate) struct ProviderRefs { + pub(crate) search_patches_batch: JsonRef, + pub(crate) search_patches_by_package: JsonRef, + pub(crate) fetch_registry_references: JsonRef, + pub(crate) fetch_patch: JsonRef, + pub(crate) download_artifact: DownloadRef, +} + +pub(crate) struct JsPatchApi { + search_patches_batch: JsonCall, + search_patches_by_package: JsonCall, + fetch_registry_references: JsonCall, + fetch_patch: JsonCall, + download_artifact: DownloadCall, +} + +fn json_call(env: &Env, function: &JsonRef) -> napi::Result { + function + .borrow_back(env)? + .build_threadsafe_function::() + .callee_handled::() + .build() +} + +impl JsPatchApi { + /// Must run on the JS thread. The threadsafe functions keep the event + /// loop alive until this value is dropped, which happens when the + /// engine future completes or is cancelled. + pub(crate) fn new(env: &Env, refs: &ProviderRefs) -> napi::Result { + Ok(Self { + search_patches_batch: json_call(env, &refs.search_patches_batch)?, + search_patches_by_package: json_call(env, &refs.search_patches_by_package)?, + fetch_registry_references: json_call(env, &refs.fetch_registry_references)?, + fetch_patch: json_call(env, &refs.fetch_patch)?, + download_artifact: refs + .download_artifact + .borrow_back(env)? + .build_threadsafe_function::() + .callee_handled::() + .build()?, + }) + } +} + +#[derive(Deserialize)] +struct ProviderFailure { + #[serde(default)] + kind: String, + #[serde(default)] + message: String, +} + +#[derive(Deserialize)] +struct Envelope { + ok: bool, + #[serde(default = "none")] + value: Option, + #[serde(default)] + error: Option, +} + +fn none() -> Option { + None +} + +#[derive(Deserialize)] +struct ReferencesBody { + #[serde(default)] + results: HashMap, +} + +fn provider_error(method: &str, kind: &str, message: &str) -> ApiError { + let detail = if message.is_empty() { + format!("patch provider {method} failed ({kind})") + } else { + format!("patch provider {method} failed ({kind}): {message}") + }; + match kind { + "unauthorized" => ApiError::Unauthorized(detail), + "forbidden" => ApiError::Forbidden(detail), + "rate_limited" => ApiError::RateLimited(detail), + "network" => ApiError::Network(detail), + "parse" => ApiError::Parse(detail), + _ => ApiError::Other(detail), + } +} + +fn transport_error(method: &str, error: &napi::Error) -> ApiError { + ApiError::Network(format!( + "patch provider {method} unavailable: {}", + error.reason + )) +} + +enum Reply { + /// A successful reply; `None` when its value is `null` (a missing patch). + Value(Option), + /// A `not_found` failure, which each method maps the way the HTTP client + /// maps that route's 404. + NotFound(String), +} + +async fn call_json( + method: &str, + function: &JsonCall, + request: String, +) -> Result, ApiError> { + let promise = function + .call_async_catch(request) + .await + .map_err(|e| transport_error(method, &e))?; + let reply = promise.await.map_err(|e| transport_error(method, &e))?; + let envelope: Envelope = serde_json::from_str(&reply).map_err(|e| { + ApiError::Parse(format!( + "patch provider {method} returned an unusable result: {e}" + )) + })?; + if envelope.ok { + return Ok(Reply::Value(envelope.value)); + } + let failure = envelope.error.unwrap_or(ProviderFailure { + kind: "other".to_string(), + message: String::new(), + }); + if failure.kind == "not_found" { + return Ok(Reply::NotFound(failure.message)); + } + Err(provider_error(method, &failure.kind, &failure.message)) +} + +fn missing_value(method: &str) -> ApiError { + ApiError::Parse(format!( + "patch provider {method} resolved ok without a value" + )) +} + +fn request_json(value: serde_json::Value) -> String { + value.to_string() +} + +impl PatchApi for JsPatchApi { + fn uses_public_proxy(&self) -> bool { + false + } + + fn search_patches_batch<'a>( + &'a self, + purls: &'a [String], + ) -> ApiFuture<'a, BatchSearchResponse> { + Box::pin(async move { + let components: Vec = purls + .iter() + .map(|purl| serde_json::json!({ "purl": purl })) + .collect(); + let request = request_json(serde_json::json!({ "components": components })); + let method = "searchPatchesBatch"; + // Like the HTTP collection-route 404: "no patches" is an empty + // success, so a miss is a misconfiguration, not zero patches. + let mut response: BatchSearchResponse = + match call_json(method, &self.search_patches_batch, request).await? { + Reply::Value(value) => value.ok_or_else(|| missing_value(method))?, + Reply::NotFound(message) => { + return Err(provider_error(method, "not_found", &message)) + } + }; + sort_batch_response(&mut response); + Ok(response) + }) + } + + fn search_patches_by_package<'a>(&'a self, purl: &'a str) -> ApiFuture<'a, SearchResponse> { + Box::pin(async move { + let method = "searchPatchesByPackage"; + let request = request_json(serde_json::json!({ "purl": purl })); + match call_json(method, &self.search_patches_by_package, request).await? { + Reply::Value(value) => value.ok_or_else(|| missing_value(method)), + Reply::NotFound(_) => Ok(SearchResponse { + patches: Vec::new(), + can_access_paid_patches: false, + }), + } + }) + } + + fn fetch_registry_references<'a>( + &'a self, + uuids: &'a [String], + ) -> ApiFuture<'a, HashMap> { + Box::pin(async move { + if uuids.is_empty() { + return Ok(HashMap::new()); + } + let method = "fetchRegistryReferences"; + let request = request_json(serde_json::json!({ "uuids": uuids })); + match call_json::(method, &self.fetch_registry_references, request) + .await? + { + Reply::Value(value) => Ok(value.ok_or_else(|| missing_value(method))?.results), + Reply::NotFound(_) => Ok(HashMap::new()), + } + }) + } + + fn fetch_patch<'a>(&'a self, uuid: &'a str) -> ApiFuture<'a, Option> { + Box::pin(async move { + let request = request_json(serde_json::json!({ "uuid": uuid })); + match call_json("fetchPatch", &self.fetch_patch, request).await? { + Reply::Value(value) => Ok(value), + Reply::NotFound(_) => Ok(None), + } + }) + } + + fn download_artifact<'a>(&'a self, url: &'a str, max_bytes: u64) -> ApiFuture<'a, Vec> { + Box::pin(async move { + let method = "downloadArtifact"; + let request = request_json(serde_json::json!({ "url": url, "maxBytes": max_bytes })); + let promise = self + .download_artifact + .call_async_catch(request) + .await + .map_err(|e| transport_error(method, &e))?; + let reply = promise.await.map_err(|e| transport_error(method, &e))?; + if !reply.ok { + return Err(provider_error( + method, + reply.kind.as_deref().unwrap_or("other"), + reply.message.as_deref().unwrap_or(""), + )); + } + let buffer = reply.value.ok_or_else(|| missing_value(method))?; + if buffer.len() as u64 > max_bytes { + return Err(ApiError::Other(format!( + "patch provider {method} returned more than {max_bytes} bytes" + ))); + } + Ok(buffer.to_vec()) + }) + } +} From 1f45298774c0e56907d0652a8d40a4eef683746a Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Sat, 26 Sep 2026 20:18:16 -0400 Subject: [PATCH 3/4] Use an External handle for addon sessions Export the scan session as an opaque External handle plus plain functions, not a napi class. The class wrapper's generated glue unwraps its native pointer through an FFI out-parameter, which CodeQL reports as a possible invalid pointer dereference. The public HostedScanSession in index.js is unchanged; it now calls the handle functions. Assisted-by: Claude Code:claude-opus-5-5 --- crates/socket-patch-node/npm/index.js | 21 ++++++---- crates/socket-patch-node/src/lib.rs | 57 ++++++++++++++++++++++----- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/crates/socket-patch-node/npm/index.js b/crates/socket-patch-node/npm/index.js index 381d33bbc..65d695e9e 100644 --- a/crates/socket-patch-node/npm/index.js +++ b/crates/socket-patch-node/npm/index.js @@ -24,7 +24,12 @@ const REQUIRED_EXPORTS = [ 'selectHostedScanPathsJson', 'hostedScanCandidateFiles', 'engineVersion', - 'NativeHostedScanSession', + 'createHostedScanSession', + 'hostedScanSessionPushChunk', + 'hostedScanSessionEndFile', + 'hostedScanSessionMarkPresent', + 'hostedScanSessionFinish', + 'hostedScanSessionCancel', ] class SocketPatchAddonUnavailableError extends Error { @@ -293,6 +298,7 @@ function engineVersion() { } class HostedScanSession { + #binding #native constructor(options, provider) { @@ -302,7 +308,8 @@ class HostedScanSession { const native = binding() const wrapped = wrapProvider(provider) const optionsJson = JSON.stringify(options) - this.#native = callNative(() => new native.NativeHostedScanSession(optionsJson, wrapped)) + this.#binding = native + this.#native = callNative(() => native.createHostedScanSession(optionsJson, wrapped)) } pushChunk(path, chunk) { @@ -310,22 +317,22 @@ class HostedScanSession { if (!bytes) { throw new TypeError('chunk must be a Buffer or Uint8Array') } - callNative(() => this.#native.pushChunk(requirePath(path), bytes)) + callNative(() => this.#binding.hostedScanSessionPushChunk(this.#native, requirePath(path), bytes)) } endFile(path) { - callNative(() => this.#native.endFile(requirePath(path))) + callNative(() => this.#binding.hostedScanSessionEndFile(this.#native, requirePath(path))) } markPresent(path, kind) { if (typeof kind !== 'string') { throw new TypeError('kind must be a string') } - callNative(() => this.#native.markPresent(requirePath(path), kind)) + callNative(() => this.#binding.hostedScanSessionMarkPresent(this.#native, requirePath(path), kind)) } async finish() { - const outcome = await callNative(() => this.#native.finish()) + const outcome = await callNative(() => this.#binding.hostedScanSessionFinish(this.#native)) if (!outcome.ok) { throw engineError( outcome.errorCode ?? 'engine_internal', @@ -342,7 +349,7 @@ class HostedScanSession { } cancel() { - callNative(() => this.#native.cancel()) + callNative(() => this.#binding.hostedScanSessionCancel(this.#native)) } } diff --git a/crates/socket-patch-node/src/lib.rs b/crates/socket-patch-node/src/lib.rs index 5f1b33fea..5f1c8ed7e 100644 --- a/crates/socket-patch-node/src/lib.rs +++ b/crates/socket-patch-node/src/lib.rs @@ -12,7 +12,7 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use napi::bindgen_prelude::{Buffer, Function, JsObjectValue, Object, PromiseRaw}; +use napi::bindgen_prelude::{Buffer, External, Function, JsObjectValue, Object, PromiseRaw}; use napi::{Env, Status}; use napi_derive::napi; use socket_patch_cli::hosted_memory::{ @@ -179,7 +179,6 @@ fn ready(outcome: NativeFinishOutcome) -> OutcomeFuture { Box::pin(async move { outcome }) } -#[napi] pub struct NativeHostedScanSession { state: SessionState, provider: Option, @@ -203,9 +202,7 @@ where }) } -#[napi] impl NativeHostedScanSession { - #[napi(constructor)] pub fn new(options_json: String, provider: Object<'_>) -> napi::Result { let options: HostedScanOptions = serde_json::from_str(&options_json) .map_err(|e| invalid_input("invalid_options", format!("session options: {e}")))?; @@ -262,17 +259,14 @@ impl NativeHostedScanSession { } } - #[napi] pub fn push_chunk(&mut self, path: String, chunk: Buffer) -> napi::Result<()> { self.with_builder(|builder| builder.push_chunk(&path, &chunk)) } - #[napi] pub fn end_file(&mut self, path: String) -> napi::Result<()> { self.with_builder(|builder| builder.end_file(&path)) } - #[napi] pub fn mark_present(&mut self, path: String, kind: String) -> napi::Result<()> { let Some(mark) = PresentKind::parse(&kind) else { return Err(invalid_input( @@ -283,7 +277,6 @@ impl NativeHostedScanSession { self.with_builder(|builder| builder.mark_present(&path, mark)) } - #[napi] pub fn finish<'env>( &mut self, env: &'env Env, @@ -317,7 +310,6 @@ impl NativeHostedScanSession { env.spawn_future(async move { Ok(outcome.await) }) } - #[napi] pub fn cancel(&mut self) { self.cancel.cancel(); // Buffered chunks are native memory V8 does not see, so waiting for @@ -328,3 +320,50 @@ impl NativeHostedScanSession { self.provider = None; } } + +#[napi(js_name = "createHostedScanSession")] +pub fn create_hosted_scan_session( + options_json: String, + provider: Object<'_>, +) -> napi::Result> { + NativeHostedScanSession::new(options_json, provider).map(External::new) +} + +#[napi(js_name = "hostedScanSessionPushChunk")] +pub fn hosted_scan_session_push_chunk( + session: &mut External, + path: String, + chunk: Buffer, +) -> napi::Result<()> { + session.push_chunk(path, chunk) +} + +#[napi(js_name = "hostedScanSessionEndFile")] +pub fn hosted_scan_session_end_file( + session: &mut External, + path: String, +) -> napi::Result<()> { + session.end_file(path) +} + +#[napi(js_name = "hostedScanSessionMarkPresent")] +pub fn hosted_scan_session_mark_present( + session: &mut External, + path: String, + kind: String, +) -> napi::Result<()> { + session.mark_present(path, kind) +} + +#[napi(js_name = "hostedScanSessionFinish")] +pub fn hosted_scan_session_finish<'env>( + env: &'env Env, + session: &mut External, +) -> napi::Result> { + session.finish(env) +} + +#[napi(js_name = "hostedScanSessionCancel")] +pub fn hosted_scan_session_cancel(session: &mut External) { + session.cancel(); +} From 5316bf8b5f3f23f622d8fe1f11fa39e543cbc961 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Sun, 27 Sep 2026 08:14:58 -0400 Subject: [PATCH 4/4] Scan excluded nested Cargo projects Keep every directory with its own Cargo.lock as a project root during path selection. Whether a nested lock belongs to an enclosing workspace depends on that workspace's members and exclude lists, which only the engine can read. Crates the workspace excludes are independent projects and are now scanned; real members still have their unused lock ignored. Point the in-memory Pipenv warning at the pipenvMajor option, the only input the engine reads, instead of PATH or an environment variable. Assisted-by: Claude Code:claude-opus-5-5 --- .../src/hosted_memory/redirect.rs | 5 +- .../src/hosted_memory/roots.rs | 29 +--- .../tests/hosted_memory_parity.rs | 133 +++++++++++++++++- 3 files changed, 139 insertions(+), 28 deletions(-) diff --git a/crates/socket-patch-cli/src/hosted_memory/redirect.rs b/crates/socket-patch-cli/src/hosted_memory/redirect.rs index 4b3849455..e514be09e 100644 --- a/crates/socket-patch-cli/src/hosted_memory/redirect.rs +++ b/crates/socket-patch-cli/src/hosted_memory/redirect.rs @@ -634,10 +634,7 @@ pub(crate) fn rewrite( if targets_pipenv_lock && pipenv_major.is_none() && rewrite.files.contains_key("Pipfile.lock") { rewrite.warnings.push(RewriteWarning { code: "redirect_pipenv_installer_unknown".into(), - detail: format!( - "Pipenv was not found on PATH, so the Pipfile.lock references use the modern `file` form (Pipenv 2018 and later). A project installed with Pipenv 7–11 needs `path` references instead: put that pipenv on PATH or set {}= and re-run `scan --mode hosted`.", - socket_patch_core::utils::pipenv::MAJOR_OVERRIDE_ENV - ), + detail: "The scan did not set `pipenvMajor`, so the Pipfile.lock references use the modern `file` form (Pipenv 2018 and later). A project installed with Pipenv 7–11 needs `path` references instead: re-run the scan with `pipenvMajor` set to that Pipenv major version.".into(), }); } diff --git a/crates/socket-patch-cli/src/hosted_memory/roots.rs b/crates/socket-patch-cli/src/hosted_memory/roots.rs index a2a48fd23..da5a0ad85 100644 --- a/crates/socket-patch-cli/src/hosted_memory/roots.rs +++ b/crates/socket-patch-cli/src/hosted_memory/roots.rs @@ -1,7 +1,9 @@ //! Project-root detection over a repository path list. A root is a //! directory holding a root LOCK marker (manifests alone never make one), //! outside vendored / test-fixture trees, and not an internal directory of -//! an enclosing Rush monorepo or Cargo workspace. +//! an enclosing Rush monorepo. A nested Cargo.lock stays a root here: only +//! the enclosing workspace's `members`/`exclude` can say whether it is a +//! member, so the engine demotes members once manifests are readable. use std::collections::{BTreeMap, BTreeSet}; @@ -96,13 +98,6 @@ pub(crate) fn join_root(root: &str, rel: &str) -> String { } } -fn is_proper_ancestor(ancestor: &str, dir: &str) -> bool { - if ancestor == dir { - return false; - } - ancestor.is_empty() || dir.starts_with(&format!("{ancestor}/")) -} - fn allowed(ecosystems: Option<&[String]>, eco: &str) -> bool { ecosystems.is_none_or(|list| list.iter().any(|e| e == eco)) } @@ -153,14 +148,8 @@ pub(crate) fn detect_roots<'a>( .filter(|(_, m)| m.contains("rush.json")) .map(|(d, _)| d.clone()) .collect(); - let cargo_lock_dirs: Vec = markers - .iter() - .filter(|(_, m)| m.contains("Cargo.lock")) - .map(|(d, _)| d.clone()) - .collect(); - let mut roots: Vec = Vec::new(); - for (dir, set) in &markers { + for dir in markers.keys() { let rush_internal = rush_roots.iter().any(|r| { let internal = |sub: &str| join_root(r, sub); *dir == internal("common/config/rush") @@ -168,14 +157,8 @@ pub(crate) fn detect_roots<'a>( || *dir == internal("common/temp") || dir.starts_with(&format!("{}/", internal("common/temp"))) }); - let mut remaining: BTreeSet<&str> = set.clone(); - if cargo_lock_dirs.iter().any(|a| is_proper_ancestor(a, dir)) { - remaining.remove("Cargo.lock"); - } let reason = if rush_internal { Some("rush_internal") - } else if remaining.is_empty() { - Some("cargo_member") } else { None }; @@ -237,7 +220,7 @@ mod tests { } #[test] - fn rush_and_cargo_internals_are_not_roots() { + fn rush_internals_are_not_roots_but_nested_cargo_locks_are() { assert_eq!( roots(&[ "rush.json", @@ -248,7 +231,7 @@ mod tests { "ws/crates/b/Cargo.lock", "ws/crates/b/package-lock.json", ]), - vec!["", "ws", "ws/crates/b"] + vec!["", "ws", "ws/crates/a", "ws/crates/b"] ); } diff --git a/crates/socket-patch-cli/tests/hosted_memory_parity.rs b/crates/socket-patch-cli/tests/hosted_memory_parity.rs index 14219cc0b..5876fe19d 100644 --- a/crates/socket-patch-cli/tests/hosted_memory_parity.rs +++ b/crates/socket-patch-cli/tests/hosted_memory_parity.rs @@ -225,7 +225,8 @@ async fn assert_native_parity( let project = &memory.projects[0]; assert!(project.error.is_none(), "{:?}", project.error); assert_eq!( - project.redirect, disk.envelope["redirect"], + without_pipenv_advice(&project.redirect), + without_pipenv_advice(&disk.envelope["redirect"]), "{}", disk.stderr ); @@ -242,6 +243,21 @@ async fn assert_native_parity( } } +/// The in-memory engine cannot find Pipenv on PATH, so its +/// `redirect_pipenv_installer_unknown` advice names the `pipenvMajor` +/// option instead of the disk run's PATH/env remedy. +fn without_pipenv_advice(redirect: &Value) -> Value { + let mut redirect = redirect.clone(); + if let Some(warnings) = redirect.get_mut("warnings").and_then(Value::as_array_mut) { + for warning in warnings.iter_mut() { + if warning["code"] == "redirect_pipenv_installer_unknown" { + warning["detail"] = Value::Null; + } + } + } + redirect +} + fn read_fixture(rel: &str) -> Vec { std::fs::read(fixtures_root().join(rel)).unwrap() } @@ -614,3 +630,118 @@ async fn cargo_patch_path_into_vendor_tree_fails_closed() { project.redirect ); } + +#[tokio::test] +async fn pipfile_advice_names_the_pipenv_major_option() { + let files = BTreeMap::from([ + ( + "Pipfile.lock".to_string(), + read_fixture("pipenv/2026.8.0/Pipfile.lock"), + ), + ( + "Pipfile".to_string(), + read_fixture("pipenv/2026.8.0/Pipfile"), + ), + ]); + let lock: Value = serde_json::from_slice(&files["Pipfile.lock"]).unwrap(); + let (name, entry) = lock["default"] + .as_object() + .and_then(|m| m.iter().next()) + .expect("a default package"); + let version = entry["version"].as_str().unwrap().trim_start_matches("=="); + let overrides = serde_json::json!([{ + "ecosystem": "pypi", "name": name, "version": version, + "token": "22222222-2222-4222-8222-222222222222", + "patchUuid": "e828efa5-5c6d-43f3-9909-03f5ac232b98", + "artifactUrl": format!("https://patch.socket.dev/patch/pypi/{name}/{version}/22222222-2222-4222-8222-222222222222/e828efa5-5c6d-43f3-9909-03f5ac232b98/{name}-{version}-py3-none-any.whl"), + "integrity": {"sha256": "c".repeat(64)} + }]); + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("overrides.json"), overrides.to_string()).unwrap(); + let server = MockServer::start().await; + let patches = patches_from_overrides(&tmp.path().join("overrides.json"), Some(&server.uri())); + mount_api(&server, &patches).await; + let memory = run_engine(&server, build_input(&files, &[], options(false))).await; + let detail = memory.projects[0].redirect["warnings"] + .as_array() + .and_then(|w| { + w.iter() + .find(|x| x["code"] == "redirect_pipenv_installer_unknown") + }) + .and_then(|w| w["detail"].as_str()) + .expect("pipenv advice warning") + .to_string(); + assert!(detail.contains("`pipenvMajor`"), "{detail}"); + assert!(!detail.contains("PATH"), "{detail}"); +} + +#[tokio::test] +async fn excluded_nested_cargo_project_is_its_own_root_through_selection() { + let ws = fixtures_root().join("redirect/cargo/cargo/workspace-member"); + let standalone = fixtures_root().join("redirect/cargo/cargo/basic"); + let server = MockServer::start().await; + let mut patches = patches_from_overrides(&ws.join("overrides.json"), Some(&server.uri())); + patches.extend(patches_from_overrides( + &standalone.join("overrides.json"), + Some(&server.uri()), + )); + mount_api(&server, &patches).await; + + let mut repo = fixture_files(&ws.join("input")); + let manifest = String::from_utf8(repo["Cargo.toml"].clone()) + .unwrap() + .replacen( + "[workspace]\n", + "[workspace]\nexclude = [\"tools/fuzz\"]\n", + 1, + ); + repo.insert("Cargo.toml".into(), manifest.into_bytes()); + let stale_member_lock = repo["Cargo.lock"].clone(); + repo.insert("a/Cargo.lock".into(), stale_member_lock); + let fuzz = fixture_files(&standalone.join("input")); + for (rel, bytes) in &fuzz { + repo.insert(format!("tools/fuzz/{rel}"), bytes.clone()); + } + + let memory = run_engine(&server, selected_input(&repo)).await; + let roots: Vec<&str> = memory.projects.iter().map(|p| p.root.as_str()).collect(); + assert!(roots.contains(&"tools/fuzz"), "{roots:?}"); + let changed = engine_changed(&memory); + + let disk = run_disk(&server, &fuzz, false); + let fuzz_project = memory + .projects + .iter() + .find(|p| p.root == "tools/fuzz") + .unwrap(); + assert!(fuzz_project.error.is_none(), "{:?}", fuzz_project.error); + assert_eq!(fuzz_project.redirect, disk.envelope["redirect"]); + let fuzz_changed: BTreeMap> = changed + .iter() + .filter_map(|(k, v)| { + k.strip_prefix("tools/fuzz/") + .map(|rel| (rel.to_string(), v.clone())) + }) + .collect(); + assert_eq!(fuzz_changed, disk.changed); + assert!( + !fuzz_project.redirected.is_empty(), + "{:#}", + fuzz_project.redirect + ); + + let workspace = memory.projects.iter().find(|p| p.root.is_empty()).unwrap(); + assert!(!workspace.redirected.is_empty(), "{:#}", workspace.redirect); + assert!( + !changed.contains_key("a/Cargo.lock"), + "{}", + describe(&changed) + ); + assert!( + memory.warnings.iter().any( + |w| w.code == "cargo_member_lock_ignored" && w.project_root.as_deref() == Some("a") + ), + "{:?}", + memory.warnings + ); +}