diff --git a/CHANGELOG.md b/CHANGELOG.md index 4449a3a..f7407b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.15.0] - 2026-10-23 + +### Added + +- **`--max-components`** on `scan` and `scan wfp`, and **`scanoss.WithMaxComponents`** with + `scanoss.ComponentLimit`, choose how many components the server reports: a positive integer + (leading zeros allowed) or `all`. `ComponentsUpTo`, `AllComponents` and `ParseComponentLimit` + build a limit; an invalid value is rejected before anything is uploaded. The SDK sends it as + the `max_components` query parameter on every block of the upload. For a full-file match the + server reports the best N components of the project ranking plus up to three context picks + (folder, file alone, origin), each labelled in the match's `context`; a snippet block gets up + to N candidates; `all` is no limit. Unset sends nothing, so requests and results are + unchanged. The scanoss.api server must support the parameter; a server that does not will + ignore it. + ## [0.14.2] - 2026-10-23 ### Fixed @@ -514,7 +529,8 @@ Initial release of the SCANOSS Go CLI and SDK (`scanoss`). CycloneDX and SPDX (with `WithTool`/`WithAuthor`/`WithTimestamp` document-metadata options). - **C shared library** (`libscanoss`) with Node.js and Python bindings. -[Unreleased]: https://github.com/scanoss/scanoss.go/compare/v0.14.2...HEAD +[Unreleased]: https://github.com/scanoss/scanoss.go/compare/v0.15.0...HEAD +[0.15.0]: https://github.com/scanoss/scanoss.go/compare/v0.14.2...v0.15.0 [0.14.2]: https://github.com/scanoss/scanoss.go/compare/v0.14.1...v0.14.2 [0.14.1]: https://github.com/scanoss/scanoss.go/compare/v0.14.0...v0.14.1 [0.14.0]: https://github.com/scanoss/scanoss.go/compare/v0.13.0...v0.14.0 diff --git a/CLIENT_HELP.md b/CLIENT_HELP.md index dbf3f86..4359c98 100644 --- a/CLIENT_HELP.md +++ b/CLIENT_HELP.md @@ -162,7 +162,9 @@ Flags (persistent flags are shared with `scan wfp`): `--api-url`, `--api-key`, `--ignore-cert-errors`, `-t, --threads` (10), `--save-wfp`, `--min-size` (bytes, default 0), `--max-size` (0 = unlimited), `--gitignore` (true), `--all-extensions`, `--all-folders`, `--all-hidden`, `-i, --identify`, -`-n, --ignore`, `--ranking-threshold` (-1 = off), `--skip-headers` +`-n, --ignore`, `--ranking-threshold` (-1 = off), +`--max-components` (a positive integer or `all`; unset = the server's +defaults), `--skip-headers` (**default true**, `--skip-headers=false` to disable), `--skip-headers-limit` (0 = no limit). @@ -170,6 +172,18 @@ default 0), `--max-size` (0 = unlimited), `--gitignore` (true), [BOM rules](#bom-rules). `--skip-headers` applies while fingerprinting, so `scan wfp` rejects it — the WFP it is handed was already assembled. +`--max-components` sets how many components the server reports: a positive +integer (leading zeros allowed) or `all`. It is sent as the `max_components` +query parameter on every block of the upload, so the server must support it. +For a full-file match the server reports the best N components of the project +ranking plus up to 3 context picks (folder, file alone, origin), each labelled +in the match's `context`; a snippet block gets up to N candidates; the allowed +set widens to at least N; `all` is no limit. Left unset, nothing is sent and +the server's defaults apply (2 for full-file matches plus the context picks, 1 +per snippet block). A value that is not a positive integer or `all` is rejected +before anything is uploaded. `results` does not take it: it only polls a scan +that was already uploaded. + ### Skipping files Files are filtered before scanning (build dirs, vendored deps, generated/binary @@ -686,6 +700,7 @@ with `--settings`. It carries BOM context and file-skip rules. | `--format` | `raw` | `raw` / `spdx` / `cyclonedx` | | `--chunk-size` (scan) | `1048576` | WFP upload block size (bytes) | | `--poll-interval` (scan) | `2s` | Scan status poll cadence | +| `--max-components` (scan) | unset | Not sent; the server's default applies | | `--min-size` / `--max-size` | `0` / `0` | Literal minimum; `0` max means unlimited | | `--chunk-size` (decoration) | `10` | PURLs per request | | `--workers` (decoration) | `5` | Max concurrent requests | diff --git a/README.md b/README.md index 439d249..5083c8c 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,15 @@ result, err := client.Scan.Folder(ctx, "./my-project") // resume by id: client.Scan.Wait(ctx, scanID) ``` +To change how many components the server reports, pass a limit: a positive integer or all. +Unset sends nothing and the server's defaults apply; the server must support the +`max_components` parameter. + +```go +limit, err := scanoss.ParseComponentLimit("5") // or scanoss.ComponentsUpTo(5), scanoss.AllComponents() +result, err := client.Scan.Folder(ctx, "./my-project", scanoss.WithMaxComponents(limit)) +``` + ### Proxy and custom CA from the SDK ```go diff --git a/cmd/bomrules.go b/cmd/bomrules.go index aa6cfe5..b089f92 100644 --- a/cmd/bomrules.go +++ b/cmd/bomrules.go @@ -32,6 +32,7 @@ import ( "github.com/spf13/cobra" "github.com/scanoss/scanoss.go/pkg/sbom" + "github.com/scanoss/scanoss.go/pkg/scanoss" "github.com/scanoss/scanoss.go/pkg/settings" ) @@ -206,6 +207,28 @@ func resolveRankingThreshold(cmd *cobra.Command, s *settings.Settings) int { return threshold } +// addComponentFlag declares --max-components on the commands that upload a scan. +func addComponentFlag(cmd *cobra.Command) { + cmd.Flags().String("max-components", "", + `How many components the server reports: a positive integer or "all" (default: the server's own limit)`) +} + +// resolveComponentLimit reads --max-components. A flag left off is an unset limit, which +// sends nothing. A flag that was passed is parsed whatever it holds, so an explicit empty value is +// refused rather than read as unset. A bad value is refused here so the command fails before it +// fingerprints or uploads. +func resolveComponentLimit(cmd *cobra.Command) (scanoss.ComponentLimit, error) { + if !cmd.Flags().Changed("max-components") { + return scanoss.ComponentLimit{}, nil + } + v, _ := cmd.Flags().GetString("max-components") + l, err := scanoss.ParseComponentLimit(v) + if err != nil { + return scanoss.ComponentLimit{}, fmt.Errorf("--max-components: %w", err) + } + return l, nil +} + // addSkipHeaderFlags declares the fingerprinting header filter flags. // // The filter is on by default: a licence header is boilerplate shared by every file that carries diff --git a/cmd/components_test.go b/cmd/components_test.go new file mode 100644 index 0000000..8601e8d --- /dev/null +++ b/cmd/components_test.go @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2026, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package cmd + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +type scanRequests struct { + mu sync.Mutex + queries []url.Values // upload (POST) requests only +} + +func (s *scanRequests) server(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + s.mu.Lock() + s.queries = append(s.queries, r.URL.Query()) + s.mu.Unlock() + w.WriteHeader(http.StatusAccepted) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "status": "completed", + "result": map[string]any{"files": []any{}, "components": map[string]any{}}, + }) + })) + t.Cleanup(srv.Close) + return srv +} + +func runScanWFPCmd(t *testing.T, srvURL string, extra ...string) error { + t.Helper() + wfpPath := filepath.Join(t.TempDir(), "in.wfp") + if err := os.WriteFile(wfpPath, []byte("file=abc,10,a.c\n"), 0o644); err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "out.json") + _, _, err := runBare(t, append([]string{ + "scan", "wfp", wfpPath, "--api-url", srvURL, "--api-key", "test", "--output", out, + "--poll-interval", "10ms", + }, extra...)...) + return err +} + +// runScanDirCmd drives `scan `, which fingerprints a directory and goes through +// scanpipeline.Run rather than the `scan wfp` path. +func runScanDirCmd(t *testing.T, srvURL string, extra ...string) error { + t.Helper() + dir := t.TempDir() + src := "package main\n\nfunc main() {\n\tprintln(\"hello components\")\n}\n" + if err := os.WriteFile(filepath.Join(dir, "a.go"), []byte(src), 0o644); err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "out.json") + _, _, err := runBare(t, append([]string{ + "scan", dir, "--api-url", srvURL, "--api-key", "test", "--output", out, + "--poll-interval", "10ms", + }, extra...)...) + return err +} + +func TestScanRejectsInvalidMaxComponents(t *testing.T) { + for _, bad := range []string{"0", "-1", "2.5", "ALL", "+3", "abc"} { + t.Run(bad, func(t *testing.T) { + var rec scanRequests + srv := rec.server(t) + err := runScanWFPCmd(t, srv.URL, "--max-components="+bad) + if err == nil || !strings.Contains(err.Error(), "--max-components") { + t.Fatalf("error = %v, want one naming --max-components", err) + } + if n := len(rec.queries); n != 0 { + t.Errorf("server saw %d requests, want 0: the value must be refused before any upload", n) + } + }) + } +} + +func TestScanSendsMaxComponents(t *testing.T) { + for _, tc := range []struct { + name string + args []string + want string + }{ + {"number", []string{"--max-components", "7"}, "7"}, + {"all", []string{"--max-components", "all"}, "all"}, + {"unset", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + var rec scanRequests + srv := rec.server(t) + if err := runScanWFPCmd(t, srv.URL, tc.args...); err != nil { + t.Fatal(err) + } + if len(rec.queries) == 0 { + t.Fatal("server saw no upload") + } + for _, q := range rec.queries { + if tc.want == "" { + if len(q) != 0 { + t.Errorf("an unset flag sent a query: %v", q) + } + continue + } + if got := q["max_components"]; len(got) != 1 || got[0] != tc.want { + t.Errorf("upload query %v, want max_components=%q", q, tc.want) + } + } + }) + } +} + +func TestScanDirRejectsInvalidMaxComponents(t *testing.T) { + for _, bad := range []string{"0", "-1", "2.5", "ALL", "+3", "abc", ""} { + t.Run("value="+bad, func(t *testing.T) { + var rec scanRequests + srv := rec.server(t) + err := runScanDirCmd(t, srv.URL, "--max-components="+bad) + if err == nil || !strings.Contains(err.Error(), "--max-components") { + t.Fatalf("error = %v, want one naming --max-components", err) + } + if n := len(rec.queries); n != 0 { + t.Errorf("server saw %d requests, want 0: the value must be refused before any upload", n) + } + }) + } +} + +func TestScanDirSendsMaxComponents(t *testing.T) { + for _, tc := range []struct { + name string + args []string + want string + }{ + {"number", []string{"--max-components", "7"}, "7"}, + {"all", []string{"--max-components", "all"}, "all"}, + {"unset", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + var rec scanRequests + srv := rec.server(t) + if err := runScanDirCmd(t, srv.URL, tc.args...); err != nil { + t.Fatal(err) + } + if len(rec.queries) == 0 { + t.Fatal("server saw no upload") + } + for _, q := range rec.queries { + if tc.want == "" { + if len(q) != 0 { + t.Errorf("an unset flag sent a query: %v", q) + } + continue + } + if got := q["max_components"]; len(got) != 1 || got[0] != tc.want { + t.Errorf("upload query %v, want max_components=%q", q, tc.want) + } + } + }) + } +} + +// An explicitly empty value is a mistake (an unset shell variable, say), not an unset limit. +func TestScanWFPRejectsEmptyMaxComponents(t *testing.T) { + var rec scanRequests + srv := rec.server(t) + err := runScanWFPCmd(t, srv.URL, "--max-components", "") + if err == nil || !strings.Contains(err.Error(), "--max-components") { + t.Fatalf("error = %v, want one naming --max-components", err) + } + if n := len(rec.queries); n != 0 { + t.Errorf("server saw %d requests, want 0", n) + } +} diff --git a/cmd/scan.go b/cmd/scan.go index f8d3191..c0297a9 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -376,6 +376,8 @@ func init() { addBOMRuleFlags(scanWFPCmd) addRankingFlag(scanCmd) addRankingFlag(scanWFPCmd) + addComponentFlag(scanCmd) + addComponentFlag(scanWFPCmd) addSkipHeaderFlags(scanWFPCmd) // declared so the flag is rejected with a reason, see runScanWFP } @@ -393,6 +395,10 @@ func runScan(cmd *cobra.Command, args []string) error { if err := validateOutputFormat(cmd); err != nil { return err } + components, err := resolveComponentLimit(cmd) + if err != nil { + return err + } layers, err := scanLayers(cmd) if err != nil { return err @@ -474,7 +480,7 @@ func runScan(cmd *cobra.Command, args []string) error { Threads: threads, ScanFilters: collectOpts, DependencyFilters: depOpts, - ScanOptions: scanTuning(cmd, scanSettings, prog), + ScanOptions: scanTuning(cmd, scanSettings, prog, components), WFPOptions: fingerprintOptions(skipHeaders, skipHeadersLimit), WFPWriter: wfpWriterOrNil(wfpOut), OnProgress: prog.layer, @@ -522,6 +528,10 @@ func runScanWFP(cmd *cobra.Command, args []string) error { if err := validateOutputFormat(cmd); err != nil { return err } + components, err := resolveComponentLimit(cmd) + if err != nil { + return err + } // A bare WFP has no source tree, so declared dependencies cannot be sourced. Rejecting deps // beats warning and carrying on: the output is an inventory the caller keeps, and one missing // the layer they asked for should not leave with a success code. @@ -581,7 +591,7 @@ func runScanWFP(cmd *cobra.Command, args []string) error { rep := scanpipeline.NewReporter(prog.layer) var bom bomReport - scanOpts := append(scanTuning(cmd, scanSettings, prog), scanoss.WithScanReporter(rep), bom.option()) + scanOpts := append(scanTuning(cmd, scanSettings, prog, components), scanoss.WithScanReporter(rep), bom.option()) res, err := client.Scan.WFP(ctx, wfp, scanOpts...) if err != nil { return renderAPIError(fmt.Errorf("scan failed: %w", err)) @@ -611,9 +621,9 @@ func buildScanClient(cmd *cobra.Command) (*scanoss.Client, error) { return scanoss.New(cfg) } -// scanTuning builds the per-scan options — chunk size, poll interval, the BOM rules, the ranking +// scanTuning builds the per-scan options — chunk size, poll interval, the component limit, the BOM rules, the ranking // threshold and the scan-id notice — from the flags and settings. -func scanTuning(cmd *cobra.Command, scanSettings *settings.Settings, prog *scanProgress) []scanoss.ScanOption { +func scanTuning(cmd *cobra.Command, scanSettings *settings.Settings, prog *scanProgress, components scanoss.ComponentLimit) []scanoss.ScanOption { chunkSize, _ := cmd.Flags().GetInt("chunk-size") pollInterval, _ := cmd.Flags().GetDuration("poll-interval") if chunkSize < 1024 { @@ -624,6 +634,7 @@ func scanTuning(cmd *cobra.Command, scanSettings *settings.Settings, prog *scanP scanOpts := []scanoss.ScanOption{ scanoss.WithChunkBytes(chunkSize), scanoss.WithPollInterval(pollInterval), + scanoss.WithMaxComponents(components), scanoss.WithScanIDNotify(func(id string) { prog.writeLine("") // separate the scan-id block from the filter/skip notices above prog.writeLine(infoLine("Scan id: %s", id)) diff --git a/pkg/scanoss/componentlimit.go b/pkg/scanoss/componentlimit.go new file mode 100644 index 0000000..9e00cb7 --- /dev/null +++ b/pkg/scanoss/componentlimit.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2026, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package scanoss + +import ( + "errors" + "fmt" + "strconv" +) + +// maxComponentsParam is the scan endpoint's query parameter that caps how many components +// the server reports. +const maxComponentsParam = "max_components" + +const allComponents = "all" + +var errInvalidComponentLimit = errors.New(`must be a positive integer or "all"`) + +// ComponentLimit says how many components the server reports for a scan: a positive number, or +// all of them. The zero value is unset: the scan request does not mention the limit +// and the server applies its own default. +type ComponentLimit struct { + n int + all bool +} + +// ComponentsUpTo limits each match to at most n components. n must be at least 1. +func ComponentsUpTo(n int) (ComponentLimit, error) { + if n < 1 { + return ComponentLimit{}, fmt.Errorf("component limit %d: %w", n, errInvalidComponentLimit) + } + return ComponentLimit{n: n}, nil +} + +// AllComponents asks for every component, with no limit. +func AllComponents() ComponentLimit { return ComponentLimit{all: true} } + +// ParseComponentLimit reads the text form: exactly "all", or a base-10 positive integer with no +// sign and no surrounding space. +func ParseComponentLimit(s string) (ComponentLimit, error) { + if s == allComponents { + return AllComponents(), nil + } + for _, r := range s { + if r < '0' || r > '9' { + return ComponentLimit{}, fmt.Errorf("component limit %q: %w", s, errInvalidComponentLimit) + } + } + n, err := strconv.Atoi(s) + if err != nil { + return ComponentLimit{}, fmt.Errorf("component limit %q: %w", s, errInvalidComponentLimit) + } + return ComponentsUpTo(n) +} + +// IsSet reports whether a limit was chosen. +func (l ComponentLimit) IsSet() bool { return l.all || l.n > 0 } + +// String is the value sent to the server, "all" or the number. It is empty when the limit is +// unset. +func (l ComponentLimit) String() string { + switch { + case l.all: + return allComponents + case l.n > 0: + return strconv.Itoa(l.n) + } + return "" +} diff --git a/pkg/scanoss/componentlimit_test.go b/pkg/scanoss/componentlimit_test.go new file mode 100644 index 0000000..f99b7cf --- /dev/null +++ b/pkg/scanoss/componentlimit_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (c) 2026, SCANOSS + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package scanoss + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestParseComponentLimit(t *testing.T) { + valid := map[string]string{"1": "1", "5": "5", "007": "7", "all": "all"} + for in, want := range valid { + l, err := ParseComponentLimit(in) + if err != nil { + t.Errorf("ParseComponentLimit(%q): %v", in, err) + continue + } + if !l.IsSet() || l.String() != want { + t.Errorf("ParseComponentLimit(%q) = %q (set %v), want %q", in, l.String(), l.IsSet(), want) + } + } + + invalid := []string{"0", "-1", "2.5", "", " 3", "3 ", "ALL", "All", "+3", "1e3", "0x10", "99999999999999999999"} + for _, in := range invalid { + _, err := ParseComponentLimit(in) + if err == nil { + t.Errorf("ParseComponentLimit(%q) accepted", in) + continue + } + if want := `must be a positive integer or "all"`; !strings.Contains(err.Error(), want) { + t.Errorf("ParseComponentLimit(%q) error %q lacks %q", in, err, want) + } + } +} + +func TestComponentLimitConstructors(t *testing.T) { + var zero ComponentLimit + if zero.IsSet() || zero.String() != "" { + t.Errorf("zero value must be unset, got %q", zero.String()) + } + if _, err := ComponentsUpTo(0); err == nil { + t.Error("ComponentsUpTo(0) accepted") + } + if _, err := ComponentsUpTo(-3); err == nil { + t.Error("ComponentsUpTo(-3) accepted") + } + if l, err := ComponentsUpTo(4); err != nil || l.String() != "4" { + t.Errorf("ComponentsUpTo(4) = %q, %v", l.String(), err) + } + if AllComponents().String() != "all" { + t.Errorf("AllComponents() = %q", AllComponents().String()) + } +} + +type queryRecorder struct { + mu sync.Mutex + queries []url.Values + inner http.HandlerFunc +} + +func (q *queryRecorder) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + q.mu.Lock() + q.queries = append(q.queries, r.URL.Query()) + q.mu.Unlock() + } + q.inner(w, r) +} + +func scanWithOptions(t *testing.T, opts ...ScanOption) []url.Values { + t.Helper() + mock := &scanMock{completeAt: 1, result: json.RawMessage(`{"files":[],"components":{}}`)} + rec := &queryRecorder{inner: mock.handler()} + c := newTestClient(t, rec) + opts = append([]ScanOption{WithChunkBytes(4), WithPollInterval(time.Millisecond)}, opts...) + if _, err := c.Scan.WFP(context.Background(), []byte("0123456789abc"), opts...); err != nil { + t.Fatalf("Scan: %v", err) + } + if len(rec.queries) != 4 { + t.Fatalf("recorded %d uploads, want 4 so that \"every chunk\" is meaningful", len(rec.queries)) + } + return rec.queries +} + +func TestMaxComponentsSentOnEveryChunk(t *testing.T) { + five, err := ComponentsUpTo(5) + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + opt ScanOption + want string + }{ + {"number", WithMaxComponents(five), "5"}, + {"all", WithMaxComponents(AllComponents()), "all"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i, q := range scanWithOptions(t, tt.opt) { + if got := q[maxComponentsParam]; len(got) != 1 || got[0] != tt.want { + t.Errorf("chunk %d: %s = %v, want [%s]", i, maxComponentsParam, got, tt.want) + } + } + }) + } +} + +func TestMaxComponentsUnsetSendsNothing(t *testing.T) { + for name, opts := range map[string][]ScanOption{ + "option absent": nil, + "zero limit": {WithMaxComponents(ComponentLimit{})}, + } { + t.Run(name, func(t *testing.T) { + for i, q := range scanWithOptions(t, opts...) { + if len(q) != 0 { + t.Errorf("chunk %d carried a query %v, want none", i, q) + } + } + }) + } +} + +func fiveComponents(t *testing.T) ScanOption { + t.Helper() + l, err := ComponentsUpTo(5) + if err != nil { + t.Fatal(err) + } + return WithMaxComponents(l) +} + +// A server that refuses the parameter (a session whose blocks disagree, or a bad value) answers 400. +// The scan has to fail and say so; swallowing it like a 409 would report a scan that never ran. +func TestMaxComponentsRejectedBlockFailsTheScan(t *testing.T) { + mock := &scanMock{completeAt: 1, result: json.RawMessage(`{"files":[],"components":{}}`)} + inner := mock.handler() + var rejected int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && strings.HasPrefix(r.Header.Get("Content-Range"), "bytes 4-") { + atomic.AddInt32(&rejected, 1) + http.Error(w, "max_components must be the same on every block", http.StatusBadRequest) + return + } + inner(w, r) + })) + t.Cleanup(srv.Close) + c := mustNew(t, Config{APIURL: srv.URL, Workers: 1, RetryBackoffBase: time.Millisecond}) + + _, err := c.Scan.WFP(context.Background(), []byte("0123456789abc"), + WithChunkBytes(4), WithPollInterval(time.Millisecond), fiveComponents(t)) + if err == nil { + t.Fatal("a 400 on one block was swallowed, want the scan to fail") + } + var statusErr *StatusError + if !errors.As(err, &statusErr) || statusErr.StatusCode != http.StatusBadRequest { + t.Errorf("error = %v, want a StatusError with status 400", err) + } + if !strings.Contains(err.Error(), "400") { + t.Errorf("error %q does not name the status", err) + } + if n := atomic.LoadInt32(&rejected); n != 1 { + t.Errorf("the 400 block was sent %d times, want 1: a client error is not retryable", n) + } +} + +// The retry rebuilds nothing by hand, but it must still carry the parameter: a block that +// reaches the server without it would split the session and earn a 400. +func TestMaxComponentsSurvivesARetry(t *testing.T) { + mock := &scanMock{completeAt: 1, result: json.RawMessage(`{"files":[],"components":{}}`)} + var mu sync.Mutex + var queries []url.Values + var posts int32 + inner := mock.handler() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + mu.Lock() + queries = append(queries, r.URL.Query()) + mu.Unlock() + if atomic.AddInt32(&posts, 1) == 1 { + http.Error(w, "briefly unavailable", http.StatusServiceUnavailable) + return + } + } + inner(w, r) + })) + t.Cleanup(srv.Close) + c := mustNew(t, Config{APIURL: srv.URL, Workers: 1, RetryBackoffBase: time.Millisecond}) + + if _, err := c.Scan.WFP(context.Background(), []byte("0123456789abc"), + WithChunkBytes(4), WithPollInterval(time.Millisecond), fiveComponents(t)); err != nil { + t.Fatalf("a scan must survive one 503: %v", err) + } + mu.Lock() + defer mu.Unlock() + if len(queries) != 5 { // 4 blocks + the one that failed + t.Fatalf("recorded %d uploads, want 5 (4 blocks and 1 retry)", len(queries)) + } + for i, q := range queries { + if got := q[maxComponentsParam]; len(got) != 1 || got[0] != "5" { + t.Errorf("upload %d (the first is the failed attempt, the second its retry): %s = %v, want [5]", i, maxComponentsParam, got) + } + } +} diff --git a/pkg/scanoss/scan.go b/pkg/scanoss/scan.go index 662e6d6..a47209b 100644 --- a/pkg/scanoss/scan.go +++ b/pkg/scanoss/scan.go @@ -31,6 +31,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "sync" "time" @@ -100,6 +101,7 @@ type scanOptions struct { onBOMReport func(postprocess.Report) // receives what the BOM selection rules concluded (WithBOMReport) wfpOptions []wfp.Option // per-file fingerprinting tuning (WithFingerprintOptions) rankThreshold int // drop matches ranked worse than this (WithRankingThreshold) + maxComponents ComponentLimit // components the server reports, sent with every block (WithMaxComponents) pollInterval time.Duration // scan status poll cadence (WithPollInterval) reporter ScanReporter // receives this scan's stages (WithScanReporter) onScanID func(string) // receives this scan's id once it is resumable (WithScanIDNotify) @@ -174,6 +176,19 @@ func WithRankingThreshold(threshold int) ScanOption { return func(o *scanOptions) { o.rankThreshold = threshold } } +// WithMaxComponents asks the server to report up to l components, or all of them. For a +// full-file match that is the best N components of the project ranking plus up to three context +// picks (folder, file alone, origin), each labelled in the match's context; a snippet block gets +// up to N candidates, and the allowed set widens to at least N. It is sent with every WFP block +// of the scan, because the server refuses a session whose blocks disagree. An unset limit sends +// nothing and leaves the server's defaults in force. +// +// The server must support the max_components parameter. Unlike WithRankingThreshold this is not +// applied client-side. +func WithMaxComponents(l ComponentLimit) ScanOption { + return func(o *scanOptions) { o.maxComponents = l } +} + // WithFingerprintOptions tunes how each file is fingerprinted by Folder and Files — today, // whether the leading licence header, comments and imports are dropped. They are dropped by // default; wfp.WithoutSkipHeaders fingerprints files whole, and wfp.WithSkipHeaders caps how many @@ -321,7 +336,7 @@ func (s scanService) upload(ctx context.Context, r io.ReaderAt, size int64, o sc ranges := chunkRanges(size, o.chunkBytes) logging.Debug("uploading WFP chunks", "scanID", scanID, "chunks", len(ranges)) prog := &chunkProgress{r: o.reporter, total: len(ranges)} - if err := s.uploadChunks(ctx, scanID, r, size, ranges, prog); err != nil { + if err := s.uploadChunks(ctx, scanID, r, size, ranges, o.uploadQuery(), prog); err != nil { return "", err } @@ -333,9 +348,18 @@ func (s scanService) upload(ctx context.Context, r io.ReaderAt, size int64, o sc return scanID, nil } +// uploadQuery is the query string every block of the upload carries. It is empty unless the +// caller set something that has to reach the server. +func (o scanOptions) uploadQuery() url.Values { + if !o.maxComponents.IsSet() { + return nil + } + return url.Values{maxComponentsParam: {o.maxComponents.String()}} +} + // uploadChunks uploads all chunks concurrently, bounded by the client's worker // limit, aborting all on the first error. -func (s scanService) uploadChunks(ctx context.Context, scanID string, r io.ReaderAt, size int64, ranges [][2]int64, prog *chunkProgress) error { +func (s scanService) uploadChunks(ctx context.Context, scanID string, r io.ReaderAt, size int64, ranges [][2]int64, query url.Values, prog *chunkProgress) error { if len(ranges) == 0 { return nil } @@ -365,7 +389,7 @@ func (s scanService) uploadChunks(ctx context.Context, scanID string, r io.Reade continue default: } - if err := s.c.uploadChunk(ctx, scanID, r, rng[0], rng[1], size); err != nil { + if err := s.c.uploadChunk(ctx, scanID, r, rng[0], rng[1], size, query); err != nil { errCh <- err cancel() continue @@ -418,12 +442,15 @@ func chunkRanges(total int64, size int64) [][2]int64 { // lost to a timeout. The ranges this client sends are deterministic and never overlap, so // a conflict cannot mean anything else. Treating it as a failure would report a scan as // broken when its upload in fact completed — and the session it names is still resumable. -func (c *Client) uploadChunk(ctx context.Context, scanID string, src io.ReaderAt, off, end, total int64) error { +func (c *Client) uploadChunk(ctx context.Context, scanID string, src io.ReaderAt, off, end, total int64, query url.Values) error { block := func() *io.SectionReader { return io.NewSectionReader(src, off, end-off+1) } req, err := c.newRequest(http.MethodPost, serviceScan.endpoint, block()) if err != nil { return err } + if len(query) > 0 { + req.URL.RawQuery = query.Encode() + } // http.NewRequest wires ContentLength and body replay (GetBody) only for bytes and // strings readers. A SectionReader gets neither: without these, the send falls back // to chunked transfer-encoding and a retried chunk has no body to resend.