diff --git a/README.md b/README.md index 1d54f8d..1c40de4 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,19 @@ for _, p := range doc.Packages { `Parse` auto-detects the format. `Detect` returns just the type without a full parse, scanning JSON keys, the XML root, or the tag-value document header. +`Inspect` reads the type and specification version from an `io.Reader`. It stops once both values are known and skips unrelated JSON values without building the package model: + +```go +file, _ := os.Open("bom.cdx.json") +defer file.Close() + +metadata, err := sbom.Inspect(file) +if err != nil { + log.Fatal(err) +} +fmt.Println(metadata.Type, metadata.SpecVersion) +``` + Packages with different document IDs or PURLs remain separate, even when their names and versions match. `AddPackage` replaces an entry only when its name, version, document ID, and set of PURLs match. `Package.PURL()` returns the first PURL; `Package.ExternalRefs` retains all declared references. ```go diff --git a/bench_test.go b/bench_test.go index eb570af..9d97a98 100644 --- a/bench_test.go +++ b/bench_test.go @@ -1,6 +1,7 @@ package sbom import ( + "bytes" "encoding/json" "fmt" "io" @@ -129,6 +130,23 @@ func BenchmarkDetect(b *testing.B) { } } +func BenchmarkInspect(b *testing.B) { + for _, f := range benchFixtures { + data := readFixture(b, f.path) + b.Run(f.name, func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(data))) + b.ResetTimer() + for range b.N { + _, benchmarkError = Inspect(bytes.NewReader(data)) + } + if benchmarkError != nil { + b.Fatal(benchmarkError) + } + }) + } +} + func BenchmarkPackagePURL(b *testing.B) { doc := mustParseBenchmark(b, readFixture(b, "cyclonedx/juice-shop.cdx.json")) b.Run(fmt.Sprintf("%d-pkgs", len(doc.Packages)), func(b *testing.B) { diff --git a/go.mod b/go.mod index 390e434..e68eea2 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/git-pkgs/sbom -go 1.26 +go 1.27 -toolchain go1.26.7 +toolchain go1.27.3 diff --git a/inspect.go b/inspect.go new file mode 100644 index 0000000..50f8e07 --- /dev/null +++ b/inspect.go @@ -0,0 +1,417 @@ +package sbom + +import ( + "bufio" + "bytes" + "encoding/json/jsontext" + "encoding/xml" + "fmt" + "io" + "strings" + "unicode" + "unicode/utf8" +) + +const ( + tagLinePrefixLimit = 4096 + tagLineInitialCapacity = 256 + spdxTextEnd = "" +) + +var spdxTextEndBytes = []byte(spdxTextEnd) + +type textEndState uint8 + +const ( + textEndAbsent textEndState = iota + textEndValid + textEndInvalid +) + +// Metadata is the format information available without parsing an SBOM model. +type Metadata struct { + Type Type + SpecVersion string +} + +// Inspect reads enough of r to identify an SBOM and its specification version. +// It returns ErrUnrecognized when the content is unsupported or malformed. +func Inspect(r io.Reader) (Metadata, error) { + if r == nil { + return Metadata{}, ErrUnrecognized + } + + source := &errorRecordingReader{reader: r} + reader := bufio.NewReader(source) + if err := trimReaderPrefix(reader); err != nil { + return inspectResult(Metadata{}, err, source.err) + } + + start, err := reader.Peek(1) + if err != nil { + return inspectResult(Metadata{}, err, source.err) + } + + var metadata Metadata + switch start[0] { + case '{': + metadata, err = inspectJSON(reader) + case '<': + metadata, err = inspectXML(reader) + default: + metadata, err = inspectTagValue(reader) + } + return inspectResult(metadata, err, source.err) +} + +func inspectResult(metadata Metadata, parseErr, readErr error) (Metadata, error) { + if readErr != nil { + return Metadata{}, fmt.Errorf("sbom: inspect: %w", readErr) + } + if parseErr != nil { + return Metadata{}, fmt.Errorf("%w: %v", ErrUnrecognized, parseErr) + } + if metadata.Type == TypeUnknown { + return Metadata{}, ErrUnrecognized + } + return metadata, nil +} + +type errorRecordingReader struct { + reader io.Reader + err error +} + +func (r *errorRecordingReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if err != nil && err != io.EOF { + r.err = err + } + return n, err +} + +func trimReaderPrefix(reader *bufio.Reader) error { + bomAllowed := true + for { + r, _, err := reader.ReadRune() + if err != nil { + return err + } + if unicode.IsSpace(r) { + continue + } + if r == '\ufeff' && bomAllowed { + bomAllowed = false + continue + } + return reader.UnreadRune() + } +} + +type jsonInspection struct { + bomFormat string + specVersion string + spdxVersion string + hasSPDXVersion bool + hasSPDXID bool + hasSBOM bool + sbomSpecVersion string + predicateType string + predicateSpecVersion string +} + +func (i jsonInspection) metadata() Metadata { + switch { + case i.bomFormat == cdxBOMFormat: + return Metadata{Type: TypeCycloneDX, SpecVersion: i.specVersion} + case i.hasSPDXVersion || i.hasSPDXID: + return Metadata{Type: TypeSPDX, SpecVersion: i.spdxVersion} + case i.hasSBOM: + return Metadata{Type: TypeSPDX, SpecVersion: i.sbomSpecVersion} + case strings.Contains(i.predicateType, "spdx"): + return Metadata{Type: TypeSPDX, SpecVersion: i.predicateSpecVersion} + default: + return Metadata{} + } +} + +func inspectJSON(reader io.Reader) (Metadata, error) { + decoder := jsontext.NewDecoder(reader, + jsontext.AllowDuplicateNames(true), + jsontext.AllowInvalidUTF8(true), + ) + inspection, err := inspectJSONObject(decoder, true) + if err != nil { + return Metadata{}, err + } + return inspection.metadata(), nil +} + +func inspectJSONObject(decoder *jsontext.Decoder, stopWhenComplete bool) (jsonInspection, error) { + opening, err := decoder.ReadToken() + if err != nil { + return jsonInspection{}, err + } + if opening.Kind() != '{' { + return jsonInspection{}, fmt.Errorf("expected JSON object") + } + + var inspection jsonInspection + for decoder.PeekKind() != '}' { + name, err := decoder.ReadToken() + if err != nil { + return jsonInspection{}, err + } + if name.Kind() != '"' { + return jsonInspection{}, fmt.Errorf("expected JSON object name") + } + + switch name.String() { + case "bomFormat": + inspection.bomFormat, err = readJSONString(decoder) + case "specVersion": + inspection.specVersion, err = readJSONString(decoder) + case "spdxVersion": + inspection.hasSPDXVersion = true + inspection.spdxVersion, err = readJSONString(decoder) + case spdxTagID: + inspection.hasSPDXID = true + err = decoder.SkipValue() + case "sbom": + inspection.hasSBOM = true + var nested jsonInspection + nested, err = inspectJSONValue(decoder, true) + if metadata := nested.metadata(); metadata.Type == TypeSPDX { + inspection.sbomSpecVersion = metadata.SpecVersion + } + case "predicateType": + inspection.predicateType, err = readJSONString(decoder) + case "predicate": + var nested jsonInspection + nested, err = inspectJSONValue(decoder, strings.Contains(inspection.predicateType, "spdx")) + if metadata := nested.metadata(); metadata.Type == TypeSPDX { + inspection.predicateSpecVersion = metadata.SpecVersion + } + default: + err = decoder.SkipValue() + } + if err != nil { + return jsonInspection{}, err + } + if metadata := inspection.metadata(); stopWhenComplete && metadata.Type != TypeUnknown && metadata.SpecVersion != "" { + return inspection, nil + } + } + if _, err := decoder.ReadToken(); err != nil { + return jsonInspection{}, err + } + return inspection, nil +} + +func inspectJSONValue(decoder *jsontext.Decoder, stopWhenComplete bool) (jsonInspection, error) { + if decoder.PeekKind() != '{' { + return jsonInspection{}, decoder.SkipValue() + } + return inspectJSONObject(decoder, stopWhenComplete) +} + +func readJSONString(decoder *jsontext.Decoder) (string, error) { + if decoder.PeekKind() != '"' { + return "", decoder.SkipValue() + } + token, err := decoder.ReadToken() + if err != nil { + return "", err + } + return token.String(), nil +} + +func inspectXML(reader io.Reader) (Metadata, error) { + decoder := xml.NewDecoder(reader) + for { + token, err := decoder.Token() + if err != nil { + return Metadata{}, err + } + start, ok := token.(xml.StartElement) + if !ok { + continue + } + if start.Name.Local != "bom" { + return Metadata{}, nil + } + version := cycloneDXXMLVersion(start.Name.Space) + if start.Name.Space != "" && version == "" { + return Metadata{}, nil + } + return Metadata{Type: TypeCycloneDX, SpecVersion: version}, nil + } +} + +func inspectTagValue(reader *bufio.Reader) (Metadata, error) { + inText := false + for { + line, textEnd, err := readLinePrefix(reader) + if err == io.EOF { + return Metadata{}, nil + } + if err != nil { + return Metadata{}, err + } + if inText { + closed, valid := inspectTextEnd(textEnd) + if !valid { + return Metadata{}, nil + } + inText = !closed + continue + } + + metadata, multiline, done := inspectTagLine(line, textEnd) + if done { + return metadata, nil + } + inText = multiline + } +} + +func inspectTextEnd(textEnd textEndState) (bool, bool) { + if textEnd == textEndAbsent { + return false, true + } + return true, textEnd == textEndValid +} + +func inspectTagLine(line []byte, textEnd textEndState) (Metadata, bool, bool) { + lineText := strings.TrimLeftFunc(string(line), unicode.IsSpace) + if lineText == "" || strings.HasPrefix(lineText, "#") { + return Metadata{}, false, false + } + tag, value, ok := strings.Cut(lineText, ":") + tag = strings.TrimSpace(tag) + if !ok || tag == spdxTagPackageName { + return Metadata{}, false, true + } + + value = strings.TrimLeftFunc(value, unicode.IsSpace) + if text, multiline := strings.CutPrefix(value, ""); multiline { + if textEnd == textEndAbsent { + return Metadata{}, true, false + } + if textEnd == textEndInvalid { + return Metadata{}, false, true + } + value, _, _ = strings.Cut(text, spdxTextEnd) + } + value = strings.TrimSpace(value) + if tag == spdxTagVersion && strings.HasPrefix(value, "SPDX-") { + return Metadata{Type: TypeSPDX, SpecVersion: value}, false, true + } + return Metadata{}, false, false +} + +func readLinePrefix(reader *bufio.Reader) ([]byte, textEndState, error) { + prefix := make([]byte, 0, tagLineInitialCapacity) + var tail [len(spdxTextEnd) - 1]byte + tailLen := 0 + foundTextEnd := false + var trailing textEndTrailing + + for { + fragment, more, err := reader.ReadLine() + if err != nil { + return nil, textEndAbsent, err + } + if foundTextEnd { + trailing.add(fragment) + } else if after, found := fragmentTextEnd(tail[:tailLen], fragment); found { + foundTextEnd = true + trailing.add(after) + } else { + tailLen = updateTextTail(tail[:], tailLen, fragment) + } + if remaining := tagLinePrefixLimit - len(prefix); remaining > 0 { + prefix = append(prefix, fragment[:min(remaining, len(fragment))]...) + } + if !more { + switch { + case !foundTextEnd: + return prefix, textEndAbsent, nil + case trailing.valid(): + return prefix, textEndValid, nil + default: + return prefix, textEndInvalid, nil + } + } + } +} + +func fragmentTextEnd(tail, fragment []byte) ([]byte, bool) { + var boundary [2 * (len(spdxTextEnd) - 1)]byte + n := copy(boundary[:], tail) + n += copy(boundary[n:], fragment[:min(len(fragment), len(spdxTextEnd)-1)]) + index := bytes.Index(boundary[:n], spdxTextEndBytes) + if index >= 0 { + consumed := index + len(spdxTextEnd) - len(tail) + return fragment[consumed:], true + } + if index := bytes.Index(fragment, spdxTextEndBytes); index >= 0 { + return fragment[index+len(spdxTextEnd):], true + } + return nil, false +} + +type textEndTrailing struct { + pending [utf8.UTFMax]byte + pendingLen int + invalid bool +} + +func (t *textEndTrailing) add(data []byte) { + if t.invalid { + return + } + if t.pendingLen > 0 { + for len(data) > 0 && !utf8.FullRune(t.pending[:t.pendingLen]) { + t.pending[t.pendingLen] = data[0] + t.pendingLen++ + data = data[1:] + } + if !utf8.FullRune(t.pending[:t.pendingLen]) { + return + } + r, _ := utf8.DecodeRune(t.pending[:t.pendingLen]) + if !unicode.IsSpace(r) { + t.invalid = true + return + } + t.pendingLen = 0 + } + for len(data) > 0 { + if !utf8.FullRune(data) { + t.pendingLen = copy(t.pending[:], data) + return + } + r, size := utf8.DecodeRune(data) + if !unicode.IsSpace(r) { + t.invalid = true + return + } + data = data[size:] + } +} + +func (t *textEndTrailing) valid() bool { + return !t.invalid && t.pendingLen == 0 +} + +func updateTextTail(tail []byte, tailLen int, fragment []byte) int { + if len(fragment) >= len(tail) { + return copy(tail, fragment[len(fragment)-len(tail):]) + } + var combined [2 * (len(spdxTextEnd) - 1)]byte + n := copy(combined[:], tail[:tailLen]) + n += copy(combined[n:], fragment) + start := max(0, n-len(tail)) + return copy(tail, combined[start:n]) +} diff --git a/inspect_test.go b/inspect_test.go new file mode 100644 index 0000000..3968d50 --- /dev/null +++ b/inspect_test.go @@ -0,0 +1,210 @@ +package sbom + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestInspect(t *testing.T) { + tests := []struct { + name string + in string + want Metadata + }{ + { + name: "cyclonedx json", + in: `{"components":[{"name":"example"}],"specVersion":"1.6","bomFormat":"CycloneDX"}`, + want: Metadata{Type: TypeCycloneDX, SpecVersion: "1.6"}, + }, + { + name: "spdx json", + in: `{"packages":[],"spdxVersion":"SPDX-2.3"}`, + want: Metadata{Type: TypeSPDX, SpecVersion: "SPDX-2.3"}, + }, + { + name: "github envelope", + in: `{"sbom":{"packages":[],"spdxVersion":"SPDX-2.3"}}`, + want: Metadata{Type: TypeSPDX, SpecVersion: "SPDX-2.3"}, + }, + { + name: "predicate before type", + in: `{"predicate":{"packages":[],"spdxVersion":"SPDX-2.2"},"predicateType":"https://spdx.dev/Document"}`, + want: Metadata{Type: TypeSPDX, SpecVersion: "SPDX-2.2"}, + }, + { + name: "cyclonedx xml", + in: ``, + want: Metadata{Type: TypeCycloneDX, SpecVersion: "1.5"}, + }, + { + name: "spdx tag value", + in: "\ufeff # generated\nDataLicense: CC0-1.0\nSPDXVersion: SPDX-2.3\nPackageName: example\n", + want: Metadata{Type: TypeSPDX, SpecVersion: "SPDX-2.3"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := Inspect(&chunkReader{reader: strings.NewReader(test.in), size: 3}) + if err != nil { + t.Fatal(err) + } + if got != test.want { + t.Fatalf("Inspect = %+v, want %+v", got, test.want) + } + }) + } +} + +func TestInspectFixtures(t *testing.T) { + err := filepath.WalkDir("testdata", func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + parsed, err := Parse(data) + if err != nil { + return err + } + t.Run(path, func(t *testing.T) { + metadata, err := Inspect(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + want := Metadata{Type: parsed.Type, SpecVersion: parsed.SpecVersion} + if metadata != want { + t.Fatalf("Inspect = %+v, want %+v", metadata, want) + } + }) + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +func TestInspectStopsAfterMetadata(t *testing.T) { + input := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[` + strings.Repeat(`{"name":"example"},`, 100_000) + `{"name":"last"}]}` + reader := &countingReader{reader: strings.NewReader(input)} + + metadata, err := Inspect(reader) + if err != nil { + t.Fatal(err) + } + if metadata != (Metadata{Type: TypeCycloneDX, SpecVersion: "1.6"}) { + t.Fatalf("Inspect = %+v", metadata) + } + if reader.read >= len(input) { + t.Fatalf("Inspect read the complete %d byte document", len(input)) + } +} + +func TestInspectSkipsLongTagValueText(t *testing.T) { + input := "DocumentComment: \n" + strings.Repeat("x", 100_000) + "\nSPDXVersion: SPDX-1.0\n\nSPDXVersion: SPDX-2.3\n" + metadata, err := Inspect(strings.NewReader(input)) + if err != nil { + t.Fatal(err) + } + if metadata != (Metadata{Type: TypeSPDX, SpecVersion: "SPDX-2.3"}) { + t.Fatalf("Inspect = %+v", metadata) + } +} + +func TestInspectLongTagValueTextEnd(t *testing.T) { + for _, test := range []struct { + name string + padding int + suffix string + wantErr bool + }{ + {name: "closing tag beyond prefix", padding: tagLinePrefixLimit + 1, suffix: " \t\u2003"}, + {name: "closing tag split at prefix", padding: tagLinePrefixLimit - 3, suffix: " \t"}, + {name: "unicode whitespace split at prefix", padding: tagLinePrefixLimit - len(spdxTextEnd) - 1, suffix: "\u2003"}, + {name: "invalid suffix", padding: tagLinePrefixLimit + 1, suffix: "suffix", wantErr: true}, + {name: "first closing tag wins", padding: tagLinePrefixLimit - 3, suffix: "suffix" + spdxTextEnd, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + input := "DocumentComment: \n" + strings.Repeat("x", test.padding) + spdxTextEnd + test.suffix + "\nSPDXVersion: SPDX-2.3\n" + metadata, err := Inspect(&chunkReader{reader: strings.NewReader(input), size: 3}) + if test.wantErr { + if metadata != (Metadata{}) || !errors.Is(err, ErrUnrecognized) { + t.Fatalf("Inspect = %+v, %v", metadata, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if metadata != (Metadata{Type: TypeSPDX, SpecVersion: "SPDX-2.3"}) { + t.Fatalf("Inspect = %+v", metadata) + } + }) + } +} + +func TestInspectErrors(t *testing.T) { + for _, input := range []string{ + "", + `{"foo":1}`, + `{"bomFormat":`, + ``, + "PackageName: example", + "DocumentComment: commentsuffix\nSPDXVersion: SPDX-2.3", + "DocumentComment: \ncomment\nsuffix\nSPDXVersion: SPDX-2.3", + } { + if metadata, err := Inspect(strings.NewReader(input)); metadata != (Metadata{}) || !errors.Is(err, ErrUnrecognized) { + t.Errorf("Inspect(%q) = %+v, %v", input, metadata, err) + } + } + + sentinel := errors.New("read failed") + reader := &failingReader{data: []byte(`{"packages":[`), err: sentinel} + if _, err := Inspect(reader); !errors.Is(err, sentinel) { + t.Fatalf("Inspect error = %v, want %v", err, sentinel) + } +} + +type chunkReader struct { + reader io.Reader + size int +} + +func (r *chunkReader) Read(p []byte) (int, error) { + return r.reader.Read(p[:min(len(p), r.size)]) +} + +type countingReader struct { + reader io.Reader + read int +} + +func (r *countingReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.read += n + return n, err +} + +type failingReader struct { + data []byte + err error +} + +func (r *failingReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, r.err + } + n := copy(p, r.data) + r.data = r.data[n:] + return n, nil +} diff --git a/sbom.go b/sbom.go index 8c2673c..ec589cd 100644 --- a/sbom.go +++ b/sbom.go @@ -513,7 +513,7 @@ func detect(data []byte) detectedFormat { return detectedFormat{typ: TypeCycloneDX} } return detectedFormat{} - case "spdxVersion", "SPDXID": + case "spdxVersion", spdxTagID: return detectedFormat{typ: TypeSPDX} case "sbom": return detectedFormat{typ: TypeSPDX, spdxEnvelope: true} diff --git a/spdx_tagvalue.go b/spdx_tagvalue.go index cb52718..5f452ab 100644 --- a/spdx_tagvalue.go +++ b/spdx_tagvalue.go @@ -8,7 +8,12 @@ import ( "unicode" ) -const spdxTripleFields = 3 +const ( + spdxTripleFields = 3 + spdxTagID = "SPDXID" + spdxTagPackageName = "PackageName" + spdxTagVersion = "SPDXVersion" +) func detectSPDXTagValue(data []byte) detectedFormat { scanner := newSPDXScanner(data) @@ -18,14 +23,14 @@ func detectSPDXTagValue(data []byte) detectedFormat { continue } tag, value, ok := strings.Cut(line, ":") - if !ok || strings.TrimSpace(tag) == "PackageName" { + if !ok || strings.TrimSpace(tag) == spdxTagPackageName { break } value, err := readSPDXText(scanner, value) if err != nil { break } - if strings.TrimSpace(tag) == "SPDXVersion" && strings.HasPrefix(value, "SPDX-") { + if strings.TrimSpace(tag) == spdxTagVersion && strings.HasPrefix(value, "SPDX-") { return detectedFormat{typ: TypeSPDX, tagValue: true} } } @@ -98,7 +103,7 @@ func readSPDXText(scanner *bufio.Scanner, value string) (string, error) { func (p *spdxTagParser) apply(tag, value string) error { switch tag { - case "SPDXVersion": + case spdxTagVersion: p.doc.SPDXVersion = value case "DataLicense": p.doc.DataLicense = value @@ -108,14 +113,14 @@ func (p *spdxTagParser) apply(tag, value string) error { p.doc.DocumentNamespace = value case "Creator", "Created", "LicenseListVersion": p.creationInfo(tag, value) - case "PackageName": + case spdxTagPackageName: p.inDocument = false p.doc.Packages = append(p.doc.Packages, spdxPackage{Name: value}) p.pkg = &p.doc.Packages[len(p.doc.Packages)-1] case "FileName", "SnippetSPDXID", "LicenseID", "Annotator", "Reviewer": p.inDocument = false p.pkg = nil - case "SPDXID": + case spdxTagID: if p.pkg != nil { p.pkg.SPDXID = value } else if p.inDocument {