From f1bfdf8b5821b6fa465317860b3dac9c919e155c Mon Sep 17 00:00:00 2001 From: Arlo Siemsen Date: Wed, 26 Feb 2025 17:07:49 -0600 Subject: [PATCH] Initial implementation of cargo-sbom detector --- .../rust/Contracts/CargoSbom.cs | 178 ++++++++++++ .../rust/RustSbomDetector.cs | 126 +++++++++ .../Configs/RustSbomVsCliExperiment.cs | 22 ++ .../Configs/RustSbomVsCrateExperiment.cs | 22 ++ .../Extensions/ServiceCollectionExtensions.cs | 3 + .../RustSbomDetectorTests.cs | 257 ++++++++++++++++++ 6 files changed, 608 insertions(+) create mode 100644 src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoSbom.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs create mode 100644 src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/RustSbomVsCliExperiment.cs create mode 100644 src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/RustSbomVsCrateExperiment.cs create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomDetectorTests.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoSbom.cs b/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoSbom.cs new file mode 100644 index 000000000..f674c0536 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoSbom.cs @@ -0,0 +1,178 @@ +// Schema for Cargo SBOM pre-cursor files (*.cargo-sbom.json) + +namespace Microsoft.ComponentDetection.Detectors.Rust.Sbom.Contracts; + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +#pragma warning disable SA1402 +#pragma warning disable SA1204 + +/// +/// Type of dependency. +/// +public enum SbomKind +{ + /// + /// A dependency linked to the artifact produced by this crate. + /// + Normal, + + /// + /// A compile-time dependency used to build this crate. + /// + Build, + + /// + /// An unexpected dependency kind. + /// + Unknown, +} + +/// +/// Represents the Cargo Software Bill of Materials (SBOM). +/// +public class CargoSbom +{ + /// + /// Gets or sets the version of the SBOM. + /// + public int Version { get; set; } + + /// + /// Gets or sets the index of the root crate. + /// + public int Root { get; set; } + + /// + /// Gets or sets the list of crates. + /// + public List Crates { get; set; } + + /// + /// Gets or sets the information about rustc used to perform the compilation. + /// + public Rustc Rustc { get; set; } + + /// + /// Deserialize from JSON. + /// + /// Cargo SBOM. + public static CargoSbom FromJson(string json) => JsonSerializer.Deserialize(json, Converter.Settings); +} + +/// +/// Represents a crate in the SBOM. +/// +public class SbomCrate +{ + /// + /// Gets or sets the Cargo Package ID specification. + /// + public string Id { get; set; } + + /// + /// Gets or sets the enabled feature flags. + /// + public List Features { get; set; } + + /// + /// Gets or sets the enabled cfg attributes set by build scripts. + /// + public List Cfgs { get; set; } + + /// + /// Gets or sets the dependencies for this crate. + /// + public List Dependencies { get; set; } +} + +/// +/// Represents a dependency of a crate. +/// +public class SbomDependency +{ + /// + /// Gets or sets the index into the crates array. + /// + public int Index { get; set; } + + /// + /// Gets or sets the kind of dependency. + /// + public SbomKind Kind { get; set; } +} + +/// +/// Represents information about rustc used to perform the compilation. +/// +public class Rustc +{ + /// + /// Gets or sets the compiler version. + /// + public string Version { get; set; } + + /// + /// Gets or sets the compiler wrapper. + /// + public string Wrapper { get; set; } + + /// + /// Gets or sets the compiler workspace wrapper. + /// + public string WorkspaceWrapper { get; set; } + + /// + /// Gets or sets the commit hash for rustc. + /// + public string CommitHash { get; set; } + + /// + /// Gets or sets the host target triple. + /// + [JsonPropertyName("host")] + public string Host { get; set; } + + /// + /// Gets or sets the verbose version string. + /// + public string VerboseVersion { get; set; } +} + +/// +/// Deserializes SbomKind. +/// +internal class SbomKindConverter : JsonConverter +{ + public static readonly SbomKindConverter Singleton = new SbomKindConverter(); + + public override bool CanConvert(Type t) => t == typeof(SbomKind); + + public override SbomKind Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var value = reader.GetString(); + return value switch + { + "build" => SbomKind.Build, + "normal" => SbomKind.Normal, + _ => SbomKind.Unknown, + }; + } + + public override void Write(Utf8JsonWriter writer, SbomKind value, JsonSerializerOptions options) => throw new NotImplementedException(); +} + +/// +/// Json converter settings. +/// +internal static class Converter +{ + public static readonly JsonSerializerOptions Settings = new(JsonSerializerDefaults.General) + { + Converters = { SbomKindConverter.Singleton }, + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + }; +} diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs new file mode 100644 index 000000000..469f4a200 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs @@ -0,0 +1,126 @@ +namespace Microsoft.ComponentDetection.Detectors.Rust; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Rust.Sbom.Contracts; +using Microsoft.Extensions.Logging; + +public class RustSbomDetector : FileComponentDetector, IDefaultOffComponentDetector +{ + private const string CargoSbomSearchPattern = "*.cargo-sbom.json"; + private const string CratesIoSource = "registry+https://github.com/rust-lang/crates.io-index"; + + /// + /// Cargo Package ID: source#name@version + /// https://rustwiki.org/en/cargo/reference/pkgid-spec.html. + /// + private static readonly Regex CargoPackageIdRegex = new Regex( + @"^(?[^#]*)#?(?[\w\-]*)[@#]?(?\d[\S]*)?$", + RegexOptions.Compiled); + + public RustSbomDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.Logger = logger; + } + + public override string Id => "RustSbom"; + + public override IList SearchPatterns => [CargoSbomSearchPattern]; + + public override IEnumerable SupportedComponentTypes => [ComponentType.Cargo]; + + public override int Version { get; } = 1; + + public override IEnumerable Categories => ["Rust"]; + + private static bool ParsePackageIdSpec(string dependency, out CargoComponent component) + { + var match = CargoPackageIdRegex.Match(dependency); + var name = match.Groups["name"].Value; + var version = match.Groups["version"].Value; + var source = match.Groups["source"].Value; + + if (!match.Success) + { + component = null; + return false; + } + + if (string.IsNullOrWhiteSpace(name)) + { + name = source[(source.LastIndexOf('/') + 1)..]; + } + + if (string.IsNullOrWhiteSpace(source)) + { + source = null; + } + + component = new CargoComponent(name, version, source: source); + return true; + } + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + { + var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; + var components = processRequest.ComponentStream; + var reader = new StreamReader(components.Stream); + var cargoSbom = CargoSbom.FromJson(await reader.ReadToEndAsync(cancellationToken)); + this.RecordLockfileVersion(cargoSbom.Version); + this.ProcessCargoSbom(cargoSbom, singleFileComponentRecorder, components); + } + + private void ProcessDependency(CargoSbom sbom, SbomCrate package, ISingleFileComponentRecorder recorder, IComponentStream components, HashSet visitedNodes, CargoComponent parent = null, int depth = 0) + { + foreach (var dependency in package.Dependencies) + { + var dep = sbom.Crates[dependency.Index]; + var parentComponent = parent; + if (ParsePackageIdSpec(dep.Id, out var component)) + { + if (component.Source == CratesIoSource) + { + parentComponent = component; + recorder.RegisterUsage(new DetectedComponent(component), isExplicitReferencedDependency: depth == 0, parent?.Id, isDevelopmentDependency: false); + } + } + else + { + this.Logger.LogError(null, "Failed to parse Cargo PackageIdSpec '{Id}' in '{Location}'", dep.Id, components.Location); + recorder.RegisterPackageParseFailure(dep.Id); + } + + if (visitedNodes.Add(dependency.Index)) + { + // Skip processing already processed nodes + this.ProcessDependency(sbom, dep, recorder, components, visitedNodes, parentComponent, depth + 1); + } + } + } + + private void ProcessCargoSbom(CargoSbom sbom, ISingleFileComponentRecorder recorder, IComponentStream components) + { + try + { + var visitedNodes = new HashSet(); + this.ProcessDependency(sbom, sbom.Crates[sbom.Root], recorder, components, visitedNodes); + } + catch (Exception e) + { + // If something went wrong, just ignore the file + this.Logger.LogError(e, "Failed to process Cargo SBOM file '{FileLocation}'", components.Location); + } + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/RustSbomVsCliExperiment.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/RustSbomVsCliExperiment.cs new file mode 100644 index 000000000..08ebb1b35 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/RustSbomVsCliExperiment.cs @@ -0,0 +1,22 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; + +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.Rust; + +/// +/// Validating the Rust SBOM detector against the Rust CLI detector. +/// +public class RustSbomVsCliExperiment : IExperimentConfiguration +{ + /// + public string Name => "RustSbomVsCliExperiment"; + + /// + public bool IsInControlGroup(IComponentDetector componentDetector) => componentDetector is RustCliDetector; + + /// + public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is RustSbomDetector; + + /// + public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true; +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/RustSbomVsCrateExperiment.cs b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/RustSbomVsCrateExperiment.cs new file mode 100644 index 000000000..4801ff2f7 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/RustSbomVsCrateExperiment.cs @@ -0,0 +1,22 @@ +namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; + +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.Rust; + +/// +/// Validating the Rust SBOM detector against the Rust crate detector. +/// +public class RustSbomVsCrateExperiment : IExperimentConfiguration +{ + /// + public string Name => "RustSbomVsCrateExperiment"; + + /// + public bool IsInControlGroup(IComponentDetector componentDetector) => componentDetector is RustCrateDetector; + + /// + public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is RustSbomDetector; + + /// + public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true; +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 306199b56..5ae7ee90b 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -65,6 +65,8 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -137,6 +139,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s // Rust services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // SPDX services.AddSingleton(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomDetectorTests.cs new file mode 100644 index 000000000..7e01e2cf6 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/RustSbomDetectorTests.cs @@ -0,0 +1,257 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Detectors.Rust; +using Microsoft.ComponentDetection.Detectors.Rust.Sbom.Contracts; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class RustSbomDetectorTests : BaseDetectorTest +{ + private readonly string testSbom = /*lang=json,strict*/ @" +{ + ""version"": 1, + ""root"": 0, + ""crates"": [ + { + ""id"": ""path+file:///temp/test-crate#0.1.0"", + ""features"": [], + ""dependencies"": [ + { + ""index"": 1, + ""kind"": ""normal"" + }, + { + ""index"": 1, + ""kind"": ""build"" + }, + { + ""index"": 2, + ""kind"": ""normal"" + } + ] + }, + { + ""id"": ""registry+https://github.com/rust-lang/crates.io-index#my_dependency@1.0.0"", + ""features"": [], + ""unexpected_new_thing_from_the_future"": ""foo"", + ""dependencies"": [] + }, + { + ""id"": ""registry+https://github.com/rust-lang/crates.io-index#other_dependency@0.4.0"", + ""features"": [], + ""dependencies"": [ + { + ""index"": 3, + ""kind"": ""normal"" + } + ] + }, + { + ""id"": ""registry+https://github.com/rust-lang/crates.io-index#other_dependency_dependency@0.1.12-alpha.6"", + ""features"": [], + ""dependencies"": [] + } + ], + ""rustc"": { + ""version"": ""1.84.1"", + ""wrapper"": null, + ""workspace_wrapper"": null, + ""commit_hash"": ""2b00e2aae6389eb20dbb690bce5a28cc50affa53"", + ""host"": ""x86_64-pc-windows-msvc"", + ""verbose_version"": ""rustc 1.84.1"" + }, + ""target"": ""x86_64-pc-windows-msvc"" +} +"; + + private readonly string testSbomWithGitDeps = /*lang=json,strict*/ @"{ + ""version"": 1, + ""root"": 2, + ""crates"": [ + { + ""id"": ""registry+https://github.com/rust-lang/crates.io-index#aho-corasick@1.1.3"", + ""features"": [ + ""perf-literal"", + ""std"" + ], + ""dependencies"": [ + { + ""index"": 3, + ""kind"": ""normal"" + } + ], + ""kind"": [ + ""lib"" + ] + }, + { + ""id"": ""path+file:///D:/temp/hello#0.1.0"", + ""features"": [], + ""dependencies"": [ + { + ""index"": 4, + ""kind"": ""normal"" + } + ], + ""kind"": [ + ""lib"" + ] + }, + { + ""id"": ""path+file:///D:/temp/hello#0.1.0"", + ""features"": [], + ""dependencies"": [ + { + ""index"": 1, + ""kind"": ""normal"" + }, + { + ""index"": 4, + ""kind"": ""normal"" + } + ], + ""kind"": [ + ""bin"" + ] + }, + { + ""id"": ""registry+https://github.com/rust-lang/crates.io-index#memchr@2.7.4"", + ""features"": [ + ""alloc"", + ""std"" + ], + ""dependencies"": [], + ""kind"": [ + ""lib"" + ] + }, + { + ""id"": ""git+https://github.com/rust-lang/regex.git#regex@1.11.1"", + ""features"": [ + ], + ""dependencies"": [ + { + ""index"": 0, + ""kind"": ""normal"" + }, + { + ""index"": 3, + ""kind"": ""normal"" + }, + { + ""index"": 5, + ""kind"": ""normal"" + }, + { + ""index"": 6, + ""kind"": ""normal"" + } + ], + ""kind"": [ + ""lib"" + ] + }, + { + ""id"": ""git+https://github.com/rust-lang/regex.git#regex-automata@0.4.9"", + ""features"": [ + ], + ""dependencies"": [ + { + ""index"": 0, + ""kind"": ""normal"" + }, + { + ""index"": 3, + ""kind"": ""normal"" + }, + { + ""index"": 6, + ""kind"": ""normal"" + } + ], + ""kind"": [ + ""lib"" + ] + }, + { + ""id"": ""git+https://github.com/rust-lang/regex.git#regex-syntax@0.8.5"", + ""features"": [ + ], + ""dependencies"": [], + ""kind"": [ + ""lib"" + ] + } + ], + ""rustc"": { + ""version"": ""1.88.0-nightly"", + ""wrapper"": null, + ""workspace_wrapper"": null, + ""commit_hash"": ""6bc57c6bf7d0024ad9ea5a2c112f3fc9c383c8a4"", + ""host"": ""x86_64-pc-windows-msvc"", + ""verbose_version"": ""rustc 1.88.0-nightly (6bc57c6bf 2025-04-22)\nbinary: rustc\ncommit-hash: 6bc57c6bf7d0024ad9ea5a2c112f3fc9c383c8a4\ncommit-date: 2025-04-22\nhost: x86_64-pc-windows-msvc\nrelease: 1.88.0-nightly\nLLVM version: 20.1.2\n"" + }, + ""target"": ""x86_64-pc-windows-msvc"" +}"; + + [TestMethod] + public async Task TestGraphIsCorrectAsync() + { + var sbom = CargoSbom.FromJson(this.testSbom); + + var (result, componentRecorder) = await this.DetectorTestUtility + .WithFile("main.exe.cargo-sbom.json", this.testSbom) + .ExecuteDetectorAsync(); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().HaveCount(3); + + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); // There should only be 1 + + // Verify explicitly referenced roots + var rootComponents = new List + { + "my_dependency 1.0.0 - Cargo", + "other_dependency 0.4.0 - Cargo", + }; + + rootComponents.ForEach(rootComponentId => graph.IsComponentExplicitlyReferenced(rootComponentId).Should().BeTrue()); + + // Verify dependencies for my_dependency + graph.GetDependenciesForComponent("my_dependency 1.0.0 - Cargo").Should().BeEmpty(); + + // Verify dependencies for other_dependency + var other_dependencyDependencies = new List + { + "other_dependency_dependency 0.1.12-alpha.6 - Cargo", + }; + + graph.GetDependenciesForComponent("other_dependency 0.4.0 - Cargo").Should().BeEquivalentTo(other_dependencyDependencies); + } + + [TestMethod] + public async Task TestGraphIsCorrectWithGitDeps() + { + var sbom = CargoSbom.FromJson(this.testSbomWithGitDeps); + + var (result, componentRecorder) = await this.DetectorTestUtility + .WithFile("main.exe.cargo-sbom.json", this.testSbomWithGitDeps) + .ExecuteDetectorAsync(); + + result.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().HaveCount(2); + + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); // There should only be 1 + + // Verify dependencies + graph.GetDependenciesForComponent("aho-corasick 1.1.3 - Cargo").Should().BeEquivalentTo("memchr 2.7.4 - Cargo"); + } +}