diff --git a/docs/schema/manifest.schema.json b/docs/schema/manifest.schema.json index f0c21bfc9..6d7623b93 100644 --- a/docs/schema/manifest.schema.json +++ b/docs/schema/manifest.schema.json @@ -118,7 +118,8 @@ "Spdx", "Vcpkg", "DockerReference", - "DotNet" + "DotNet", + "Conan" ] } } @@ -345,7 +346,8 @@ "Spdx", "Vcpkg", "DockerReference", - "DotNet" + "DotNet", + "Conan" ] }, "id": { diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index 7a9be52bf..aa72fb234 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -12,11 +12,12 @@ namespace Microsoft.ComponentDetection.Common.DependencyGraph; +using System.Collections.ObjectModel; using Microsoft.Extensions.Logging; public class ComponentRecorder : IComponentRecorder { - private readonly ConcurrentBag singleFileRecorders = []; + private readonly ConcurrentDictionary singleFileRecorders = []; private readonly bool enableManualTrackingOfExplicitReferences; @@ -30,7 +31,7 @@ public ComponentRecorder(ILogger logger = null, bool enableManualTrackingOfExpli public TypedComponent GetComponent(string componentId) { - return this.singleFileRecorders.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null); + return this.singleFileRecorders.Values.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null); } public IEnumerable GetDetectedComponents() @@ -41,25 +42,21 @@ public IEnumerable GetDetectedComponents() return []; } - detectedComponents = this.singleFileRecorders - .Select(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values) - .SelectMany(x => x) + detectedComponents = this.singleFileRecorders.Values + .SelectMany(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values) .GroupBy(x => x.Component.Id) .Select(grouping => { // We pick a winner here -- any stateful props could get lost at this point. Only stateful prop still outstanding is ContainerDetails. var winningDetectedComponent = grouping.First(); - foreach (var component in grouping) + foreach (var component in grouping.Skip(1)) { - foreach (var containerDetailId in component.ContainerDetailIds) - { - winningDetectedComponent.ContainerDetailIds.Add(containerDetailId); - } + winningDetectedComponent.ContainerDetailIds.UnionWith(component.ContainerDetailIds); } return winningDetectedComponent; }) - .ToImmutableList(); + .ToArray(); return detectedComponents; } @@ -71,11 +68,10 @@ public IEnumerable GetSkippedComponents() return []; } - return this.singleFileRecorders - .Select(x => x.GetSkippedComponents().Keys) - .SelectMany(x => x) + return this.singleFileRecorders.Values + .SelectMany(x => x.GetSkippedComponents().Keys) .Distinct() - .ToImmutableList(); + .ToArray(); } public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string location) @@ -85,25 +81,20 @@ public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string loc throw new ArgumentNullException(nameof(location)); } - var matching = this.singleFileRecorders.FirstOrDefault(x => x.ManifestFileLocation == location); - if (matching == null) - { - matching = new SingleFileComponentRecorder(location, this, this.enableManualTrackingOfExplicitReferences, this.logger); - this.singleFileRecorders.Add(matching); - } - - return matching; + return this.singleFileRecorders.GetOrAdd(location, loc => new SingleFileComponentRecorder(loc, this, this.enableManualTrackingOfExplicitReferences, this.logger)); } public IReadOnlyDictionary GetDependencyGraphsByLocation() { - return this.singleFileRecorders.Where(x => x.DependencyGraph.HasComponents()) - .ToImmutableDictionary(x => x.ManifestFileLocation, x => x.DependencyGraph as IDependencyGraph); + return new ReadOnlyDictionary( + this.singleFileRecorders.Values + .Where(x => x.DependencyGraph.HasComponents()) + .ToDictionary(x => x.ManifestFileLocation, x => x.DependencyGraph as IDependencyGraph)); } internal DependencyGraph GetDependencyGraphForLocation(string location) { - return this.singleFileRecorders.Single(x => x.ManifestFileLocation == location).DependencyGraph; + return this.singleFileRecorders[location].DependencyGraph; } public sealed class SingleFileComponentRecorder : ISingleFileComponentRecorder @@ -183,16 +174,15 @@ public void RegisterUsage( #endif var componentId = detectedComponent.Component.Id; - DetectedComponent storedComponent = null; - lock (this.registerUsageLock) - { - storedComponent = this.detectedComponentsInternal.GetOrAdd(componentId, detectedComponent); + var storedComponent = this.detectedComponentsInternal.GetOrAdd(componentId, detectedComponent); - if (!string.IsNullOrWhiteSpace(targetFramework)) - { - storedComponent.TargetFrameworks.Add(targetFramework.Trim()); - } + if (!string.IsNullOrWhiteSpace(targetFramework)) + { + storedComponent.TargetFrameworks.Add(targetFramework.Trim()); + } + lock (this.registerUsageLock) + { this.AddComponentToGraph(this.ManifestFileLocation, detectedComponent, isExplicitReferencedDependency, parentComponentId, isDevelopmentDependency, dependencyScope); } } diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index 7fbbcf6c6..cff395fc2 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -97,7 +97,7 @@ public void AddAdditionalRelatedFile(string additionalRelatedFile) public HashSet GetAdditionalRelatedFiles() { - return this.AdditionalRelatedFiles.Keys.ToImmutableHashSet().ToHashSet(); + return this.AdditionalRelatedFiles.Keys.ToHashSet(); } public bool HasComponents() diff --git a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs index 9714d095c..cb24dd944 100644 --- a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs @@ -1,8 +1,10 @@ namespace Microsoft.ComponentDetection.Contracts; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Reactive.Linq; using System.Threading; using System.Threading.Tasks; @@ -56,7 +58,7 @@ public abstract class FileComponentDetector : IComponentDetector public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; } - protected Dictionary Telemetry { get; set; } = []; + protected ConcurrentDictionary Telemetry { get; set; } = []; /// /// List of any any additional properties as key-value pairs that we would like to capture for the detector. @@ -135,7 +137,7 @@ private async Task ProcessAsync( return new IndividualDetectorScanResult { ResultCode = ProcessingResultCode.Success, - AdditionalTelemetryDetails = this.Telemetry, + AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), }; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs index 7ac040943..a59c95e88 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs @@ -36,7 +36,7 @@ public CargoComponent(string name, string version, string author = null, string public override ComponentType Type => ComponentType.Cargo; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("cargo", string.Empty, this.Name, this.Version, null, string.Empty); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs index cbe5abdda..9c26e66b4 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs @@ -29,7 +29,7 @@ public ConanComponent(string name, string version, string previous, string packa public override ComponentType Type => ComponentType.Conan; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("conan", string.Empty, this.Name, this.Version, null, string.Empty); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs index 04edb3f76..6d46f111d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs @@ -37,5 +37,5 @@ private CondaComponent() public override ComponentType Type => ComponentType.Conda; - public override string Id => $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}"; + protected override string ComputeId() => $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs index f24b90b0c..69c6ea7be 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs @@ -22,5 +22,5 @@ public DockerImageComponent(string hash, string name = null, string tag = null) public override ComponentType Type => ComponentType.DockerImage; - public override string Id => $"{this.Name} {this.Tag} {this.Digest}"; + protected override string ComputeId() => $"{this.Name} {this.Tag} {this.Digest}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs index 62ce707d1..224087152 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs @@ -36,5 +36,5 @@ public DockerReference FullReference } } - public override string Id => $"{this.Repository} {this.Tag} {this.Digest}"; + protected override string ComputeId() => $"{this.Repository} {this.Tag} {this.Digest}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs index edafc3d61..c3f9dc611 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs @@ -43,5 +43,6 @@ public DotNetComponent(string sdkVersion, string targetFramework = null, string /// /// Provides an id like `{SdkVersion} - {TargetFramework} - {ProjectType} - dotnet` where unspecified values are represented as 'unknown'. /// - public override string Id => $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}"; + /// Id of the component. + protected override string ComputeId() => $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs index aeee1683e..2ed88e3e1 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs @@ -26,5 +26,5 @@ private GitComponent() public override ComponentType Type => ComponentType.Git; - public override string Id => $"{this.RepositoryUrl} : {this.CommitHash} - {this.Type}"; + protected override string ComputeId() => $"{this.RepositoryUrl} : {this.CommitHash} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs index 6246ef702..e09fc32e9 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs @@ -36,7 +36,7 @@ private GoComponent() public override ComponentType Type => ComponentType.Go; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; public override bool Equals(object obj) { diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs index 23ccd2db9..239c89854 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs @@ -36,8 +36,6 @@ public LinuxComponent(string distribution, string release, string name, string v public override ComponentType Type => ComponentType.Linux; - public override string Id => $"{this.Distribution} {this.Release} {this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl { get @@ -62,6 +60,8 @@ public override PackageURL PackageUrl } } + protected override string ComputeId() => $"{this.Distribution} {this.Release} {this.Name} {this.Version} - {this.Type}"; + private bool IsUbuntu() { return this.Distribution.Equals("UBUNTU", StringComparison.OrdinalIgnoreCase); diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs index b02730658..75ed72c5b 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs @@ -24,7 +24,7 @@ private MavenComponent() public override ComponentType Type => ComponentType.Maven; - public override string Id => $"{this.GroupId} {this.ArtifactId} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("maven", this.GroupId, this.ArtifactId, this.Version, null, null); + + protected override string ComputeId() => $"{this.GroupId} {this.ArtifactId} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs index e758eedae..67bec60f5 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs @@ -28,7 +28,7 @@ public NpmComponent(string name, string version, string hash = null, NpmAuthor a public override ComponentType Type => ComponentType.Npm; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("npm", null, this.Name, this.Version, null, null); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs index dad787527..4c1e578e8 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs @@ -24,7 +24,7 @@ public NuGetComponent(string name, string version, string[] authors = null) public override ComponentType Type => ComponentType.NuGet; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("nuget", null, this.Name, this.Version, null, null); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs index 7b8013090..e76b3cb72 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs @@ -27,5 +27,5 @@ public OtherComponent(string name, string version, Uri downloadUrl, string hash) public override ComponentType Type => ComponentType.Other; - public override string Id => $"{this.Name} {this.Version} {this.DownloadUrl} - {this.Type}"; + protected override string ComputeId() => $"{this.Name} {this.Version} {this.DownloadUrl} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs index 2eebd0dea..7efe80901 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs @@ -33,8 +33,8 @@ public PipComponent(string name, string version, string author = null, string li public override ComponentType Type => ComponentType.Pip; - [SuppressMessage("Usage", "CA1308:Normalize String to Uppercase", Justification = "Casing cannot be overwritten.")] - public override string Id => $"{this.Name} {this.Version} - {this.Type}".ToLowerInvariant(); - public override PackageURL PackageUrl => new PackageURL("pypi", null, this.Name, this.Version, null, null); + + [SuppressMessage("Usage", "CA1308:Normalize String to Uppercase", Justification = "Casing cannot be overwritten.")] + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}".ToLowerInvariant(); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs index 63b00d3cb..bb690a782 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs @@ -25,8 +25,6 @@ public PodComponent(string name, string version, string specRepo = "") public override ComponentType Type => ComponentType.Pod; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl { get @@ -40,4 +38,6 @@ public override PackageURL PackageUrl return new PackageURL("cocoapods", null, this.Name, this.Version, qualifiers, null); } } + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs index e1d9845b3..a02657203 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs @@ -24,7 +24,7 @@ public RubyGemsComponent(string name, string version, string source = "") public override ComponentType Type => ComponentType.RubyGems; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("gem", null, this.Name, this.Version, null, null); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs index c4328bfb8..5d00c5198 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs @@ -33,5 +33,5 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str public string Path { get; set; } - public override string Id => $"{this.Name}-{this.SpdxVersion}-{this.Checksum}"; + protected override string ComputeId() => $"{this.Name}-{this.SpdxVersion}-{this.Checksum}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs index d871525bb..2203574fe 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs @@ -35,8 +35,6 @@ public SwiftComponent(string name, string version, string packageUrl, string has public override ComponentType Type => ComponentType.Swift; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; - // Example PackageURL -> pkg:swift/github.com/apple/swift-asn1 // type: swift // namespace: github.com/apple @@ -52,6 +50,8 @@ public SwiftComponent(string name, string version, string packageUrl, string has }, subpath: null); + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; + private string GetNamespaceFromPackageUrl() { // In the case of github.com, the namespace should contain the user/organization diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs index c5930414a..23d9d4313 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs @@ -14,6 +14,9 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; [DebuggerDisplay("{DebuggerDisplay,nq}")] public abstract class TypedComponent { + [JsonIgnore] + private string id; + internal TypedComponent() { // Reserved for deserialization. @@ -23,7 +26,8 @@ internal TypedComponent() [JsonConverter(typeof(StringEnumConverter))] public abstract ComponentType Type { get; } - public abstract string Id { get; } + /// Gets the id of the component. + public string Id => this.id ??= this.ComputeId(); public virtual PackageURL PackageUrl { get; } @@ -47,4 +51,6 @@ protected string NullPropertyExceptionMessage(string propertyName, string compon { return $"Property {propertyName} of component type {componentType} is required"; } + + protected abstract string ComputeId(); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs index 64105d404..e62457650 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -38,26 +38,6 @@ public VcpkgComponent(string spdxid, string name, string version, string triplet public override ComponentType Type => ComponentType.Vcpkg; - public override string Id - { - get - { - var componentLocationPrefix = string.Empty; - if (!string.IsNullOrWhiteSpace(this.DownloadLocation) && !this.DownloadLocation.Trim().Equals("NONE", System.StringComparison.InvariantCultureIgnoreCase)) - { - componentLocationPrefix = $"{this.DownloadLocation} : "; - } - - var componentPortVersionSuffix = " "; - if (this.PortVersion > 0) - { - componentPortVersionSuffix = $"#{this.PortVersion} "; - } - - return $"{componentLocationPrefix}{this.Name} {this.Version}{componentPortVersionSuffix}- {this.Type}"; - } - } - public override PackageURL PackageUrl { get @@ -76,4 +56,21 @@ public override PackageURL PackageUrl } } } + + protected override string ComputeId() + { + var componentLocationPrefix = string.Empty; + if (!string.IsNullOrWhiteSpace(this.DownloadLocation) && !this.DownloadLocation.Trim().Equals("NONE", System.StringComparison.InvariantCultureIgnoreCase)) + { + componentLocationPrefix = $"{this.DownloadLocation} : "; + } + + var componentPortVersionSuffix = " "; + if (this.PortVersion > 0) + { + componentPortVersionSuffix = $"#{this.PortVersion} "; + } + + return $"{componentLocationPrefix}{this.Name} {this.Version}{componentPortVersionSuffix}- {this.Type}"; + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index c2110e2a1..3a6d937bf 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -36,7 +36,7 @@ public YarnLockComponentDetector( public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Npm]; - public override int Version => 8; + public override int Version => 9; public override IEnumerable Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.Npm)]; @@ -44,6 +44,9 @@ public YarnLockComponentDetector( /// "Package" is a more common substring, enclose it with \ to verify it is a folder. protected override IList SkippedFolders => ["node_modules", "pnpm-store", "\\package\\"]; + /// + protected override bool EnableParallelism { get; set; } = true; + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; @@ -243,9 +246,9 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRec yarnRoots.Add(entry); var locationMapDictonaryKey = this.GetLocationMapKey(name, version.Key); - if (workspaceDependencyVsLocationMap.ContainsKey(locationMapDictonaryKey)) + if (workspaceDependencyVsLocationMap.TryGetValue(locationMapDictonaryKey, out location)) { - entry.Location = workspaceDependencyVsLocationMap[locationMapDictonaryKey]; + entry.Location = location; } } } @@ -324,10 +327,7 @@ private void ProcessWorkspaceDependency(IDictionary workspaceDependencyVsLocationMap, string streamLocation, string dependencyName, string dependencyVersion) { var locationMapDictionaryKey = this.GetLocationMapKey(dependencyName, dependencyVersion); - if (!workspaceDependencyVsLocationMap.ContainsKey(locationMapDictionaryKey)) - { - workspaceDependencyVsLocationMap[locationMapDictionaryKey] = streamLocation; - } + workspaceDependencyVsLocationMap.TryAdd(locationMapDictionaryKey, streamLocation); } private string GetLocationMapKey(string dependencyName, string dependencyVersion) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 92b3154af..248c401d1 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -117,8 +117,8 @@ public async Task ProcessDetectorsAsync( record.DetectorId = detector.Id; record.DetectedComponentCount = detectedComponents.Count(); var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation().Values; - record.ExplicitlyReferencedComponentCount = dependencyGraphs.Select(dependencyGraph => dependencyGraph.GetAllExplicitlyReferencedComponents()) - .SelectMany(x => x) + record.ExplicitlyReferencedComponentCount = dependencyGraphs + .SelectMany(dependencyGraph => dependencyGraph.GetAllExplicitlyReferencedComponents()) .Distinct() .Count(); diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index ceed83e84..e9b335ff1 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -13,7 +13,6 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Orchestrator.Commands; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; public class DefaultGraphTranslationService : IGraphTranslationService { @@ -101,12 +100,15 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn foreach (var component in detectedComponents) { // clone custom locations and make them relative to root. - var declaredRawFilePaths = component.FilePaths ?? []; - var componentCustomLocations = JsonConvert.DeserializeObject>(JsonConvert.SerializeObject(declaredRawFilePaths)); + var componentCustomLocations = component.FilePaths ?? []; if (updateLocations) { - component.FilePaths?.Clear(); + if (component.FilePaths != null) + { + componentCustomLocations = [.. component.FilePaths]; + component.FilePaths?.Clear(); + } } // Information about each component is relative to all of the graphs it is present in, so we take all graphs containing a given component and apply the graph data. @@ -124,13 +126,13 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn component.DependencyScope = DependencyScopeComparer.GetMergedDependencyScope(component.DependencyScope, dependencyGraph.GetDependencyScope(component.Component.Id)); component.DetectedBy = detector; - // Return in a format that allows us to add the additional files for the components - var locations = dependencyGraph.GetAdditionalRelatedFiles(); - // Experiments uses this service to build the dependency graph for analysis. In this case, we do not want to update the locations of the component. // Updating the locations of the component will propogate to the final depenendcy graph and cause the graph to be incorrect. if (updateLocations) { + // Return in a format that allows us to add the additional files for the components + var locations = dependencyGraph.GetAdditionalRelatedFiles(); + // graph authoritatively stores the location of the component locations.Add(location); @@ -261,7 +263,7 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root // Make relative Uri needs a trailing separator to ensure that we turn "directory we are scanning" into "/" var rootDirectoryFullName = rootDirectory.FullName; - if (!rootDirectory.FullName.EndsWith(Path.DirectorySeparatorChar.ToString()) && !rootDirectory.FullName.EndsWith(Path.AltDirectorySeparatorChar.ToString())) + if (!rootDirectory.FullName.EndsWith(Path.DirectorySeparatorChar) && !rootDirectory.FullName.EndsWith(Path.AltDirectorySeparatorChar)) { rootDirectoryFullName += Path.DirectorySeparatorChar; } @@ -270,20 +272,19 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root var relativePathSet = new HashSet(); foreach (var path in filePaths) { - try + if (!Uri.TryCreate(path, UriKind.Absolute, out var uriPath)) { - var relativePath = rootUri.MakeRelativeUri(new Uri(path)).ToString(); - if (!relativePath.StartsWith('/')) - { - relativePath = "/" + relativePath; - } - - relativePathSet.Add(relativePath); + logger.LogDebug("The path: {Path} is not a valid absolute path and so could not be resolved relative to the root {RootUri}", path, rootUri); + continue; } - catch (UriFormatException e) + + var relativePath = rootUri.MakeRelativeUri(uriPath).ToString(); + if (!relativePath.StartsWith('/')) { - logger.LogDebug(e, "The path: {Path} could not be resolved relative to the root {RootUri}", path, rootUri); + relativePath = "/" + relativePath; } + + relativePathSet.Add(relativePath); } return relativePathSet; diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 b/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 index 3c490884c..9e9dcd0d9 100755 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 @@ -58,6 +58,11 @@ function main() $env:GITHUB_NEW_ARTIFACTS_DIR = $output $env:ALLOWED_TIME_DRIFT_RATIO = "0.75" + if ([string]::IsNullOrEmpty($env:GITHUB_WORKSPACE)) { + $env:GITHUB_WORKSPACE = $repoPath + Write-Host "Setting GITHUB_WORKSPACE environment variable to $repoPath" + } + Write-Progress "Executing verification tests....." Set-Location ((Get-Item $repoPath).FullName + "\test\Microsoft.ComponentDetection.VerificationTests\") dotnet restore