Some optimizations for component detection (#1384)

* Optimize component detection scanning

Author: narasamdya

docs/schema/manifest.schema.json

@@ -118,7 +118,8 @@
               "Spdx",
               "Vcpkg",
               "DockerReference",
-              "DotNet"
+              "DotNet",
+              "Conan"
             ]
           }
         }
@@ -345,7 +346,8 @@
             "Spdx",
             "Vcpkg",
             "DockerReference",
-            "DotNet"
+            "DotNet",
+            "Conan"
           ]
         },
         "id": {

src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs

@@ -12,11 +12,12 @@
 
 namespace Microsoft.ComponentDetection.Common.DependencyGraph;
 
+using System.Collections.ObjectModel;
 using Microsoft.Extensions.Logging;
 
 public class ComponentRecorder : IComponentRecorder
 {
-    private readonly ConcurrentBag<SingleFileComponentRecorder> singleFileRecorders = [];
+    private readonly ConcurrentDictionary<string, SingleFileComponentRecorder> singleFileRecorders = [];
 
     private readonly bool enableManualTrackingOfExplicitReferences;
 
@@ -30,7 +31,7 @@ public ComponentRecorder(ILogger logger = null, bool enableManualTrackingOfExpli
 
     public TypedComponent GetComponent(string componentId)
     {
-        return this.singleFileRecorders.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null);
+        return this.singleFileRecorders.Values.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null);
     }
 
     public IEnumerable<DetectedComponent> GetDetectedComponents()
@@ -41,25 +42,21 @@ public IEnumerable<DetectedComponent> GetDetectedComponents()
             return [];
         }
 
-        detectedComponents = this.singleFileRecorders
-            .Select(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values)
-            .SelectMany(x => x)
+        detectedComponents = this.singleFileRecorders.Values
+            .SelectMany(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values)
             .GroupBy(x => x.Component.Id)
             .Select(grouping =>
             {
                 // We pick a winner here -- any stateful props could get lost at this point. Only stateful prop still outstanding is ContainerDetails.
                 var winningDetectedComponent = grouping.First();
-                foreach (var component in grouping)
+                foreach (var component in grouping.Skip(1))
                 {
-                    foreach (var containerDetailId in component.ContainerDetailIds)
-                    {
-                        winningDetectedComponent.ContainerDetailIds.Add(containerDetailId);
-                    }
+                    winningDetectedComponent.ContainerDetailIds.UnionWith(component.ContainerDetailIds);
                 }
 
                 return winningDetectedComponent;
             })
-            .ToImmutableList();
+            .ToArray();
 
         return detectedComponents;
     }
@@ -71,11 +68,10 @@ public IEnumerable<string> GetSkippedComponents()
             return [];
         }
 
-        return this.singleFileRecorders
-            .Select(x => x.GetSkippedComponents().Keys)
-            .SelectMany(x => x)
+        return this.singleFileRecorders.Values
+            .SelectMany(x => x.GetSkippedComponents().Keys)
             .Distinct()
-            .ToImmutableList();
+            .ToArray();
     }
 
     public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string location)
@@ -85,25 +81,20 @@ public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string loc
             throw new ArgumentNullException(nameof(location));
         }
 
-        var matching = this.singleFileRecorders.FirstOrDefault(x => x.ManifestFileLocation == location);
-        if (matching == null)
-        {
-            matching = new SingleFileComponentRecorder(location, this, this.enableManualTrackingOfExplicitReferences, this.logger);
-            this.singleFileRecorders.Add(matching);
-        }
-
-        return matching;
+        return this.singleFileRecorders.GetOrAdd(location, loc => new SingleFileComponentRecorder(loc, this, this.enableManualTrackingOfExplicitReferences, this.logger));
     }
 
     public IReadOnlyDictionary<string, IDependencyGraph> GetDependencyGraphsByLocation()
     {
-        return this.singleFileRecorders.Where(x => x.DependencyGraph.HasComponents())
-            .ToImmutableDictionary(x => x.ManifestFileLocation, x => x.DependencyGraph as IDependencyGraph);
+        return new ReadOnlyDictionary<string, IDependencyGraph>(
+            this.singleFileRecorders.Values
+                .Where(x => x.DependencyGraph.HasComponents())
+                .ToDictionary(x => x.ManifestFileLocation, x => x.DependencyGraph as IDependencyGraph));
     }
 
     internal DependencyGraph GetDependencyGraphForLocation(string location)
     {
-        return this.singleFileRecorders.Single(x => x.ManifestFileLocation == location).DependencyGraph;
+        return this.singleFileRecorders[location].DependencyGraph;
     }
 
     public sealed class SingleFileComponentRecorder : ISingleFileComponentRecorder
@@ -183,16 +174,15 @@ public void RegisterUsage(
 #endif
 
             var componentId = detectedComponent.Component.Id;
-            DetectedComponent storedComponent = null;
-            lock (this.registerUsageLock)
-            {
-                storedComponent = this.detectedComponentsInternal.GetOrAdd(componentId, detectedComponent);
+            var storedComponent = this.detectedComponentsInternal.GetOrAdd(componentId, detectedComponent);
 
-                if (!string.IsNullOrWhiteSpace(targetFramework))
-                {
-                    storedComponent.TargetFrameworks.Add(targetFramework.Trim());
-                }
+            if (!string.IsNullOrWhiteSpace(targetFramework))
+            {
+                storedComponent.TargetFrameworks.Add(targetFramework.Trim());
+            }
 
+            lock (this.registerUsageLock)
+            {
                 this.AddComponentToGraph(this.ManifestFileLocation, detectedComponent, isExplicitReferencedDependency, parentComponentId, isDevelopmentDependency, dependencyScope);
             }
         }

src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs

@@ -97,7 +97,7 @@ public void AddAdditionalRelatedFile(string additionalRelatedFile)
 
     public HashSet<string> GetAdditionalRelatedFiles()
     {
-        return this.AdditionalRelatedFiles.Keys.ToImmutableHashSet().ToHashSet();
+        return this.AdditionalRelatedFiles.Keys.ToHashSet();
     }
 
     public bool HasComponents()

src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs

@@ -1,8 +1,10 @@
 namespace Microsoft.ComponentDetection.Contracts;
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Reactive.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -56,7 +58,7 @@ public abstract class FileComponentDetector : IComponentDetector
 
     public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; }
 
-    protected Dictionary<string, string> Telemetry { get; set; } = [];
+    protected ConcurrentDictionary<string, string> Telemetry { get; set; } = [];
 
     /// <summary>
     /// List of any any additional properties as key-value pairs that we would like to capture for the detector.
@@ -135,7 +137,7 @@ private async Task<IndividualDetectorScanResult> ProcessAsync(
         return new IndividualDetectorScanResult
         {
             ResultCode = ProcessingResultCode.Success,
-            AdditionalTelemetryDetails = this.Telemetry,
+            AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value),
         };
     }
 

src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs

@@ -36,7 +36,7 @@ public CargoComponent(string name, string version, string author = null, string
 
     public override ComponentType Type => ComponentType.Cargo;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl => new PackageURL("cargo", string.Empty, this.Name, this.Version, null, string.Empty);
+
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs

@@ -29,7 +29,7 @@ public ConanComponent(string name, string version, string previous, string packa
 
     public override ComponentType Type => ComponentType.Conan;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl => new PackageURL("conan", string.Empty, this.Name, this.Version, null, string.Empty);
+
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs

@@ -37,5 +37,5 @@ private CondaComponent()
 
     public override ComponentType Type => ComponentType.Conda;
 
-    public override string Id => $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}";
+    protected override string ComputeId() => $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs

@@ -22,5 +22,5 @@ public DockerImageComponent(string hash, string name = null, string tag = null)
 
     public override ComponentType Type => ComponentType.DockerImage;
 
-    public override string Id => $"{this.Name} {this.Tag} {this.Digest}";
+    protected override string ComputeId() => $"{this.Name} {this.Tag} {this.Digest}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs

@@ -36,5 +36,5 @@ public DockerReference FullReference
         }
     }
 
-    public override string Id => $"{this.Repository} {this.Tag} {this.Digest}";
+    protected override string ComputeId() => $"{this.Repository} {this.Tag} {this.Digest}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs

@@ -43,5 +43,6 @@ public DotNetComponent(string sdkVersion, string targetFramework = null, string
     /// <summary>
     /// Provides an id like `{SdkVersion} - {TargetFramework} - {ProjectType} - dotnet` where unspecified values are represented as 'unknown'.
     /// </summary>
-    public override string Id => $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}";
+    /// <returns>Id of the component.</returns>
+    protected override string ComputeId() => $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs

@@ -26,5 +26,5 @@ private GitComponent()
 
     public override ComponentType Type => ComponentType.Git;
 
-    public override string Id => $"{this.RepositoryUrl} : {this.CommitHash} - {this.Type}";
+    protected override string ComputeId() => $"{this.RepositoryUrl} : {this.CommitHash} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs

@@ -36,7 +36,7 @@ private GoComponent()
 
     public override ComponentType Type => ComponentType.Go;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
 
     public override bool Equals(object obj)
     {

src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs

@@ -36,8 +36,6 @@ public LinuxComponent(string distribution, string release, string name, string v
 
     public override ComponentType Type => ComponentType.Linux;
 
-    public override string Id => $"{this.Distribution} {this.Release} {this.Name} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl
     {
         get
@@ -62,6 +60,8 @@ public override PackageURL PackageUrl
         }
     }
 
+    protected override string ComputeId() => $"{this.Distribution} {this.Release} {this.Name} {this.Version} - {this.Type}";
+
     private bool IsUbuntu()
     {
         return this.Distribution.Equals("UBUNTU", StringComparison.OrdinalIgnoreCase);

src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs

@@ -24,7 +24,7 @@ private MavenComponent()
 
     public override ComponentType Type => ComponentType.Maven;
 
-    public override string Id => $"{this.GroupId} {this.ArtifactId} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl => new PackageURL("maven", this.GroupId, this.ArtifactId, this.Version, null, null);
+
+    protected override string ComputeId() => $"{this.GroupId} {this.ArtifactId} {this.Version} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs

@@ -28,7 +28,7 @@ public NpmComponent(string name, string version, string hash = null, NpmAuthor a
 
     public override ComponentType Type => ComponentType.Npm;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl => new PackageURL("npm", null, this.Name, this.Version, null, null);
+
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs

@@ -24,7 +24,7 @@ public NuGetComponent(string name, string version, string[] authors = null)
 
     public override ComponentType Type => ComponentType.NuGet;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl => new PackageURL("nuget", null, this.Name, this.Version, null, null);
+
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs

@@ -27,5 +27,5 @@ public OtherComponent(string name, string version, Uri downloadUrl, string hash)
 
     public override ComponentType Type => ComponentType.Other;
 
-    public override string Id => $"{this.Name} {this.Version} {this.DownloadUrl} - {this.Type}";
+    protected override string ComputeId() => $"{this.Name} {this.Version} {this.DownloadUrl} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs

@@ -33,8 +33,8 @@ public PipComponent(string name, string version, string author = null, string li
 
     public override ComponentType Type => ComponentType.Pip;
 
-    [SuppressMessage("Usage", "CA1308:Normalize String to Uppercase", Justification = "Casing cannot be overwritten.")]
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}".ToLowerInvariant();
-
     public override PackageURL PackageUrl => new PackageURL("pypi", null, this.Name, this.Version, null, null);
+
+    [SuppressMessage("Usage", "CA1308:Normalize String to Uppercase", Justification = "Casing cannot be overwritten.")]
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}".ToLowerInvariant();
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs

@@ -25,8 +25,6 @@ public PodComponent(string name, string version, string specRepo = "")
 
     public override ComponentType Type => ComponentType.Pod;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl
     {
         get
@@ -40,4 +38,6 @@ public override PackageURL PackageUrl
             return new PackageURL("cocoapods", null, this.Name, this.Version, qualifiers, null);
         }
     }
+
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs

@@ -24,7 +24,7 @@ public RubyGemsComponent(string name, string version, string source = "")
 
     public override ComponentType Type => ComponentType.RubyGems;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
-
     public override PackageURL PackageUrl => new PackageURL("gem", null, this.Name, this.Version, null, null);
+
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs

@@ -33,5 +33,5 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str
 
     public string Path { get; set; }
 
-    public override string Id => $"{this.Name}-{this.SpdxVersion}-{this.Checksum}";
+    protected override string ComputeId() => $"{this.Name}-{this.SpdxVersion}-{this.Checksum}";
 }

src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs

@@ -35,8 +35,6 @@ public SwiftComponent(string name, string version, string packageUrl, string has
 
     public override ComponentType Type => ComponentType.Swift;
 
-    public override string Id => $"{this.Name} {this.Version} - {this.Type}";
-
     // Example PackageURL -> pkg:swift/github.com/apple/swift-asn1
     // type: swift
     // namespace: github.com/apple
@@ -52,6 +50,8 @@ public SwiftComponent(string name, string version, string packageUrl, string has
         },
         subpath: null);
 
+    protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}";
+
     private string GetNamespaceFromPackageUrl()
     {
         // In the case of github.com, the namespace should contain the user/organization

src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs

@@ -14,6 +14,9 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent;
 [DebuggerDisplay("{DebuggerDisplay,nq}")]
 public abstract class TypedComponent
 {
+    [JsonIgnore]
+    private string id;
+
     internal TypedComponent()
     {
         // Reserved for deserialization.
@@ -23,7 +26,8 @@ internal TypedComponent()
     [JsonConverter(typeof(StringEnumConverter))]
     public abstract ComponentType Type { get; }
 
-    public abstract string Id { get; }
+    /// <summary>Gets the id of the component.</summary>
+    public string Id => this.id ??= this.ComputeId();
 
     public virtual PackageURL PackageUrl { get; }
 
@@ -47,4 +51,6 @@ protected string NullPropertyExceptionMessage(string propertyName, string compon
     {
         return $"Property {propertyName} of component type {componentType} is required";
     }
+
+    protected abstract string ComputeId();
 }