From a5ad73c14ad4a4d58a7fe31def098f8bcc274cb5 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Fri, 11 Apr 2025 16:58:55 -0700 Subject: [PATCH 01/17] Attempt to optimize CG --- .../DependencyGraph/ComponentRecorder.cs | 49 +++++++++---------- .../DependencyGraph/DependencyGraph.cs | 2 +- .../DefaultGraphTranslationService.cs | 12 +++-- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index 7a9be52bf..95e6c0f0b 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -12,11 +12,12 @@ namespace Microsoft.ComponentDetection.Common.DependencyGraph; +using System.Collections.ObjectModel; using Microsoft.Extensions.Logging; public class ComponentRecorder : IComponentRecorder { - private readonly ConcurrentBag singleFileRecorders = []; + private readonly ConcurrentDictionary singleFileRecorders = []; private readonly bool enableManualTrackingOfExplicitReferences; @@ -30,7 +31,7 @@ public ComponentRecorder(ILogger logger = null, bool enableManualTrackingOfExpli public TypedComponent GetComponent(string componentId) { - return this.singleFileRecorders.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null); + return this.singleFileRecorders.Values.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null); } public IEnumerable GetDetectedComponents() @@ -41,25 +42,21 @@ public IEnumerable GetDetectedComponents() return []; } - detectedComponents = this.singleFileRecorders - .Select(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values) - .SelectMany(x => x) + detectedComponents = this.singleFileRecorders.Values + .SelectMany(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values) .GroupBy(x => x.Component.Id) .Select(grouping => { // We pick a winner here -- any stateful props could get lost at this point. Only stateful prop still outstanding is ContainerDetails. var winningDetectedComponent = grouping.First(); - foreach (var component in grouping) + foreach (var component in grouping.Skip(1)) { - foreach (var containerDetailId in component.ContainerDetailIds) - { - winningDetectedComponent.ContainerDetailIds.Add(containerDetailId); - } + winningDetectedComponent.ContainerDetailIds.UnionWith(component.ContainerDetailIds); } return winningDetectedComponent; }) - .ToImmutableList(); + .ToArray(); return detectedComponents; } @@ -71,11 +68,10 @@ public IEnumerable GetSkippedComponents() return []; } - return this.singleFileRecorders - .Select(x => x.GetSkippedComponents().Keys) - .SelectMany(x => x) + return this.singleFileRecorders.Values + .SelectMany(x => x.GetSkippedComponents().Keys) .Distinct() - .ToImmutableList(); + .ToArray(); } public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string location) @@ -85,25 +81,26 @@ public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string loc throw new ArgumentNullException(nameof(location)); } - var matching = this.singleFileRecorders.FirstOrDefault(x => x.ManifestFileLocation == location); - if (matching == null) - { - matching = new SingleFileComponentRecorder(location, this, this.enableManualTrackingOfExplicitReferences, this.logger); - this.singleFileRecorders.Add(matching); - } - - return matching; + return this.singleFileRecorders.GetOrAdd(location, loc => new SingleFileComponentRecorder(loc, this, this.enableManualTrackingOfExplicitReferences, this.logger)); } public IReadOnlyDictionary GetDependencyGraphsByLocation() { - return this.singleFileRecorders.Where(x => x.DependencyGraph.HasComponents()) - .ToImmutableDictionary(x => x.ManifestFileLocation, x => x.DependencyGraph as IDependencyGraph); + return new ReadOnlyDictionary( + this.singleFileRecorders.Values + .Where(x => x.DependencyGraph.HasComponents()) + .ToDictionary(x => x.ManifestFileLocation, x => x.DependencyGraph as IDependencyGraph)); } internal DependencyGraph GetDependencyGraphForLocation(string location) { - return this.singleFileRecorders.Single(x => x.ManifestFileLocation == location).DependencyGraph; + if (this.singleFileRecorders.TryGetValue(location, out var singleFileRecorder)) + { + return singleFileRecorder.DependencyGraph; + } + + // TODO: Should we throw an exception here instead of returning null? + return null; } public sealed class SingleFileComponentRecorder : ISingleFileComponentRecorder diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index 7fbbcf6c6..cff395fc2 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -97,7 +97,7 @@ public void AddAdditionalRelatedFile(string additionalRelatedFile) public HashSet GetAdditionalRelatedFiles() { - return this.AdditionalRelatedFiles.Keys.ToImmutableHashSet().ToHashSet(); + return this.AdditionalRelatedFiles.Keys.ToHashSet(); } public bool HasComponents() diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index ceed83e84..112eb7127 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -13,7 +13,6 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Orchestrator.Commands; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; public class DefaultGraphTranslationService : IGraphTranslationService { @@ -101,12 +100,15 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn foreach (var component in detectedComponents) { // clone custom locations and make them relative to root. - var declaredRawFilePaths = component.FilePaths ?? []; - var componentCustomLocations = JsonConvert.DeserializeObject>(JsonConvert.SerializeObject(declaredRawFilePaths)); + var componentCustomLocations = component.FilePaths ?? []; if (updateLocations) { - component.FilePaths?.Clear(); + if (component.FilePaths != null) + { + componentCustomLocations = [.. component.FilePaths]; + component.FilePaths?.Clear(); + } } // Information about each component is relative to all of the graphs it is present in, so we take all graphs containing a given component and apply the graph data. @@ -261,7 +263,7 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root // Make relative Uri needs a trailing separator to ensure that we turn "directory we are scanning" into "/" var rootDirectoryFullName = rootDirectory.FullName; - if (!rootDirectory.FullName.EndsWith(Path.DirectorySeparatorChar.ToString()) && !rootDirectory.FullName.EndsWith(Path.AltDirectorySeparatorChar.ToString())) + if (!rootDirectory.FullName.EndsWith(Path.DirectorySeparatorChar) && !rootDirectory.FullName.EndsWith(Path.AltDirectorySeparatorChar)) { rootDirectoryFullName += Path.DirectorySeparatorChar; } From e116c5b259c1848157957dd6966fde723a5a1fe1 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Sat, 12 Apr 2025 14:05:03 -0700 Subject: [PATCH 02/17] Turn off logging in make relative --- .../DefaultGraphTranslationService.cs | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index 112eb7127..fb7399741 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -272,20 +272,34 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root var relativePathSet = new HashSet(); foreach (var path in filePaths) { - try + if (!Uri.TryCreate(path, UriKind.Absolute, out var uriPath)) { - var relativePath = rootUri.MakeRelativeUri(new Uri(path)).ToString(); - if (!relativePath.StartsWith('/')) - { - relativePath = "/" + relativePath; - } - - relativePathSet.Add(relativePath); + // logger.LogDebug("The path: {Path} is not a valid absolute path", path); + continue; } - catch (UriFormatException e) + + var relativePath = rootUri.MakeRelativeUri(uriPath).ToString(); + if (!relativePath.StartsWith('/')) { - logger.LogDebug(e, "The path: {Path} could not be resolved relative to the root {RootUri}", path, rootUri); + relativePath = "/" + relativePath; } + + relativePathSet.Add(relativePath); + + // try + // { + // var relativePath = rootUri.MakeRelativeUri(new Uri(path)).ToString(); + // if (!relativePath.StartsWith('/')) + // { + // relativePath = "/" + relativePath; + // } + // + // relativePathSet.Add(relativePath); + // } + // catch (UriFormatException e) + // { + // logger.LogDebug(e, "The path: {Path} could not be resolved relative to the root {RootUri}", path, rootUri); + // } } return relativePathSet; From dc9be9544c01c882b6280991e3b4066b92902ff1 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Sat, 12 Apr 2025 14:21:06 -0700 Subject: [PATCH 03/17] Turn off Yarn logger --- .../yarn/Parsers/YarnLockParser.cs | 6 ++-- .../yarn/YarnLockComponentDetector.cs | 28 ++++++++++--------- .../DefaultGraphTranslationService.cs | 17 +---------- 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs index eb5bfc77e..1159c49d2 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs @@ -64,13 +64,13 @@ public YarnLockFile Parse(ISingleFileComponentRecorder singleFileComponentRecord if (string.IsNullOrWhiteSpace(yarnEntry.Name)) { - logger.LogWarning("Failed to read a name for block {BlockTitle}. The entry will be skipped.", block.Title); + // logger.LogWarning("Failed to read a name for block {BlockTitle}. The entry will be skipped.", block.Title); continue; } if (!block.Values.TryGetValue(VersionString, out var version)) { - logger.LogWarning("Failed to read a version for {YarnEntryName}. The entry will be skipped.", yarnEntry.Name); + // logger.LogWarning("Failed to read a version for {YarnEntryName}. The entry will be skipped.", yarnEntry.Name); singleFileComponentRecorder.RegisterPackageParseFailure(yarnEntry.Name); continue; } @@ -138,7 +138,7 @@ private Func GenerateBlockTitleNormalizer(YarnBlock block) var versionValue = block.Values.FirstOrDefault(x => string.Equals(x.Key, VersionString, StringComparison.OrdinalIgnoreCase)); if (default(KeyValuePair).Equals(versionValue)) { - this.logger.LogWarning("Block without version detected"); + // this.logger.LogWarning("Block without version detected"); return blockTitleMember; } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index c2110e2a1..80dee723e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -52,11 +52,11 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var skippedFolder = this.SkippedFolders.FirstOrDefault(folder => file.Location.Contains(folder)); if (!string.IsNullOrEmpty(skippedFolder)) { - this.Logger.LogInformation("Yarn.Lock file {YarnLockLocation} was found in a {SkippedFolder} folder and will be skipped.", file.Location, skippedFolder); + // this.Logger.LogInformation("Yarn.Lock file {YarnLockLocation} was found in a {SkippedFolder} folder and will be skipped.", file.Location, skippedFolder); return; } - this.Logger.LogInformation("Processing file {YarnLockLocation}", file.Location); + /* this.Logger.LogInformation("Processing file {YarnLockLocation}", file.Location); */ try { @@ -85,10 +85,11 @@ private void DetectComponents(YarnLockFile file, string location, ISingleFileCom { var key = $"{entry.Name}@{satisfiedVersion}"; var addSuccessful = yarnPackages.TryAdd(key, entry); - if (!addSuccessful) - { - this.Logger.LogWarning("Found duplicate entry {Key} in {Location}", key, location); - } + + // if (!addSuccessful) + // { + // this.Logger.LogWarning("Found duplicate entry {Key} in {Location}", key, location); + // } } } @@ -174,10 +175,11 @@ private void ParseTreeWithAssignedRoot(YarnEntry root, Dictionary yarnWorkspaces, DirectoryInf foreach (var stream in componentStreams) { - this.Logger.LogInformation("{ComponentLocation} found for workspace {WorkspacePattern}", stream.Location, workspacePattern); + // this.Logger.LogInformation("{ComponentLocation} found for workspace {WorkspacePattern}", stream.Location, workspacePattern); var combinedDependencies = NpmComponentUtilities.TryGetAllPackageJsonDependencies(stream.Stream, out _); foreach (var dependency in combinedDependencies) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index fb7399741..066bfabda 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -274,7 +274,7 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root { if (!Uri.TryCreate(path, UriKind.Absolute, out var uriPath)) { - // logger.LogDebug("The path: {Path} is not a valid absolute path", path); + logger.LogDebug("The path: {Path} is not a valid absolute path", path); continue; } @@ -285,21 +285,6 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root } relativePathSet.Add(relativePath); - - // try - // { - // var relativePath = rootUri.MakeRelativeUri(new Uri(path)).ToString(); - // if (!relativePath.StartsWith('/')) - // { - // relativePath = "/" + relativePath; - // } - // - // relativePathSet.Add(relativePath); - // } - // catch (UriFormatException e) - // { - // logger.LogDebug(e, "The path: {Path} could not be resolved relative to the root {RootUri}", path, rootUri); - // } } return relativePathSet; From 66c4ad04673da970bfe98b27c6f458f072f36b88 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Sat, 12 Apr 2025 14:25:23 -0700 Subject: [PATCH 04/17] Revert "Turn off Yarn logger" This reverts commit dc9be9544c01c882b6280991e3b4066b92902ff1. --- .../yarn/Parsers/YarnLockParser.cs | 6 ++-- .../yarn/YarnLockComponentDetector.cs | 28 +++++++++---------- .../DefaultGraphTranslationService.cs | 17 ++++++++++- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs index 1159c49d2..eb5bfc77e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs @@ -64,13 +64,13 @@ public YarnLockFile Parse(ISingleFileComponentRecorder singleFileComponentRecord if (string.IsNullOrWhiteSpace(yarnEntry.Name)) { - // logger.LogWarning("Failed to read a name for block {BlockTitle}. The entry will be skipped.", block.Title); + logger.LogWarning("Failed to read a name for block {BlockTitle}. The entry will be skipped.", block.Title); continue; } if (!block.Values.TryGetValue(VersionString, out var version)) { - // logger.LogWarning("Failed to read a version for {YarnEntryName}. The entry will be skipped.", yarnEntry.Name); + logger.LogWarning("Failed to read a version for {YarnEntryName}. The entry will be skipped.", yarnEntry.Name); singleFileComponentRecorder.RegisterPackageParseFailure(yarnEntry.Name); continue; } @@ -138,7 +138,7 @@ private Func GenerateBlockTitleNormalizer(YarnBlock block) var versionValue = block.Values.FirstOrDefault(x => string.Equals(x.Key, VersionString, StringComparison.OrdinalIgnoreCase)); if (default(KeyValuePair).Equals(versionValue)) { - // this.logger.LogWarning("Block without version detected"); + this.logger.LogWarning("Block without version detected"); return blockTitleMember; } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index 80dee723e..c2110e2a1 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -52,11 +52,11 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var skippedFolder = this.SkippedFolders.FirstOrDefault(folder => file.Location.Contains(folder)); if (!string.IsNullOrEmpty(skippedFolder)) { - // this.Logger.LogInformation("Yarn.Lock file {YarnLockLocation} was found in a {SkippedFolder} folder and will be skipped.", file.Location, skippedFolder); + this.Logger.LogInformation("Yarn.Lock file {YarnLockLocation} was found in a {SkippedFolder} folder and will be skipped.", file.Location, skippedFolder); return; } - /* this.Logger.LogInformation("Processing file {YarnLockLocation}", file.Location); */ + this.Logger.LogInformation("Processing file {YarnLockLocation}", file.Location); try { @@ -85,11 +85,10 @@ private void DetectComponents(YarnLockFile file, string location, ISingleFileCom { var key = $"{entry.Name}@{satisfiedVersion}"; var addSuccessful = yarnPackages.TryAdd(key, entry); - - // if (!addSuccessful) - // { - // this.Logger.LogWarning("Found duplicate entry {Key} in {Location}", key, location); - // } + if (!addSuccessful) + { + this.Logger.LogWarning("Found duplicate entry {Key} in {Location}", key, location); + } } } @@ -175,11 +174,10 @@ private void ParseTreeWithAssignedRoot(YarnEntry root, Dictionary yarnWorkspaces, DirectoryInf foreach (var stream in componentStreams) { - // this.Logger.LogInformation("{ComponentLocation} found for workspace {WorkspacePattern}", stream.Location, workspacePattern); + this.Logger.LogInformation("{ComponentLocation} found for workspace {WorkspacePattern}", stream.Location, workspacePattern); var combinedDependencies = NpmComponentUtilities.TryGetAllPackageJsonDependencies(stream.Stream, out _); foreach (var dependency in combinedDependencies) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index 066bfabda..fb7399741 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -274,7 +274,7 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root { if (!Uri.TryCreate(path, UriKind.Absolute, out var uriPath)) { - logger.LogDebug("The path: {Path} is not a valid absolute path", path); + // logger.LogDebug("The path: {Path} is not a valid absolute path", path); continue; } @@ -285,6 +285,21 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root } relativePathSet.Add(relativePath); + + // try + // { + // var relativePath = rootUri.MakeRelativeUri(new Uri(path)).ToString(); + // if (!relativePath.StartsWith('/')) + // { + // relativePath = "/" + relativePath; + // } + // + // relativePathSet.Add(relativePath); + // } + // catch (UriFormatException e) + // { + // logger.LogDebug(e, "The path: {Path} could not be resolved relative to the root {RootUri}", path, rootUri); + // } } return relativePathSet; From 840bff94757662509389d2f55cb45e7921fc1459 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Sat, 12 Apr 2025 14:27:20 -0700 Subject: [PATCH 05/17] Restore --- .../DefaultGraphTranslationService.cs | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index fb7399741..b78f53f45 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -274,7 +274,7 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root { if (!Uri.TryCreate(path, UriKind.Absolute, out var uriPath)) { - // logger.LogDebug("The path: {Path} is not a valid absolute path", path); + logger.LogDebug("The path: {Path} is not a valid absolute path and so could not be resolved relative to the root {RootUri}", path, rootUri); continue; } @@ -285,21 +285,6 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root } relativePathSet.Add(relativePath); - - // try - // { - // var relativePath = rootUri.MakeRelativeUri(new Uri(path)).ToString(); - // if (!relativePath.StartsWith('/')) - // { - // relativePath = "/" + relativePath; - // } - // - // relativePathSet.Add(relativePath); - // } - // catch (UriFormatException e) - // { - // logger.LogDebug(e, "The path: {Path} could not be resolved relative to the root {RootUri}", path, rootUri); - // } } return relativePathSet; From b7a2c92ad6227012e618636a93b06048d30cf22c Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Wed, 16 Apr 2025 17:50:59 -0700 Subject: [PATCH 06/17] Cache component ids to avoid repeated boxing --- .../TypedComponent/CargoComponent.cs | 2 +- .../TypedComponent/ConanComponent.cs | 2 +- .../TypedComponent/CondaComponent.cs | 2 +- .../TypedComponent/DockerImageComponent.cs | 2 +- .../TypedComponent/DockerReferenceComponent.cs | 2 +- .../TypedComponent/DotNetComponent.cs | 2 +- .../TypedComponent/GitComponent.cs | 2 +- .../TypedComponent/GoComponent.cs | 2 +- .../TypedComponent/LinuxComponent.cs | 2 +- .../TypedComponent/MavenComponent.cs | 2 +- .../TypedComponent/NpmComponent.cs | 2 +- .../TypedComponent/NugetComponent.cs | 2 +- .../TypedComponent/OtherComponent.cs | 2 +- .../TypedComponent/PipComponent.cs | 2 +- .../TypedComponent/PodComponent.cs | 2 +- .../TypedComponent/RubyGemsComponent.cs | 2 +- .../TypedComponent/SpdxComponent.cs | 4 ++-- .../TypedComponent/SwiftComponent.cs | 2 +- .../TypedComponent/TypedComponent.cs | 4 ++++ .../TypedComponent/VcpkgComponent.cs | 7 ++++++- 20 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs index 7ac040943..a9a0c1843 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs @@ -36,7 +36,7 @@ public CargoComponent(string name, string version, string author = null, string public override ComponentType Type => ComponentType.Cargo; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("cargo", string.Empty, this.Name, this.Version, null, string.Empty); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs index cbe5abdda..be6f91fd5 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs @@ -29,7 +29,7 @@ public ConanComponent(string name, string version, string previous, string packa public override ComponentType Type => ComponentType.Conan; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("conan", string.Empty, this.Name, this.Version, null, string.Empty); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs index 04edb3f76..32a210b83 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs @@ -37,5 +37,5 @@ private CondaComponent() public override ComponentType Type => ComponentType.Conda; - public override string Id => $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs index f24b90b0c..4f9322107 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs @@ -22,5 +22,5 @@ public DockerImageComponent(string hash, string name = null, string tag = null) public override ComponentType Type => ComponentType.DockerImage; - public override string Id => $"{this.Name} {this.Tag} {this.Digest}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Tag} {this.Digest}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs index 62ce707d1..4fa79799f 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs @@ -36,5 +36,5 @@ public DockerReference FullReference } } - public override string Id => $"{this.Repository} {this.Tag} {this.Digest}"; + public override string Id => this.CacheId ??= $"{this.Repository} {this.Tag} {this.Digest}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs index edafc3d61..bf09b95bf 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs @@ -43,5 +43,5 @@ public DotNetComponent(string sdkVersion, string targetFramework = null, string /// /// Provides an id like `{SdkVersion} - {TargetFramework} - {ProjectType} - dotnet` where unspecified values are represented as 'unknown'. /// - public override string Id => $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs index aeee1683e..5f68cfeb5 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs @@ -26,5 +26,5 @@ private GitComponent() public override ComponentType Type => ComponentType.Git; - public override string Id => $"{this.RepositoryUrl} : {this.CommitHash} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.RepositoryUrl} {this.CommitHash} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs index 6246ef702..5246273f4 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs @@ -36,7 +36,7 @@ private GoComponent() public override ComponentType Type => ComponentType.Go; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; public override bool Equals(object obj) { diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs index 23ccd2db9..4838773ca 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs @@ -36,7 +36,7 @@ public LinuxComponent(string distribution, string release, string name, string v public override ComponentType Type => ComponentType.Linux; - public override string Id => $"{this.Distribution} {this.Release} {this.Name} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Distribution} {this.Release} {this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl { diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs index b02730658..f2b516587 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs @@ -24,7 +24,7 @@ private MavenComponent() public override ComponentType Type => ComponentType.Maven; - public override string Id => $"{this.GroupId} {this.ArtifactId} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.GroupId} {this.ArtifactId} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("maven", this.GroupId, this.ArtifactId, this.Version, null, null); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs index e758eedae..d3d3259ca 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs @@ -28,7 +28,7 @@ public NpmComponent(string name, string version, string hash = null, NpmAuthor a public override ComponentType Type => ComponentType.Npm; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("npm", null, this.Name, this.Version, null, null); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs index dad787527..e8ab88ac2 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs @@ -24,7 +24,7 @@ public NuGetComponent(string name, string version, string[] authors = null) public override ComponentType Type => ComponentType.NuGet; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("nuget", null, this.Name, this.Version, null, null); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs index 7b8013090..cabd9e5bb 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs @@ -27,5 +27,5 @@ public OtherComponent(string name, string version, Uri downloadUrl, string hash) public override ComponentType Type => ComponentType.Other; - public override string Id => $"{this.Name} {this.Version} {this.DownloadUrl} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} {this.DownloadUrl} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs index 2eebd0dea..47910ee55 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs @@ -34,7 +34,7 @@ public PipComponent(string name, string version, string author = null, string li public override ComponentType Type => ComponentType.Pip; [SuppressMessage("Usage", "CA1308:Normalize String to Uppercase", Justification = "Casing cannot be overwritten.")] - public override string Id => $"{this.Name} {this.Version} - {this.Type}".ToLowerInvariant(); + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}".ToLowerInvariant(); public override PackageURL PackageUrl => new PackageURL("pypi", null, this.Name, this.Version, null, null); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs index 63b00d3cb..df2b2ecdd 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs @@ -25,7 +25,7 @@ public PodComponent(string name, string version, string specRepo = "") public override ComponentType Type => ComponentType.Pod; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl { diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs index e1d9845b3..737416b10 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs @@ -24,7 +24,7 @@ public RubyGemsComponent(string name, string version, string source = "") public override ComponentType Type => ComponentType.RubyGems; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; public override PackageURL PackageUrl => new PackageURL("gem", null, this.Name, this.Version, null, null); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs index c4328bfb8..bcf3176e0 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs @@ -21,6 +21,8 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str public override ComponentType Type => ComponentType.Spdx; + public override string Id => this.CacheId ??= $"{this.Name} {this.SpdxVersion} - {this.Checksum}"; + public string RootElementId { get; set; } public string Name { get; set; } @@ -32,6 +34,4 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str public string Checksum { get; set; } public string Path { get; set; } - - public override string Id => $"{this.Name}-{this.SpdxVersion}-{this.Checksum}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs index d871525bb..f8c0fd979 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs @@ -35,7 +35,7 @@ public SwiftComponent(string name, string version, string packageUrl, string has public override ComponentType Type => ComponentType.Swift; - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; + public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; // Example PackageURL -> pkg:swift/github.com/apple/swift-asn1 // type: swift diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs index c5930414a..3fb110fd3 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs @@ -30,6 +30,10 @@ internal TypedComponent() [JsonIgnore] internal string DebuggerDisplay => $"{this.Id}"; + [field: JsonIgnore] + [JsonIgnore] + protected string CacheId { get; set; } + protected string ValidateRequiredInput(string input, string fieldName, string componentType) { return string.IsNullOrWhiteSpace(input) diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs index 64105d404..0d7cb7359 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -42,6 +42,11 @@ public override string Id { get { + if (this.CacheId != null) + { + return this.CacheId; + } + var componentLocationPrefix = string.Empty; if (!string.IsNullOrWhiteSpace(this.DownloadLocation) && !this.DownloadLocation.Trim().Equals("NONE", System.StringComparison.InvariantCultureIgnoreCase)) { @@ -54,7 +59,7 @@ public override string Id componentPortVersionSuffix = $"#{this.PortVersion} "; } - return $"{componentLocationPrefix}{this.Name} {this.Version}{componentPortVersionSuffix}- {this.Type}"; + return this.CacheId = $"{componentLocationPrefix}{this.Name} {this.Version}{componentPortVersionSuffix}- {this.Type}"; } } From 84578c337733f537957af74f4427b00890212841 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Wed, 16 Apr 2025 19:45:36 -0700 Subject: [PATCH 07/17] Cache getting additional related files --- .../DefaultGraphTranslationService.cs | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index b78f53f45..c0e159bf5 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -86,6 +86,7 @@ private void LogComponentScopeTelemetry(List components) private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEnumerable<(IComponentDetector Detector, ComponentRecorder Recorder)> recorderDetectorPairs, DirectoryInfo rootDirectory, bool updateLocations) { + var dependencyGraphsAdditionalRelatedFiles = new Dictionary<(string, IDependencyGraph), HashSet>(); return recorderDetectorPairs .Where(recorderDetectorPair => recorderDetectorPair.Recorder != null) .SelectMany(recorderDetectorPair => @@ -126,22 +127,28 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn component.DependencyScope = DependencyScopeComparer.GetMergedDependencyScope(component.DependencyScope, dependencyGraph.GetDependencyScope(component.Component.Id)); component.DetectedBy = detector; - // Return in a format that allows us to add the additional files for the components - var locations = dependencyGraph.GetAdditionalRelatedFiles(); - // Experiments uses this service to build the dependency graph for analysis. In this case, we do not want to update the locations of the component. // Updating the locations of the component will propogate to the final depenendcy graph and cause the graph to be incorrect. if (updateLocations) { - // graph authoritatively stores the location of the component - locations.Add(location); - - foreach (var customLocation in componentCustomLocations) + if (!dependencyGraphsAdditionalRelatedFiles.TryGetValue((location, dependencyGraph), out var locations)) { - locations.Add(customLocation); + locations = dependencyGraph.GetAdditionalRelatedFiles(); + locations.Add(location); + dependencyGraphsAdditionalRelatedFiles[(location, dependencyGraph)] = locations; } - var relativePaths = this.MakeFilePathsRelative(this.logger, rootDirectory, locations); + // Return in a format that allows us to add the additional files for the components + // var locations = dependencyGraph.GetAdditionalRelatedFiles(); + + // graph authoritatively stores the location of the component + // locations.Add(location); + + // foreach (var customLocation in componentCustomLocations) + // { + // locations.Add(customLocation); + // } + var relativePaths = this.MakeFilePathsRelative(this.logger, rootDirectory, locations.Union(componentCustomLocations)); foreach (var additionalRelatedFile in relativePaths ?? Enumerable.Empty()) { @@ -254,7 +261,7 @@ private void AddAncestorsToDetectedComponent(DetectedComponent detectedComponent } } - private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo rootDirectory, HashSet filePaths) + private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo rootDirectory, IEnumerable filePaths) { if (rootDirectory == null) { From f0a46af4a4f4b51379a57b492bd1e20525b47868 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Thu, 17 Apr 2025 11:58:22 -0700 Subject: [PATCH 08/17] Experiment with parallel.foreach --- .../DefaultGraphTranslationService.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index c0e159bf5..3f398c561 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -86,7 +86,6 @@ private void LogComponentScopeTelemetry(List components) private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEnumerable<(IComponentDetector Detector, ComponentRecorder Recorder)> recorderDetectorPairs, DirectoryInfo rootDirectory, bool updateLocations) { - var dependencyGraphsAdditionalRelatedFiles = new Dictionary<(string, IDependencyGraph), HashSet>(); return recorderDetectorPairs .Where(recorderDetectorPair => recorderDetectorPair.Recorder != null) .SelectMany(recorderDetectorPair => @@ -98,7 +97,14 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn // Note that it looks like we are building up detected components functionally, but they are not immutable -- the code is just written // to look like a pipeline. - foreach (var component in detectedComponents) + Parallel.ForEach(detectedComponents, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount * 2 }, component => + + // { + // // We need to make sure that we are not adding the same component multiple times. This is a problem when we have multiple detectors that can detect the same component. + // // We need to make sure that we are not adding the same component multiple times. This is a problem when we have multiple detectors that can detect the same component. + // x.DetectedBy = detector; + // }); + // foreach (var component in detectedComponents) { // clone custom locations and make them relative to root. var componentCustomLocations = component.FilePaths ?? []; @@ -131,23 +137,17 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn // Updating the locations of the component will propogate to the final depenendcy graph and cause the graph to be incorrect. if (updateLocations) { - if (!dependencyGraphsAdditionalRelatedFiles.TryGetValue((location, dependencyGraph), out var locations)) - { - locations = dependencyGraph.GetAdditionalRelatedFiles(); - locations.Add(location); - dependencyGraphsAdditionalRelatedFiles[(location, dependencyGraph)] = locations; - } - // Return in a format that allows us to add the additional files for the components - // var locations = dependencyGraph.GetAdditionalRelatedFiles(); + var locations = dependencyGraph.GetAdditionalRelatedFiles(); // graph authoritatively stores the location of the component - // locations.Add(location); + locations.Add(location); + + foreach (var customLocation in componentCustomLocations) + { + locations.Add(customLocation); + } - // foreach (var customLocation in componentCustomLocations) - // { - // locations.Add(customLocation); - // } var relativePaths = this.MakeFilePathsRelative(this.logger, rootDirectory, locations.Union(componentCustomLocations)); foreach (var additionalRelatedFile in relativePaths ?? Enumerable.Empty()) @@ -156,7 +156,7 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn } } } - } + }); return detectedComponents; }).ToList(); From 2d3fc750d5612a8693efe7125aa3d1f65425d368 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Thu, 17 Apr 2025 14:49:11 -0700 Subject: [PATCH 09/17] Restore non-parallel --- .../DefaultGraphTranslationService.cs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index 3f398c561..a3d615b1f 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -97,14 +97,7 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn // Note that it looks like we are building up detected components functionally, but they are not immutable -- the code is just written // to look like a pipeline. - Parallel.ForEach(detectedComponents, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount * 2 }, component => - - // { - // // We need to make sure that we are not adding the same component multiple times. This is a problem when we have multiple detectors that can detect the same component. - // // We need to make sure that we are not adding the same component multiple times. This is a problem when we have multiple detectors that can detect the same component. - // x.DetectedBy = detector; - // }); - // foreach (var component in detectedComponents) + foreach (var component in detectedComponents) { // clone custom locations and make them relative to root. var componentCustomLocations = component.FilePaths ?? []; @@ -156,7 +149,7 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn } } } - }); + }; return detectedComponents; }).ToList(); From 2cfc7929825ff665f794c9a9b2b9af2f7d494efb Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Thu, 17 Apr 2025 14:51:21 -0700 Subject: [PATCH 10/17] Add more perf logging --- .../Services/ScanExecutionService.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/ScanExecutionService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/ScanExecutionService.cs index 94ef6f64b..c0407ee79 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/ScanExecutionService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/ScanExecutionService.cs @@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services; using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; @@ -44,8 +45,14 @@ public async Task ExecuteScanAsync(ScanSettings settings) this.logger.LogDebug("Finished applying restrictions to detectors."); + var stopwatch = Stopwatch.StartNew(); + this.logger.LogInformation("Starting detector processing."); var processingResult = await this.detectorProcessingService.ProcessDetectorsAsync(settings, detectorsWithAppliedRestrictions, detectorRestrictions); + this.logger.LogInformation("Finished detector processing in {ElapsedMilliseconds}ms", stopwatch.ElapsedMilliseconds); + stopwatch.Restart(); + this.logger.LogInformation("Starting generate scan result."); var scanResult = this.graphTranslationService.GenerateScanResultFromProcessingResult(processingResult, settings); + this.logger.LogInformation("Finished generating scan result in {ElapsedMilliseconds}ms", stopwatch.ElapsedMilliseconds); scanResult.DetectorsInScan = detectorsWithAppliedRestrictions.Select(ConvertToContract).ToList(); scanResult.DetectorsNotInScan = detectorRestrictions.DisabledDetectors.Select(ConvertToContract).ToList(); From 9d9cd00689634aa4502772274f48834d9dbb9586 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Thu, 17 Apr 2025 14:58:04 -0700 Subject: [PATCH 11/17] Fix minor error --- .../Services/GraphTranslation/DefaultGraphTranslationService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index a3d615b1f..2df07ed59 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -149,7 +149,7 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn } } } - }; + } return detectedComponents; }).ToList(); From 0118eb966b9d7c27da387317348b72a434a137ba Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Thu, 17 Apr 2025 18:48:08 -0700 Subject: [PATCH 12/17] Final for test --- .../DependencyGraph/ComponentRecorder.cs | 15 +++++++-------- .../Services/DetectorProcessingService.cs | 4 ++-- .../Services/ScanExecutionService.cs | 7 ------- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index 95e6c0f0b..3323585a3 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -180,16 +180,15 @@ public void RegisterUsage( #endif var componentId = detectedComponent.Component.Id; - DetectedComponent storedComponent = null; - lock (this.registerUsageLock) - { - storedComponent = this.detectedComponentsInternal.GetOrAdd(componentId, detectedComponent); + var storedComponent = this.detectedComponentsInternal.GetOrAdd(componentId, detectedComponent); - if (!string.IsNullOrWhiteSpace(targetFramework)) - { - storedComponent.TargetFrameworks.Add(targetFramework.Trim()); - } + if (!string.IsNullOrWhiteSpace(targetFramework)) + { + storedComponent.TargetFrameworks.Add(targetFramework.Trim()); + } + lock (this.registerUsageLock) + { this.AddComponentToGraph(this.ManifestFileLocation, detectedComponent, isExplicitReferencedDependency, parentComponentId, isDevelopmentDependency, dependencyScope); } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 92b3154af..248c401d1 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -117,8 +117,8 @@ public async Task ProcessDetectorsAsync( record.DetectorId = detector.Id; record.DetectedComponentCount = detectedComponents.Count(); var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation().Values; - record.ExplicitlyReferencedComponentCount = dependencyGraphs.Select(dependencyGraph => dependencyGraph.GetAllExplicitlyReferencedComponents()) - .SelectMany(x => x) + record.ExplicitlyReferencedComponentCount = dependencyGraphs + .SelectMany(dependencyGraph => dependencyGraph.GetAllExplicitlyReferencedComponents()) .Distinct() .Count(); diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/ScanExecutionService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/ScanExecutionService.cs index c0407ee79..94ef6f64b 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/ScanExecutionService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/ScanExecutionService.cs @@ -3,7 +3,6 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services; using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; @@ -45,14 +44,8 @@ public async Task ExecuteScanAsync(ScanSettings settings) this.logger.LogDebug("Finished applying restrictions to detectors."); - var stopwatch = Stopwatch.StartNew(); - this.logger.LogInformation("Starting detector processing."); var processingResult = await this.detectorProcessingService.ProcessDetectorsAsync(settings, detectorsWithAppliedRestrictions, detectorRestrictions); - this.logger.LogInformation("Finished detector processing in {ElapsedMilliseconds}ms", stopwatch.ElapsedMilliseconds); - stopwatch.Restart(); - this.logger.LogInformation("Starting generate scan result."); var scanResult = this.graphTranslationService.GenerateScanResultFromProcessingResult(processingResult, settings); - this.logger.LogInformation("Finished generating scan result in {ElapsedMilliseconds}ms", stopwatch.ElapsedMilliseconds); scanResult.DetectorsInScan = detectorsWithAppliedRestrictions.Select(ConvertToContract).ToList(); scanResult.DetectorsNotInScan = detectorRestrictions.DisabledDetectors.Select(ConvertToContract).ToList(); From 1bce48f3aab666c6064455973b44bc3134057515 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Fri, 18 Apr 2025 15:31:04 -0700 Subject: [PATCH 13/17] Enable parallelism for Yarn.lock component detector --- .../FileComponentDetector.cs | 6 ++++-- .../yarn/YarnLockComponentDetector.cs | 12 ++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs index 9714d095c..cb24dd944 100644 --- a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs @@ -1,8 +1,10 @@ namespace Microsoft.ComponentDetection.Contracts; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Reactive.Linq; using System.Threading; using System.Threading.Tasks; @@ -56,7 +58,7 @@ public abstract class FileComponentDetector : IComponentDetector public virtual bool NeedsAutomaticRootDependencyCalculation { get; protected set; } - protected Dictionary Telemetry { get; set; } = []; + protected ConcurrentDictionary Telemetry { get; set; } = []; /// /// List of any any additional properties as key-value pairs that we would like to capture for the detector. @@ -135,7 +137,7 @@ private async Task ProcessAsync( return new IndividualDetectorScanResult { ResultCode = ProcessingResultCode.Success, - AdditionalTelemetryDetails = this.Telemetry, + AdditionalTelemetryDetails = this.Telemetry.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), }; } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index c2110e2a1..33054f942 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -44,6 +44,9 @@ public YarnLockComponentDetector( /// "Package" is a more common substring, enclose it with \ to verify it is a folder. protected override IList SkippedFolders => ["node_modules", "pnpm-store", "\\package\\"]; + /// + protected override bool EnableParallelism { get; set; } = true; + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) { var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; @@ -243,9 +246,9 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRec yarnRoots.Add(entry); var locationMapDictonaryKey = this.GetLocationMapKey(name, version.Key); - if (workspaceDependencyVsLocationMap.ContainsKey(locationMapDictonaryKey)) + if (workspaceDependencyVsLocationMap.TryGetValue(locationMapDictonaryKey, out location)) { - entry.Location = workspaceDependencyVsLocationMap[locationMapDictonaryKey]; + entry.Location = location; } } } @@ -324,10 +327,7 @@ private void ProcessWorkspaceDependency(IDictionary workspaceDependencyVsLocationMap, string streamLocation, string dependencyName, string dependencyVersion) { var locationMapDictionaryKey = this.GetLocationMapKey(dependencyName, dependencyVersion); - if (!workspaceDependencyVsLocationMap.ContainsKey(locationMapDictionaryKey)) - { - workspaceDependencyVsLocationMap[locationMapDictionaryKey] = streamLocation; - } + workspaceDependencyVsLocationMap.TryAdd(locationMapDictionaryKey, streamLocation); } private string GetLocationMapKey(string dependencyName, string dependencyVersion) From 6b75e15e8b555c4d771e553ce1882bd50f0146c0 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Fri, 18 Apr 2025 17:54:20 -0700 Subject: [PATCH 14/17] Restore MakeFilePathsRelative --- .../GraphTranslation/DefaultGraphTranslationService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index 2df07ed59..e9b335ff1 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -141,7 +141,7 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn locations.Add(customLocation); } - var relativePaths = this.MakeFilePathsRelative(this.logger, rootDirectory, locations.Union(componentCustomLocations)); + var relativePaths = this.MakeFilePathsRelative(this.logger, rootDirectory, locations); foreach (var additionalRelatedFile in relativePaths ?? Enumerable.Empty()) { @@ -254,7 +254,7 @@ private void AddAncestorsToDetectedComponent(DetectedComponent detectedComponent } } - private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo rootDirectory, IEnumerable filePaths) + private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo rootDirectory, HashSet filePaths) { if (rootDirectory == null) { From 9bb70620601be120eab56717a043dde6576e43d5 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Tue, 29 Apr 2025 14:47:19 -0700 Subject: [PATCH 15/17] Addressed PR comments --- .../DependencyGraph/ComponentRecorder.cs | 8 +--- .../TypedComponent/CargoComponent.cs | 4 +- .../TypedComponent/ConanComponent.cs | 4 +- .../TypedComponent/CondaComponent.cs | 2 +- .../TypedComponent/DockerImageComponent.cs | 2 +- .../DockerReferenceComponent.cs | 2 +- .../TypedComponent/DotNetComponent.cs | 3 +- .../TypedComponent/GitComponent.cs | 2 +- .../TypedComponent/GoComponent.cs | 2 +- .../TypedComponent/LinuxComponent.cs | 4 +- .../TypedComponent/MavenComponent.cs | 4 +- .../TypedComponent/NpmComponent.cs | 4 +- .../TypedComponent/NugetComponent.cs | 4 +- .../TypedComponent/OtherComponent.cs | 2 +- .../TypedComponent/PipComponent.cs | 6 +-- .../TypedComponent/PodComponent.cs | 4 +- .../TypedComponent/RubyGemsComponent.cs | 4 +- .../TypedComponent/SpdxComponent.cs | 4 +- .../TypedComponent/SwiftComponent.cs | 4 +- .../TypedComponent/TypedComponent.cs | 12 +++--- .../TypedComponent/VcpkgComponent.cs | 42 ++++++++----------- .../yarn/YarnLockComponentDetector.cs | 2 +- 22 files changed, 57 insertions(+), 68 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index 3323585a3..aa72fb234 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -94,13 +94,7 @@ public IReadOnlyDictionary GetDependencyGraphsByLocati internal DependencyGraph GetDependencyGraphForLocation(string location) { - if (this.singleFileRecorders.TryGetValue(location, out var singleFileRecorder)) - { - return singleFileRecorder.DependencyGraph; - } - - // TODO: Should we throw an exception here instead of returning null? - return null; + return this.singleFileRecorders[location].DependencyGraph; } public sealed class SingleFileComponentRecorder : ISingleFileComponentRecorder diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs index a9a0c1843..a59c95e88 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs @@ -36,7 +36,7 @@ public CargoComponent(string name, string version, string author = null, string public override ComponentType Type => ComponentType.Cargo; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("cargo", string.Empty, this.Name, this.Version, null, string.Empty); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs index be6f91fd5..9c26e66b4 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/ConanComponent.cs @@ -29,7 +29,7 @@ public ConanComponent(string name, string version, string previous, string packa public override ComponentType Type => ComponentType.Conan; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("conan", string.Empty, this.Name, this.Version, null, string.Empty); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs index 32a210b83..6d46f111d 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/CondaComponent.cs @@ -37,5 +37,5 @@ private CondaComponent() public override ComponentType Type => ComponentType.Conda; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}"; + protected override string ComputeId() => $"{this.Name} {this.Version} {this.Build} {this.Channel} {this.Subdir} {this.Namespace} {this.Url} {this.MD5} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs index 4f9322107..69c6ea7be 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerImageComponent.cs @@ -22,5 +22,5 @@ public DockerImageComponent(string hash, string name = null, string tag = null) public override ComponentType Type => ComponentType.DockerImage; - public override string Id => this.CacheId ??= $"{this.Name} {this.Tag} {this.Digest}"; + protected override string ComputeId() => $"{this.Name} {this.Tag} {this.Digest}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs index 4fa79799f..224087152 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DockerReferenceComponent.cs @@ -36,5 +36,5 @@ public DockerReference FullReference } } - public override string Id => this.CacheId ??= $"{this.Repository} {this.Tag} {this.Digest}"; + protected override string ComputeId() => $"{this.Repository} {this.Tag} {this.Digest}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs index bf09b95bf..c3f9dc611 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/DotNetComponent.cs @@ -43,5 +43,6 @@ public DotNetComponent(string sdkVersion, string targetFramework = null, string /// /// Provides an id like `{SdkVersion} - {TargetFramework} - {ProjectType} - dotnet` where unspecified values are represented as 'unknown'. /// - public override string Id => this.CacheId ??= $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}"; + /// Id of the component. + protected override string ComputeId() => $"{this.SdkVersion} {this.TargetFramework} {this.ProjectType} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs index 5f68cfeb5..9b7a9fe3c 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs @@ -26,5 +26,5 @@ private GitComponent() public override ComponentType Type => ComponentType.Git; - public override string Id => this.CacheId ??= $"{this.RepositoryUrl} {this.CommitHash} - {this.Type}"; + protected override string ComputeId() => $"{this.RepositoryUrl} {this.CommitHash} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs index 5246273f4..e09fc32e9 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GoComponent.cs @@ -36,7 +36,7 @@ private GoComponent() public override ComponentType Type => ComponentType.Go; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; public override bool Equals(object obj) { diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs index 4838773ca..239c89854 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/LinuxComponent.cs @@ -36,8 +36,6 @@ public LinuxComponent(string distribution, string release, string name, string v public override ComponentType Type => ComponentType.Linux; - public override string Id => this.CacheId ??= $"{this.Distribution} {this.Release} {this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl { get @@ -62,6 +60,8 @@ public override PackageURL PackageUrl } } + protected override string ComputeId() => $"{this.Distribution} {this.Release} {this.Name} {this.Version} - {this.Type}"; + private bool IsUbuntu() { return this.Distribution.Equals("UBUNTU", StringComparison.OrdinalIgnoreCase); diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs index f2b516587..75ed72c5b 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/MavenComponent.cs @@ -24,7 +24,7 @@ private MavenComponent() public override ComponentType Type => ComponentType.Maven; - public override string Id => this.CacheId ??= $"{this.GroupId} {this.ArtifactId} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("maven", this.GroupId, this.ArtifactId, this.Version, null, null); + + protected override string ComputeId() => $"{this.GroupId} {this.ArtifactId} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs index d3d3259ca..67bec60f5 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NpmComponent.cs @@ -28,7 +28,7 @@ public NpmComponent(string name, string version, string hash = null, NpmAuthor a public override ComponentType Type => ComponentType.Npm; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("npm", null, this.Name, this.Version, null, null); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs index e8ab88ac2..4c1e578e8 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/NugetComponent.cs @@ -24,7 +24,7 @@ public NuGetComponent(string name, string version, string[] authors = null) public override ComponentType Type => ComponentType.NuGet; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("nuget", null, this.Name, this.Version, null, null); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs index cabd9e5bb..e76b3cb72 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/OtherComponent.cs @@ -27,5 +27,5 @@ public OtherComponent(string name, string version, Uri downloadUrl, string hash) public override ComponentType Type => ComponentType.Other; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} {this.DownloadUrl} - {this.Type}"; + protected override string ComputeId() => $"{this.Name} {this.Version} {this.DownloadUrl} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs index 47910ee55..7efe80901 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PipComponent.cs @@ -33,8 +33,8 @@ public PipComponent(string name, string version, string author = null, string li public override ComponentType Type => ComponentType.Pip; - [SuppressMessage("Usage", "CA1308:Normalize String to Uppercase", Justification = "Casing cannot be overwritten.")] - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}".ToLowerInvariant(); - public override PackageURL PackageUrl => new PackageURL("pypi", null, this.Name, this.Version, null, null); + + [SuppressMessage("Usage", "CA1308:Normalize String to Uppercase", Justification = "Casing cannot be overwritten.")] + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}".ToLowerInvariant(); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs index df2b2ecdd..bb690a782 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/PodComponent.cs @@ -25,8 +25,6 @@ public PodComponent(string name, string version, string specRepo = "") public override ComponentType Type => ComponentType.Pod; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl { get @@ -40,4 +38,6 @@ public override PackageURL PackageUrl return new PackageURL("cocoapods", null, this.Name, this.Version, qualifiers, null); } } + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs index 737416b10..a02657203 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/RubyGemsComponent.cs @@ -24,7 +24,7 @@ public RubyGemsComponent(string name, string version, string source = "") public override ComponentType Type => ComponentType.RubyGems; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; - public override PackageURL PackageUrl => new PackageURL("gem", null, this.Name, this.Version, null, null); + + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs index bcf3176e0..18b04bc5b 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs @@ -21,8 +21,6 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str public override ComponentType Type => ComponentType.Spdx; - public override string Id => this.CacheId ??= $"{this.Name} {this.SpdxVersion} - {this.Checksum}"; - public string RootElementId { get; set; } public string Name { get; set; } @@ -34,4 +32,6 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str public string Checksum { get; set; } public string Path { get; set; } + + protected override string ComputeId() => $"{this.Name} {this.SpdxVersion} - {this.Checksum}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs index f8c0fd979..2203574fe 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SwiftComponent.cs @@ -35,8 +35,6 @@ public SwiftComponent(string name, string version, string packageUrl, string has public override ComponentType Type => ComponentType.Swift; - public override string Id => this.CacheId ??= $"{this.Name} {this.Version} - {this.Type}"; - // Example PackageURL -> pkg:swift/github.com/apple/swift-asn1 // type: swift // namespace: github.com/apple @@ -52,6 +50,8 @@ public SwiftComponent(string name, string version, string packageUrl, string has }, subpath: null); + protected override string ComputeId() => $"{this.Name} {this.Version} - {this.Type}"; + private string GetNamespaceFromPackageUrl() { // In the case of github.com, the namespace should contain the user/organization diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs index 3fb110fd3..23d9d4313 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/TypedComponent.cs @@ -14,6 +14,9 @@ namespace Microsoft.ComponentDetection.Contracts.TypedComponent; [DebuggerDisplay("{DebuggerDisplay,nq}")] public abstract class TypedComponent { + [JsonIgnore] + private string id; + internal TypedComponent() { // Reserved for deserialization. @@ -23,17 +26,14 @@ internal TypedComponent() [JsonConverter(typeof(StringEnumConverter))] public abstract ComponentType Type { get; } - public abstract string Id { get; } + /// Gets the id of the component. + public string Id => this.id ??= this.ComputeId(); public virtual PackageURL PackageUrl { get; } [JsonIgnore] internal string DebuggerDisplay => $"{this.Id}"; - [field: JsonIgnore] - [JsonIgnore] - protected string CacheId { get; set; } - protected string ValidateRequiredInput(string input, string fieldName, string componentType) { return string.IsNullOrWhiteSpace(input) @@ -51,4 +51,6 @@ protected string NullPropertyExceptionMessage(string propertyName, string compon { return $"Property {propertyName} of component type {componentType} is required"; } + + protected abstract string ComputeId(); } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs index 0d7cb7359..e62457650 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/VcpkgComponent.cs @@ -38,31 +38,6 @@ public VcpkgComponent(string spdxid, string name, string version, string triplet public override ComponentType Type => ComponentType.Vcpkg; - public override string Id - { - get - { - if (this.CacheId != null) - { - return this.CacheId; - } - - var componentLocationPrefix = string.Empty; - if (!string.IsNullOrWhiteSpace(this.DownloadLocation) && !this.DownloadLocation.Trim().Equals("NONE", System.StringComparison.InvariantCultureIgnoreCase)) - { - componentLocationPrefix = $"{this.DownloadLocation} : "; - } - - var componentPortVersionSuffix = " "; - if (this.PortVersion > 0) - { - componentPortVersionSuffix = $"#{this.PortVersion} "; - } - - return this.CacheId = $"{componentLocationPrefix}{this.Name} {this.Version}{componentPortVersionSuffix}- {this.Type}"; - } - } - public override PackageURL PackageUrl { get @@ -81,4 +56,21 @@ public override PackageURL PackageUrl } } } + + protected override string ComputeId() + { + var componentLocationPrefix = string.Empty; + if (!string.IsNullOrWhiteSpace(this.DownloadLocation) && !this.DownloadLocation.Trim().Equals("NONE", System.StringComparison.InvariantCultureIgnoreCase)) + { + componentLocationPrefix = $"{this.DownloadLocation} : "; + } + + var componentPortVersionSuffix = " "; + if (this.PortVersion > 0) + { + componentPortVersionSuffix = $"#{this.PortVersion} "; + } + + return $"{componentLocationPrefix}{this.Name} {this.Version}{componentPortVersionSuffix}- {this.Type}"; + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index 33054f942..3a6d937bf 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -36,7 +36,7 @@ public YarnLockComponentDetector( public override IEnumerable SupportedComponentTypes { get; } = [ComponentType.Npm]; - public override int Version => 8; + public override int Version => 9; public override IEnumerable Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.Npm)]; From 0398e3c2045a34bb244c8f26e6031a5354f0c2e3 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Thu, 1 May 2025 13:26:31 -0700 Subject: [PATCH 16/17] Fix id for spdx and git --- .../TypedComponent/GitComponent.cs | 2 +- .../TypedComponent/SpdxComponent.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs index 9b7a9fe3c..2ed88e3e1 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/GitComponent.cs @@ -26,5 +26,5 @@ private GitComponent() public override ComponentType Type => ComponentType.Git; - protected override string ComputeId() => $"{this.RepositoryUrl} {this.CommitHash} - {this.Type}"; + protected override string ComputeId() => $"{this.RepositoryUrl} : {this.CommitHash} - {this.Type}"; } diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs index 18b04bc5b..5d00c5198 100644 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs +++ b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxComponent.cs @@ -33,5 +33,5 @@ public SpdxComponent(string spdxVersion, Uri documentNamespace, string name, str public string Path { get; set; } - protected override string ComputeId() => $"{this.Name} {this.SpdxVersion} - {this.Checksum}"; + protected override string ComputeId() => $"{this.Name}-{this.SpdxVersion}-{this.Checksum}"; } From f93e0c4ba4c884e2033e7a590666307ec9bdfa91 Mon Sep 17 00:00:00 2001 From: Iman Narasamdya Date: Thu, 1 May 2025 13:41:44 -0700 Subject: [PATCH 17/17] Fix integration test --- docs/schema/manifest.schema.json | 6 ++++-- .../resources/VerificationTest.ps1 | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/schema/manifest.schema.json b/docs/schema/manifest.schema.json index f0c21bfc9..6d7623b93 100644 --- a/docs/schema/manifest.schema.json +++ b/docs/schema/manifest.schema.json @@ -118,7 +118,8 @@ "Spdx", "Vcpkg", "DockerReference", - "DotNet" + "DotNet", + "Conan" ] } } @@ -345,7 +346,8 @@ "Spdx", "Vcpkg", "DockerReference", - "DotNet" + "DotNet", + "Conan" ] }, "id": { diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 b/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 index 3c490884c..9e9dcd0d9 100755 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 @@ -58,6 +58,11 @@ function main() $env:GITHUB_NEW_ARTIFACTS_DIR = $output $env:ALLOWED_TIME_DRIFT_RATIO = "0.75" + if ([string]::IsNullOrEmpty($env:GITHUB_WORKSPACE)) { + $env:GITHUB_WORKSPACE = $repoPath + Write-Host "Setting GITHUB_WORKSPACE environment variable to $repoPath" + } + Write-Progress "Executing verification tests....." Set-Location ((Get-Item $repoPath).FullName + "\test\Microsoft.ComponentDetection.VerificationTests\") dotnet restore