diff --git a/purldb-toolkit/README.rst b/purldb-toolkit/README.rst index 7c85e360..f394e585 100644 --- a/purldb-toolkit/README.rst +++ b/purldb-toolkit/README.rst @@ -29,12 +29,12 @@ The purlcli command exposes multiple subcommands. Run this to command to get bas purlcli --help Usage: purlcli [OPTIONS] COMMAND [ARGS]... - + Return information for a PURL or list of PURLs. - + Options: --help Show this message and exit. - + Commands: d2d Run deploy-to-devel "back2source" analysis between packages. metadata Fetch package metadata for a PURL. @@ -561,14 +561,14 @@ from a https://github.com/nexB/purldb/ clone:: cd purldb make dev make envfile - SECRET_KEY="1" make postgres_matchcodeio - SECRET_KEY="1" make run_matchcodeio + SECRET_KEY="1" make postgres_matchcodeio + SECRET_KEY="1" make run_matchcodeio Then in another shell:: cd purldb source venv/bin/activate - + Finally run the command: .. code-block:: console diff --git a/purldb-toolkit/src/purldb_toolkit/purlcli.py b/purldb-toolkit/src/purldb_toolkit/purlcli.py index 431c9164..b866d934 100644 --- a/purldb-toolkit/src/purldb_toolkit/purlcli.py +++ b/purldb-toolkit/src/purldb_toolkit/purlcli.py @@ -8,8 +8,6 @@ # import json -import logging -import os import re import sys import time @@ -24,15 +22,12 @@ import click import requests from fetchcode.package import info -from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS, versions -from packageurl import PackageURL +from fetchcode.package_versions import versions from packageurl.contrib import purl2url # in seconds POLLING_INTERVAL = 5 -LOG_FILE_LOCATION = os.path.join(os.path.expanduser("~"), "purlcli.log") - @click.group() def purlcli(): @@ -65,70 +60,41 @@ def purlcli(): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) -def get_metadata(purls, output, file, unique): +def get_metadata(purls, output, file): """ Fetch package metadata for a PURL. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - metadata_info = get_metadata_details(purls, output, file, unique, command_name) + metadata_info = get_metadata_details(purls, output, file, command_name) json.dump(metadata_info, output, indent=4) -def get_metadata_details(purls, output, file, unique, command_name): +def get_metadata_details(purls, output, file, command_name): """ Return a dictionary containing metadata for each PURL in the `purls` input - list. `check_metadata_purl()` will print an error message to the console - (also displayed in the JSON output) when necessary. + list. """ metadata_details = {} metadata_details["headers"] = [] metadata_details["packages"] = [] + deduplicated_purls = deduplicate_purls(purls) - metadata_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - - clear_log_file() - - for purl in input_purls: + for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - - purl_data = {} - purl_data["purl"] = purl - - metadata_purl = check_metadata_purl(purl) - - if command_name == "metadata" and metadata_purl: - metadata_warnings[purl] = metadata_purl - continue - metadata_collection = collect_metadata(purl) - purl_data["metadata"] = metadata_collection - metadata_details["packages"].append(purl_data) + metadata_details["packages"].extend(metadata_collection) metadata_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, output=output, file=file, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, - purl_warnings=metadata_warnings, ) return metadata_details @@ -136,91 +102,42 @@ def get_metadata_details(purls, output, file, unique, command_name): def collect_metadata(purl): """ - Return a list of release-based metadata collections from fetchcode/package.py. + Return a list of release-based metadata collections (a fetchcode Package) + from fetchcode/package.py. """ collected_metadata = [] - for release in list(info(purl)): - release_detail = release.to_dict() - release_detail.move_to_end("purl", last=False) - collected_metadata.append(release_detail) + purl_metadata = info(purl) + if purl_metadata: + for release in list(purl_metadata): + release_detail = release.to_dict() + release_detail.move_to_end("purl", last=False) + collected_metadata.append(release_detail) return collected_metadata -def check_metadata_purl(purl): +def deduplicate_purls(purls): """ - Return a variable identifying the message for printing to the console by - get_metadata_details() if (1) the input PURL is invalid, (2) its type is not - supported by `metadata` or (3) its existence was not validated (e.g., - "does not exist in the upstream repo"). - - This message will also be reported by construct_headers() in the - `warnings` field of the `header` section of the JSON object returned by - the `metadata` command. + Deduplicate all input PURLs. PURLs with different versions or no version + are treated as unique for purposes of deduplication. """ - check_validation = validate_purl(purl) - if check_validation is None: - return "validation_error" - results = check_validation - - if results["valid"] == False: - return "not_valid" - - # This is manually constructed from a visual inspection of fetchcode/package.py. - metadata_supported_ecosystems = [ - "bitbucket", - "cargo", - "github", - "npm", - "pypi", - "rubygems", - ] - metadata_purl = PackageURL.from_string(purl) - - if metadata_purl.type not in metadata_supported_ecosystems: - return "valid_but_not_supported" - - if results["exists"] == False: - return "not_in_upstream_repo" - - if results["exists"] == None: - return "check_existence_not_supported" - - -def normalize_purls(purls, unique): - """ - If the command includes the `--unique` flag, take the list of input PURLs, - remove the portion of the PURL that starts with a PURL separator (`@`, `?` - or `#`), and return a deduplicated list of the resulting PURLs (in - `input_purls`) and a list of tuples of each pair of the original input PURL - and the normalized PURL (in `normalized_purls`). - """ - input_purls = [] - normalized_purls = [] - if unique: - for purl in purls: - input_purl = purl - purl = purl.strip() - purl = re.split("[@,?,#,]+", purl)[0] - normalized_purl = purl - normalized_purls.append((input_purl, normalized_purl)) - if normalized_purl not in input_purls: - input_purls.append(normalized_purl) - else: - input_purls = purls + reviewed = set() + deduplicated_purls = [] + for purl in purls: + purl = purl.strip() + if purl not in reviewed: + reviewed.add(purl) + deduplicated_purls.append(purl) - return input_purls, normalized_purls + return deduplicated_purls def construct_headers( - purls=None, + deduplicated_purls=None, output=None, file=None, command_name=None, head=None, - normalized_purls=None, - unique=None, - purl_warnings=None, ): """ Return a list comprising the `headers` content of the dictionary output. @@ -231,7 +148,7 @@ def construct_headers( errors = [] warnings = [] - context_purls = [p for p in purls] + context_purls = [p for p in deduplicated_purls] context_file = file context_file_name = None if context_file: @@ -247,52 +164,12 @@ def construct_headers( if head: options["--head"] = True - if unique: - options["--unique"] = True - if isinstance(output, str): options["--output"] = output else: options["--output"] = output.name headers_content["options"] = options - headers_content["purls"] = purls - - if (command_name in ["metadata", "urls", "validate", "versions"]) and unique: - for input_purl, normalized_purl in normalized_purls: - if input_purl != normalized_purl: - warnings.append( - f"input PURL: '{input_purl}' normalized to '{normalized_purl}'" - ) - - for purl in purls: - if not purl: - continue - - warning_text = { - "error_fetching_purl": f"'error fetching {purl}'", - "validation_error": f"'{purl}' encountered a validation error", - "not_valid": f"'{purl}' not valid", - "valid_but_not_supported": f"'{purl}' not supported with `{command_name}` command", - "valid_but_not_fully_supported": f"'{purl}' not fully supported with `urls` command", - "not_in_upstream_repo": f"'{purl}' does not exist in the upstream repo", - "check_existence_not_supported": f"'check_existence' is not supported for '{purl}'", - } - - if command_name in ["metadata", "urls", "validate", "versions"]: - purl_warning = purl_warnings.get(purl, None) - - if purl_warning: - warning = warning_text[purl_warning] - warnings.append(warning) - continue - - log_file = Path(LOG_FILE_LOCATION) - if log_file.is_file(): - with open(log_file, "r") as f: - for line in f: - errors.append(line) - headers_content["errors"] = errors headers_content["warnings"] = warnings headers.append(headers_content) @@ -324,19 +201,13 @@ def construct_headers( required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) @click.option( "--head", is_flag=True, required=False, help="Validate each URL's existence with a head request.", ) -def get_urls(purls, output, file, unique, head): +def get_urls(purls, output, file, head): """ Return known URLs for a PURL. @@ -345,84 +216,57 @@ def get_urls(purls, output, file, unique, head): this package on the package repository. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - urls_info = get_urls_details(purls, output, file, unique, head, command_name) + urls_info = get_urls_details(purls, output, file, head, command_name) json.dump(urls_info, output, indent=4) -def get_urls_details(purls, output, file, unique, head, command_name): +def get_urls_details(purls, output, file, head, command_name): """ Return a dictionary containing URLs for each PURL in the `purls` input - list. `check_urls_purl()` will print an error message to the console - (also displayed in the JSON output) when necessary. + list. """ urls_details = {} urls_details["headers"] = [] urls_details["packages"] = [] + deduplicated_purls = deduplicate_purls(purls) - urls_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - - clear_log_file() - - for purl in input_purls: + for purl in deduplicated_purls: url_detail = {} url_detail["purl"] = purl - purl = purl.strip() if not purl: continue - purl_status = check_urls_purl(purl) - - if command_name == "urls" and purl_status in [ - "validation_error", - "not_valid", - "valid_but_not_supported", - "not_in_upstream_repo", - ]: - urls_warnings[purl] = purl_status - continue - - if command_name == "urls" and purl_status in ["valid_but_not_fully_supported"]: - urls_warnings[purl] = purl_status - - # Add the URLs. - url_purl = PackageURL.from_string(purl) - - url_detail["download_url"] = {"url": purl2url.get_download_url(purl)} + url_detail["download_url"] = purl2url.get_download_url(purl) + if head: + url_detail["download_url"] = {"url": purl2url.get_download_url(purl)} url_detail["inferred_urls"] = [ - {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) + inferred for inferred in purl2url.get_inferred_urls(purl) ] + if head: + url_detail["inferred_urls"] = [ + {"url": inferred} for inferred in purl2url.get_inferred_urls(purl) + ] - url_detail["repo_download_url"] = {"url": purl2url.get_repo_download_url(purl)} - - url_detail["repo_download_url_by_package_type"] = { - "url": purl2url.get_repo_download_url_by_package_type( - url_purl.type, url_purl.namespace, url_purl.name, url_purl.version - ) - } - - url_detail["repo_url"] = {"url": purl2url.get_repo_url(purl)} + url_detail["repository_download_url"] = purl2url.get_repo_download_url(purl) + if head: + url_detail["repository_download_url"] = { + "url": purl2url.get_repo_download_url(purl) + } - url_detail["url"] = {"url": purl2url.get_url(purl)} + url_detail["repository_homepage_url"] = purl2url.get_repo_url(purl) + if head: + url_detail["repository_homepage_url"] = {"url": purl2url.get_repo_url(purl)} - # Add the http status code data. url_list = [ "download_url", - # "inferred_urls" has to be handled separately because it has a nested list - "repo_download_url", - "repo_download_url_by_package_type", - "repo_url", - "url", + "repository_download_url", + "repository_homepage_url", ] if head: for purlcli_url in url_list: @@ -432,7 +276,6 @@ def get_urls_details(purls, output, file, unique, head, command_name): url_detail[purlcli_url]["head_request_status_code"] = make_head_request( url_detail[purlcli_url]["url"] ).get("head_request") - for inferred_url in url_detail["inferred_urls"]: inferred_url["get_request_status_code"] = make_head_request( inferred_url["url"] @@ -440,18 +283,14 @@ def get_urls_details(purls, output, file, unique, head, command_name): inferred_url["head_request_status_code"] = make_head_request( inferred_url["url"] ).get("head_request") - urls_details["packages"].append(url_detail) urls_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, output=output, file=file, head=head, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, - purl_warnings=urls_warnings, ) return urls_details @@ -459,98 +298,21 @@ def get_urls_details(purls, output, file, unique, head, command_name): def make_head_request(url_detail): """ - Make a head request (and as noted below, a get request as well, at least - for now) and return a dictionary containing status code data for the - incoming PURL URL. - - For now, this returns both get and head request status code data so the - user can evaluate -- requests.get() and requests.head() sometimes return - different status codes and sometimes return inaccurate codes, e.g., a - 404 when the URL actually exists. + Return both get and head request status code data for each URL. """ if url_detail is None: return {"get_request": "N/A", "head_request": "N/A"} - get_response = requests.get(url_detail) get_request_status_code = get_response.status_code - head_response = requests.head(url_detail) head_request_status_code = head_response.status_code - # Return a dictionary for readability. return { "get_request": get_request_status_code, "head_request": head_request_status_code, } -def check_urls_purl(purl): - """ - If applicable, return a variable indicating that the input PURL is invalid, - or its type is not supported (or not fully supported) by `urls`, or it - does not exist in the upstream repo. - """ - check_validation = validate_purl(purl) - if check_validation is None: - return "validation_error" - results = check_validation - - if results["valid"] == False: - return "not_valid" - - # Both of these lists are manually constructed from a visual inspection of - # packageurl-python/src/packageurl/contrib/purl2url.py. - - # This list applies to the purl2url.py `repo_url` section: - urls_supported_ecosystems_repo_url = [ - "bitbucket", - "cargo", - "gem", - "github", - "gitlab", - "golang", - "hackage", - "npm", - "nuget", - "pypi", - "rubygems", - ] - - # This list applies to the purl2url.py `download_url` section: - urls_supported_ecosystems_download_url = [ - "bitbucket", - "cargo", - "gem", - "github", - "gitlab", - "hackage", - "npm", - "nuget", - "rubygems", - ] - - urls_purl = PackageURL.from_string(purl) - - if ( - urls_purl.type not in urls_supported_ecosystems_repo_url - and urls_purl.type not in urls_supported_ecosystems_download_url - ): - return "valid_but_not_supported" - - if results["exists"] == False: - return "not_in_upstream_repo" - - if ( - urls_purl.type in urls_supported_ecosystems_repo_url - and urls_purl.type not in urls_supported_ecosystems_download_url - ) or ( - urls_purl.type not in urls_supported_ecosystems_repo_url - and urls_purl.type in urls_supported_ecosystems_download_url - ): - - return "valid_but_not_fully_supported" - - @purlcli.command(name="validate") @click.option( "--purl", @@ -576,150 +338,63 @@ def check_urls_purl(purl): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) -def validate(purls, output, file, unique): +def validate(purls, output, file): """ Validate PURL syntax and existence. Check that the syntax of a PURL is correct. Check that the PURL exists using the PurlDB. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - validated_purls = get_validate_details(purls, output, file, unique, command_name) + validated_purls = get_validate_details(purls, output, file, command_name) json.dump(validated_purls, output, indent=4) -def get_validate_details(purls, output, file, unique, command_name): +def get_validate_details(purls, output, file, command_name): """ Return a dictionary containing validation data for each PURL in the `purls` input list. """ validate_details = {} validate_details["headers"] = [] - - validate_warnings = {} - - input_purls, normalized_purls = normalize_purls(purls, unique) - validate_details["packages"] = [] + deduplicated_purls = deduplicate_purls(purls) - clear_log_file() - - for purl in input_purls: + for purl in deduplicated_purls: purl = purl.strip() if not purl: continue - validated_purl = check_validate_purl(purl) - - if command_name == "validate" and validated_purl in [ - "validation_error", - "not_valid", - "valid_but_not_supported", - "not_in_upstream_repo", - "check_existence_not_supported", - ]: - validate_warnings[purl] = validated_purl - - if validated_purl: - validate_details["packages"].append(validate_purl(purl)) + original_validate_purl = validate_purl(purl) + reordered_validate_purl = { + "purl": original_validate_purl.pop("purl"), + **original_validate_purl, + } + validate_details["packages"].append(reordered_validate_purl) validate_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=deduplicated_purls, output=output, file=file, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, - purl_warnings=validate_warnings, ) return validate_details -def check_validate_purl(purl): - """ - As applicable, return a variable indicating that the input PURL is - valid/invalid or does not exist in the upstream repo. - """ - check_validation = validate_purl(purl) - if check_validation is None: - return "validation_error" - results = check_validation - - if results["valid"] == False: - return "not_valid" - - if results["exists"] == False: - return "not_in_upstream_repo" - - if results["exists"] == True: - return check_validation - - if results["exists"] == None: - return "check_existence_not_supported" - - def validate_purl(purl): """ Return a JSON object containing data from the PurlDB `validate` endpoint regarding the validity of the input PURL. - - Based on packagedb.package_managers VERSION_API_CLASSES_BY_PACKAGE_TYPE - and packagedb/api.py class PurlValidateViewSet(viewsets.ViewSet) - -- and supported by testing the command -- it appears that the `validate` - command `check_existence` parameter supports the following PURL types: - - "cargo", - "composer", - "deb", - "gem", - "golang", - "hex", - "maven", - "npm", - "nuget", - "pypi", """ - logger = logging.getLogger(__name__) - api_query = "https://public.purldb.io/api/validate/" request_body = {"purl": purl, "check_existence": True} + response = requests.get(api_query, params=request_body).json() - try: - response = requests.get(api_query, params=request_body).json() - - except json.decoder.JSONDecodeError as e: - - print(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") - - logging.basicConfig( - filename=LOG_FILE_LOCATION, - level=logging.ERROR, - format="%(levelname)s - %(message)s", - filemode="w", - ) - - logger.error(f"validate_purl(): json.decoder.JSONDecodeError for '{purl}': {e}") - - except Exception as e: - print(f"'validate' endpoint error for '{purl}': {e}") - - else: - if response is None: - print(f"'{purl}' -- response.status_code for None = {response.status_code}") - return response + return response @purlcli.command(name="versions") @@ -747,71 +422,49 @@ def validate_purl(purl): required=False, help="Read a list of PURLs from a FILE, one per line.", ) -@click.option( - "--unique", - is_flag=True, - required=False, - help="Return data only for unique PURLs.", -) -def get_versions(purls, output, file, unique): +def get_versions(purls, output, file): """ List all known versions for a PURL. """ check_for_duplicate_input_sources(purls, file) - if file: purls = file.read().splitlines(False) - context = click.get_current_context() command_name = context.command.name - - purl_versions = get_versions_details(purls, output, file, unique, command_name) + purl_versions = get_versions_details(purls, output, file, command_name) json.dump(purl_versions, output, indent=4) -def get_versions_details(purls, output, file, unique, command_name): +def get_versions_details(purls, output, file, command_name): """ Return a list of dictionaries containing version-related data for each PURL - in the `purls` input list. `check_versions_purl()` will print an error - message to the console (also displayed in the JSON output) when necessary. + in the `purls` input list. """ versions_details = {} versions_details["headers"] = [] versions_details["packages"] = [] - versions_warnings = {} + raw_purls = [] + for input_purl in purls: + raw_purl = re.split("[@,]+", input_purl)[0] + raw_purls.append(raw_purl) - input_purls, normalized_purls = normalize_purls(purls, unique) - - clear_log_file() - - for purl in input_purls: - purl = purl.strip() - if not purl: + deduplicated_purls = deduplicate_purls(raw_purls) + for deduplicated_purl in deduplicated_purls: + deduplicated_purl = deduplicated_purl.strip() + if not deduplicated_purl: continue - purl_data = {} - purl_data["purl"] = purl - - versions_purl = check_versions_purl(purl) - - if command_name == "versions" and versions_purl: - versions_warnings[purl] = versions_purl - continue + purl_data["purl"] = deduplicated_purl - version_collection = collect_versions(purl) - - purl_data["versions"] = version_collection - versions_details["packages"].append(purl_data) + version_collection = collect_versions(deduplicated_purl) + versions_details["packages"].extend(version_collection) versions_details["headers"] = construct_headers( - purls=purls, + deduplicated_purls=purls, output=output, file=file, command_name=command_name, - normalized_purls=normalized_purls, - unique=unique, - purl_warnings=versions_warnings, ) return versions_details @@ -819,100 +472,35 @@ def get_versions_details(purls, output, file, unique, command_name): def collect_versions(purl): """ - Return a list of version objects collected from fetchcode/package_versions.py. + Return a list of version objects for each input PURL. """ collected_versions = [] for package_version in list(versions(purl)): purl_version_data = {} purl_version = package_version.value - # We use `versions()` from fetchcode/package_versions.py, which - # keeps the version (if any) of the input PURL in its output, so - # "pkg:pypi/fetchcode@0.3.0" is returned as - # "pkg:pypi/fetchcode@0.3.0@0.1.0", "pkg:pypi/fetchcode@0.3.0@0.2.0" - # etc. Thus, we remove any string starting with `@` first. - raw_purl = purl = re.split("[@,]+", purl)[0] - nested_purl = raw_purl + "@" + f"{purl_version}" - - purl_version_data["purl"] = nested_purl + purl_version_data["purl"] = purl purl_version_data["version"] = f"{purl_version}" - purl_version_data["release_date"] = f"{package_version.release_date}" + pkg_ver_release_date_no_time = None + if package_version.release_date: + pkg_ver_release_date = package_version.release_date + pkg_ver_release_date_no_time = str(pkg_ver_release_date.date()) + purl_version_data["release_date"] = f"{pkg_ver_release_date_no_time}" + + purl_version_data["release_date"] = pkg_ver_release_date_no_time collected_versions.append(purl_version_data) return collected_versions -def check_versions_purl(purl): - """ - Return a variable identifying the message for printing to the console by - get_versions_details() if (1) the input PURL is invalid, (2) its type is not - supported by `versions` or (3) its existence was not validated (e.g., - "does not exist in the upstream repo"). - - This message will also be reported by construct_headers() in the - `warnings` field of the `header` section of the JSON object returned by - the `versions` command. - - Note for dev purposes: SUPPORTED_ECOSYSTEMS (imported from - fetchcode.package_versions) comprises the following types: - [ - "cargo", - "composer", - "conan", - "deb", - "gem", - "github", - "golang", - "hex", - "maven", - "npm", - "nuget", - "pypi", - ] - """ - check_validation = validate_purl(purl) - if check_validation is None: - return "validation_error" - results = check_validation - - if results["valid"] == False: - return "not_valid" - - supported = SUPPORTED_ECOSYSTEMS - versions_purl = PackageURL.from_string(purl) - - if versions_purl.type not in supported: - return "valid_but_not_supported" - - if results["exists"] == False: - return "not_in_upstream_repo" - - if results["exists"] == None: - return "check_existence_not_supported" - - # This handles the conflict between the `validate`` endpoint (treats - # both "pkg:deb/debian/2ping" and "pkg:deb/2ping" as valid) and - # fetchcode.package_versions versions() (returns None for "pkg:deb/2ping"). - if versions(purl) is None: - return "valid_but_not_supported" - - def check_for_duplicate_input_sources(purls, file): if purls and file: raise click.UsageError("Use either purls or file but not both.") - - if not (purls or file): + elif not (purls or file): raise click.UsageError("Use either purls or file.") -def clear_log_file(): - log_file = Path(LOG_FILE_LOCATION) - - if log_file.is_file(): - os.remove(log_file) - - class D2DPackage(NamedTuple): """ A package to use in d2d, identifier by its PURL and qualified by is package_content which is one diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json index 1f5cb267..f4300124 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_metadata_output.json @@ -12,711 +12,381 @@ "pkg:pypi/fetchcode@0.3.0os=windows", "pkg:pypi/fetchcode@5.0.0", "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" + "pkg:rubygems/pronto-goodcheck" ], "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `metadata` command", - "'pkg:gem/rails' not supported with `metadata` command", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] + "warnings": [] } ], "packages": [ { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "purl": "pkg:pypi/fetchcode@0.1.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.2.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:pypi/fetchcode@0.4.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.4.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { "purl": "pkg:pypi/fetchcode@0.3.0", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": null, + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null }, { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": { - "os": "windows" - }, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "purl": "pkg:pypi/fetchcode@0.3.0", + "type": "pypi", + "namespace": null, + "name": "fetchcode", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/nexB/fetchcode", + "download_url": null, + "api_url": "https://pypi.org/pypi/fetchcode/json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "Apache-2.0", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:cargo/banquo@0.1.0", + "type": "cargo", + "namespace": null, + "name": "banquo", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", + "api_url": "https://crates.io/api/v1/crates/banquo", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/cpslab-asu/banquo", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": "BSD-3-Clause", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + }, + { + "purl": "pkg:rubygems/pronto-goodcheck@0.2.0", + "type": "rubygems", + "namespace": null, + "name": "pronto-goodcheck", + "version": "0.2.0", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://rubygems.org/gems/pronto-goodcheck", + "repository_download_url": null, + "api_data_url": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/aergonaut/pronto-goodcheck", + "download_url": "https://rubygems.org/gems/pronto-goodcheck-0.2.0.gem", + "api_url": "https://rubygems.org/api/v2/rubygems/pronto-goodcheck/versions/0.2.0.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": [ + "MIT" + ], + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [] }, { - "purl": "pkg:cargo/banquo", - "metadata": [ - { - "purl": "pkg:cargo/banquo", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "purl": "pkg:rubygems/pronto-goodcheck@0.1.2", + "type": "rubygems", + "namespace": null, + "name": "pronto-goodcheck", + "version": "0.1.2", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://rubygems.org/gems/pronto-goodcheck", + "repository_download_url": null, + "api_data_url": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/aergonaut/pronto-goodcheck", + "download_url": "https://rubygems.org/gems/pronto-goodcheck-0.1.2.gem", + "api_url": "https://rubygems.org/api/v2/rubygems/pronto-goodcheck/versions/0.1.2.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": [ + "MIT" + ], + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [] }, { - "purl": "pkg:rubygems/rails", - "metadata": [ - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": [ - "MIT" - ], - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] + "purl": "pkg:rubygems/pronto-goodcheck@0.1.1", + "type": "rubygems", + "namespace": null, + "name": "pronto-goodcheck", + "version": "0.1.1", + "qualifiers": {}, + "subpath": null, + "repository_homepage_url": "https://rubygems.org/gems/pronto-goodcheck", + "repository_download_url": null, + "api_data_url": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/aergonaut/pronto-goodcheck", + "download_url": "https://rubygems.org/gems/pronto-goodcheck-0.1.1.gem", + "api_url": "https://rubygems.org/api/v2/rubygems/pronto-goodcheck/versions/0.1.1.json", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": [ + "MIT" + ], + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [] } ] } diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json deleted file mode 100644 index a237dae1..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_superseded.json +++ /dev/null @@ -1,588 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.1.0", - "options": { - "command": "metadata", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "--file": null, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `metadata` command", - "'pkg:gem/rails' not supported with `metadata` command" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": { - "os": "windows" - }, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": [ - "MIT" - ], - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json deleted file mode 100644 index 5ede4644..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_metadata_output_unique.json +++ /dev/null @@ -1,353 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "metadata", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "'pkg:nginx/nginx' not supported with `metadata` command", - "'pkg:gem/rails' not supported with `metadata` command", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - { - "purl": "pkg:pypi/fetchcode", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": null, - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.2.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.3.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "type": "pypi", - "namespace": null, - "name": "fetchcode", - "version": "0.4.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/nexB/fetchcode", - "download_url": "https://files.pythonhosted.org/packages/38/76/4c303fb8e4dd29b0a72915dd74d687cd323ee6836ba7d8cddb080b175eca/fetchcode-0.4.0-py3-none-any.whl", - "api_url": "https://pypi.org/pypi/fetchcode/json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "Apache-2.0", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - }, - { - "purl": "pkg:cargo/banquo", - "metadata": [ - { - "purl": "pkg:cargo/banquo", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - }, - { - "purl": "pkg:cargo/banquo@0.1.0", - "type": "cargo", - "namespace": null, - "name": "banquo", - "version": "0.1.0", - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": "https://crates.io//api/v1/crates/banquo/0.1.0/download", - "api_url": "https://crates.io/api/v1/crates/banquo", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/cpslab-asu/banquo", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": "BSD-3-Clause", - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - }, - { - "purl": "pkg:rubygems/rails", - "metadata": [ - { - "purl": "pkg:rubygems/rails", - "type": "rubygems", - "namespace": null, - "name": "rails", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": null, - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://rubyonrails.org", - "download_url": "https://rubygems.org/gems/rails-7.1.3.2.gem", - "api_url": "https://rubygems.org/api/v1/gems/rails.json", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": "https://github.com/rails/rails/issues", - "code_view_url": "https://github.com/rails/rails/tree/v7.1.3.2", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": [ - "MIT" - ], - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null - } - ] - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json index dc1d1e4a..4a940958 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output.json @@ -8,349 +8,113 @@ "--purl": [ "pkg:pypi/fetchcode", "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", "pkg:pypi/dejacode", "pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", "pkg:cargo/banquo", "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", "pkg:gem/bundler-sass", "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", "pkg:nuget/auth0-aspnet@1.1.0" ], "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command", - "'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:pypi/dejacode' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command", - "'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `urls` command", - "'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command", - "'pkg:pypi/matchcode' does not exist in the upstream repo", - "'abcdefg' not valid", - "'pkg/abc' not valid" - ] + "warnings": [] } ], "packages": [ { "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/" - } + "https://pypi.org/project/fetchcode/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/" - }, - "url": { - "url": "https://pypi.org/project/fetchcode/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/fetchcode/" }, { "purl": "pkg:pypi/fetchcode@0.3.0", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/0.3.0/" - } + "https://pypi.org/project/fetchcode/0.3.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/0.3.0/" - }, - "url": { - "url": "https://pypi.org/project/fetchcode/0.3.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/fetchcode/0.3.0/" }, { "purl": "pkg:pypi/dejacode", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/" - } + "https://pypi.org/project/dejacode/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/" }, { "purl": "pkg:pypi/dejacode@5.0.0", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:pypi/dejacode@5.0.0?os=windows", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } - }, - { - "purl": "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "https://pypi.org/project/dejacode/5.0.0/" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } - }, - { - "purl": "pkg:pypi/dejacode@5.0.0#how/are/you", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/" - } + "repository_download_url": null, + "repository_homepage_url": "https://pypi.org/project/dejacode/5.0.0/" }, { "purl": "pkg:cargo/banquo", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://crates.io/crates/banquo" - } + "https://crates.io/crates/banquo" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/banquo" - }, - "url": { - "url": "https://crates.io/crates/banquo" - } + "repository_download_url": null, + "repository_homepage_url": "https://crates.io/crates/banquo" }, { "purl": "pkg:cargo/socksprox", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://crates.io/crates/socksprox" - } + "https://crates.io/crates/socksprox" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/socksprox" - }, - "url": { - "url": "https://crates.io/crates/socksprox" - } + "repository_download_url": null, + "repository_homepage_url": "https://crates.io/crates/socksprox" }, { "purl": "pkg:gem/bundler-sass", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } + "https://rubygems.org/gems/bundler-sass" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" - } + "repository_download_url": null, + "repository_homepage_url": "https://rubygems.org/gems/bundler-sass" }, { "purl": "pkg:rubygems/bundler-sass", - "download_url": { - "url": null - }, + "download_url": null, "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } + "https://rubygems.org/gems/bundler-sass" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" - } + "repository_download_url": null, + "repository_homepage_url": "https://rubygems.org/gems/bundler-sass" }, { "purl": "pkg:nuget/auth0-aspnet@1.1.0", - "download_url": { - "url": "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0" - }, + "download_url": "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0", "inferred_urls": [ - { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" - }, - { - "url": "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0" - } + "https://www.nuget.org/packages/auth0-aspnet/1.1.0", + "https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0" ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" - }, - "url": { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" - } + "repository_download_url": null, + "repository_homepage_url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0" } ] } diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json index d5b256e4..0237229f 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head.json @@ -8,69 +8,21 @@ "--purl": [ "pkg:pypi/fetchcode", "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", "pkg:pypi/dejacode", "pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", "pkg:cargo/banquo", "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", "pkg:gem/bundler-sass", "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", "pkg:nuget/auth0-aspnet@1.1.0" ], "--file": null, "--head": true, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command", - "'pkg:pypi/fetchcode@0.3.0' not fully supported with `urls` command", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:pypi/dejacode' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0?os=windows' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' not fully supported with `urls` command", - "'pkg:pypi/dejacode@5.0.0#how/are/you' not fully supported with `urls` command", - "'pkg:pypi/dejacode@10.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `urls` command", - "'pkg:nginx/nginx@0.8.9?os=windows' not supported with `urls` command", - "'pkg:pypi/matchcode' does not exist in the upstream repo", - "'abcdefg' not valid", - "'pkg/abc' not valid" - ] + "warnings": [] } ], "packages": [ @@ -88,22 +40,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { + "repository_homepage_url": { "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -123,22 +65,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/0.3.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { + "repository_homepage_url": { "url": "https://pypi.org/project/fetchcode/0.3.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -158,22 +90,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -193,22 +115,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -228,92 +140,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - }, - { - "purl": "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - }, - { - "purl": "pkg:pypi/dejacode@5.0.0#how/are/you", - "download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - } - ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/5.0.0/", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { + "repository_homepage_url": { "url": "https://pypi.org/project/dejacode/5.0.0/", "get_request_status_code": 200, "head_request_status_code": 200 @@ -333,22 +165,12 @@ "head_request_status_code": 404 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { - "url": "https://crates.io/crates/banquo", - "get_request_status_code": 404, - "head_request_status_code": 404 - }, - "url": { + "repository_homepage_url": { "url": "https://crates.io/crates/banquo", "get_request_status_code": 404, "head_request_status_code": 404 @@ -368,22 +190,12 @@ "head_request_status_code": 404 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { - "url": "https://crates.io/crates/socksprox", - "get_request_status_code": 404, - "head_request_status_code": 404 - }, - "url": { + "repository_homepage_url": { "url": "https://crates.io/crates/socksprox", "get_request_status_code": 404, "head_request_status_code": 404 @@ -403,22 +215,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { + "repository_homepage_url": { "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 @@ -438,22 +240,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { + "repository_homepage_url": { "url": "https://rubygems.org/gems/bundler-sass", "get_request_status_code": 200, "head_request_status_code": 200 @@ -478,22 +270,12 @@ "head_request_status_code": 404 } ], - "repo_download_url": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_download_url_by_package_type": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_url": { - "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0", - "get_request_status_code": 200, - "head_request_status_code": 404 - }, - "url": { + "repository_homepage_url": { "url": "https://www.nuget.org/packages/auth0-aspnet/1.1.0", "get_request_status_code": 200, "head_request_status_code": 404 diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json index 5f6c2e21..2c157d29 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json +++ b/purldb-toolkit/tests/data/purlcli/expected_urls_output_head_mock.json @@ -12,13 +12,8 @@ "--head": true, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode" - ], "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command" - ] + "warnings": [] } ], "packages": [ @@ -36,22 +31,12 @@ "head_request_status_code": 200 } ], - "repo_download_url": { - "url": null, - "get_request_status_code": "N/A", - "head_request_status_code": "N/A" - }, - "repo_download_url_by_package_type": { + "repository_download_url": { "url": null, "get_request_status_code": "N/A", "head_request_status_code": "N/A" }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/", - "get_request_status_code": 200, - "head_request_status_code": 200 - }, - "url": { + "repository_homepage_url": { "url": "https://pypi.org/project/fetchcode/", "get_request_status_code": 200, "head_request_status_code": 200 diff --git a/purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json deleted file mode 100644 index 193b8096..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_urls_output_unique.json +++ /dev/null @@ -1,240 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.1.0", - "options": { - "command": "urls", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - "pkg:gem/bundler-sass", - "pkg:rubygems/bundler-sass", - "pkg:pypi/matchcode", - "abcdefg", - "pkg/abc", - "pkg:nuget/auth0-aspnet@1.1.0" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0?os=windows' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0os=windows' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@5.0.0#how/are/you' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:pypi/dejacode@10.0.0' normalized to 'pkg:pypi/dejacode'", - "input PURL: 'pkg:nginx/nginx@0.8.9?os=windows' normalized to 'pkg:nginx/nginx'", - "input PURL: 'pkg:nuget/auth0-aspnet@1.1.0' normalized to 'pkg:nuget/auth0-aspnet'", - "'pkg:pypi/fetchcode' not fully supported with `urls` command", - "'pkg:pypi/dejacode' not fully supported with `urls` command", - "'pkg:nginx/nginx' not supported with `urls` command", - "'pkg:pypi/matchcode' does not exist in the upstream repo", - "'abcdefg' not valid", - "'pkg/abc' not valid" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/" - }, - "url": { - "url": "https://pypi.org/project/fetchcode/" - } - }, - { - "purl": "pkg:pypi/dejacode", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/dejacode/" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://pypi.org/project/dejacode/" - }, - "url": { - "url": "https://pypi.org/project/dejacode/" - } - }, - { - "purl": "pkg:cargo/banquo", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://crates.io/crates/banquo" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/banquo" - }, - "url": { - "url": "https://crates.io/crates/banquo" - } - }, - { - "purl": "pkg:cargo/socksprox", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://crates.io/crates/socksprox" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://crates.io/crates/socksprox" - }, - "url": { - "url": "https://crates.io/crates/socksprox" - } - }, - { - "purl": "pkg:gem/bundler-sass", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" - } - }, - { - "purl": "pkg:rubygems/bundler-sass", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass" - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass" - } - }, - { - "purl": "pkg:nuget/auth0-aspnet", - "download_url": { - "url": null - }, - "inferred_urls": [ - { - "url": "https://www.nuget.org/packages/auth0-aspnet" - } - ], - "repo_download_url": { - "url": null - }, - "repo_download_url_by_package_type": { - "url": null - }, - "repo_url": { - "url": "https://www.nuget.org/packages/auth0-aspnet" - }, - "url": { - "url": "https://www.nuget.org/packages/auth0-aspnet" - } - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_validate_output.json b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json index ac85f7c7..de84f52d 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_validate_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_validate_output.json @@ -19,24 +19,8 @@ "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'check_existence' is not supported for 'pkg:nginx/nginx'", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] + "warnings": [] } ], "packages": [ diff --git a/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json deleted file mode 100644 index 280388f2..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_validate_output_unique.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "validate", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:gem/rails", - "pkg:rubygems/rails" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "'check_existence' is not supported for 'pkg:nginx/nginx'", - "'check_existence' is not supported for 'pkg:rubygems/rails'" - ] - } - ], - "packages": [ - { - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode" - }, - { - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:cargo/banquo" - }, - { - "exists": null, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx", - "valid": true - }, - { - "valid": true, - "exists": true, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:gem/rails" - }, - { - "exists": null, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/rails", - "valid": true - } - ] -} diff --git a/purldb-toolkit/tests/data/purlcli/expected_versions_output.json b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json index 93feeb96..428979d9 100644 --- a/purldb-toolkit/tests/data/purlcli/expected_versions_output.json +++ b/purldb-toolkit/tests/data/purlcli/expected_versions_output.json @@ -12,165 +12,95 @@ "pkg:pypi/fetchcode@0.3.0os=windows", "pkg:pypi/fetchcode@5.0.0", "pkg:cargo/banquo", - "pkg:nginx/nginx", "pkg:hex/coherence@0.1.0" ], "--file": null, "--output": "" }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:hex/coherence@0.1.0" - ], "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@0.3.0os=windows' does not exist in the upstream repo", - "'pkg:pypi/fetchcode@5.0.0' does not exist in the upstream repo", - "'pkg:nginx/nginx' not supported with `versions` command" - ] + "warnings": [] } ], "packages": [ { "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] + "version": "0.1.0", + "release_date": "2021-08-25" }, { - "purl": "pkg:pypi/fetchcode@0.3.0", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] + "purl": "pkg:pypi/fetchcode", + "version": "0.2.0", + "release_date": "2022-09-14" }, { - "purl": "pkg:pypi/fetchcode@0.3.0?os=windows", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] + "purl": "pkg:pypi/fetchcode", + "version": "0.3.0", + "release_date": "2023-12-18" }, { "purl": "pkg:cargo/banquo", - "versions": [ - { - "purl": "pkg:cargo/banquo@0.1.0", - "version": "0.1.0", - "release_date": "2024-02-07 23:21:50.548891+00:00" - } - ] + "version": "0.1.0", + "release_date": "2024-02-07" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.8.0", + "release_date": "2023-09-22" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.5.2", + "release_date": "2018-09-03" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.5.1", + "release_date": "2018-08-28" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.5.0", + "release_date": "2017-08-02" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.4.0", + "release_date": "2017-07-03" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.3.1", + "release_date": "2016-11-27" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.3.0", + "release_date": "2016-08-28" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.2.0", + "release_date": "2016-07-30" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.1.3", + "release_date": "2016-07-19" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.1.2", + "release_date": "2016-07-12" + }, + { + "purl": "pkg:hex/coherence", + "version": "0.1.1", + "release_date": "2016-07-11" }, { - "purl": "pkg:hex/coherence@0.1.0", - "versions": [ - { - "purl": "pkg:hex/coherence@0.8.0", - "version": "0.8.0", - "release_date": "2023-09-22 18:28:36.224103+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.2", - "version": "0.5.2", - "release_date": "2018-09-03 23:52:38.161321+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.1", - "version": "0.5.1", - "release_date": "2018-08-28 01:33:14.565151+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.0", - "version": "0.5.0", - "release_date": "2017-08-02 06:23:12.948525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.4.0", - "version": "0.4.0", - "release_date": "2017-07-03 21:55:56.591426+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.1", - "version": "0.3.1", - "release_date": "2016-11-27 05:30:34.553920+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.0", - "version": "0.3.0", - "release_date": "2016-08-28 19:04:10.794525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.2.0", - "version": "0.2.0", - "release_date": "2016-07-30 21:07:45.377540+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.3", - "version": "0.1.3", - "release_date": "2016-07-19 03:33:09.185782+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.2", - "version": "0.1.2", - "release_date": "2016-07-12 18:41:27.084599+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.1", - "version": "0.1.1", - "release_date": "2016-07-11 13:56:26.388096+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.0", - "version": "0.1.0", - "release_date": "2016-07-11 06:52:43.545719+00:00" - } - ] + "purl": "pkg:hex/coherence", + "version": "0.1.0", + "release_date": "2016-07-11" } ] } diff --git a/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json b/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json deleted file mode 100644 index 1991c21f..00000000 --- a/purldb-toolkit/tests/data/purlcli/expected_versions_output_unique.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:hex/coherence@0.1.0" - ], - "--file": null, - "--unique": true, - "--output": "" - }, - "purls": [ - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "pkg:pypi/fetchcode@0.3.0os=windows", - "pkg:pypi/fetchcode@5.0.0", - "pkg:cargo/banquo", - "pkg:nginx/nginx", - "pkg:hex/coherence@0.1.0" - ], - "errors": [], - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.3.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0?os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.3.0os=windows' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@5.0.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:hex/coherence@0.1.0' normalized to 'pkg:hex/coherence'", - "'pkg:nginx/nginx' not supported with `versions` command" - ] - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00" - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00" - } - ] - }, - { - "purl": "pkg:cargo/banquo", - "versions": [ - { - "purl": "pkg:cargo/banquo@0.1.0", - "version": "0.1.0", - "release_date": "2024-02-07 23:21:50.548891+00:00" - } - ] - }, - { - "purl": "pkg:hex/coherence", - "versions": [ - { - "purl": "pkg:hex/coherence@0.8.0", - "version": "0.8.0", - "release_date": "2023-09-22 18:28:36.224103+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.2", - "version": "0.5.2", - "release_date": "2018-09-03 23:52:38.161321+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.1", - "version": "0.5.1", - "release_date": "2018-08-28 01:33:14.565151+00:00" - }, - { - "purl": "pkg:hex/coherence@0.5.0", - "version": "0.5.0", - "release_date": "2017-08-02 06:23:12.948525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.4.0", - "version": "0.4.0", - "release_date": "2017-07-03 21:55:56.591426+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.1", - "version": "0.3.1", - "release_date": "2016-11-27 05:30:34.553920+00:00" - }, - { - "purl": "pkg:hex/coherence@0.3.0", - "version": "0.3.0", - "release_date": "2016-08-28 19:04:10.794525+00:00" - }, - { - "purl": "pkg:hex/coherence@0.2.0", - "version": "0.2.0", - "release_date": "2016-07-30 21:07:45.377540+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.3", - "version": "0.1.3", - "release_date": "2016-07-19 03:33:09.185782+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.2", - "version": "0.1.2", - "release_date": "2016-07-12 18:41:27.084599+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.1", - "version": "0.1.1", - "release_date": "2016-07-11 13:56:26.388096+00:00" - }, - { - "purl": "pkg:hex/coherence@0.1.0", - "version": "0.1.0", - "release_date": "2016-07-11 06:52:43.545719+00:00" - } - ] - } - ] -} diff --git a/purldb-toolkit/tests/test_purlcli.py b/purldb-toolkit/tests/test_purlcli.py index 91e364a7..5b898dba 100644 --- a/purldb-toolkit/tests/test_purlcli.py +++ b/purldb-toolkit/tests/test_purlcli.py @@ -10,15 +10,14 @@ import json import os from collections import OrderedDict -from contextlib import redirect_stdout -from io import StringIO from unittest import mock import pytest import requests from click.testing import CliRunner from commoncode.testcase import FileDrivenTesting -from purldb_toolkit import cli_test_utils, purlcli +from purldb_toolkit import cli_test_utils +from purldb_toolkit import purlcli test_env = FileDrivenTesting() test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") @@ -57,57 +56,20 @@ def test_metadata_cli_no_input_sources(self): assert result.exit_code == 2 @mock.patch("purldb_toolkit.purlcli.collect_metadata") - @mock.patch("purldb_toolkit.purlcli.check_metadata_purl") - def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata): - + def test_metadata_details(self, mock_collect_metadata): mock_collect_metadata.return_value = [ OrderedDict( [ - ("purl", "pkg:pypi/fetchcode"), + ("purl", "pkg:pypi/fetchcode@0.1.0"), ("type", "pypi"), ("namespace", None), ("name", "fetchcode"), - ("version", None), + ("version", "0.1.0"), ("qualifiers", OrderedDict()), ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), ("repository_homepage_url", None), ("repository_download_url", None), ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.1.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.1.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -135,9 +97,6 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -149,6 +108,9 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) ("version", "0.2.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -176,9 +138,6 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), OrderedDict( @@ -190,6 +149,9 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) ("version", "0.3.0"), ("qualifiers", OrderedDict()), ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), ("primary_language", None), ("description", None), ("release_date", None), @@ -217,15 +179,10 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) ("dependencies", []), ("contains_source_code", None), ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), ] ), ] - mock_check_metadata_purl.return_value = None - expected_data = { "headers": [ { @@ -237,193 +194,146 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } ], "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ("download_url", None), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.1.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.1.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ( - "download_url", - "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - ), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.1.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.1.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.2.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.2.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ( - "download_url", - "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - ), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.2.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.2.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.3.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.3.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ("homepage_url", "https://github.com/nexB/fetchcode"), - ( - "download_url", - "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - ), - ("api_url", "https://pypi.org/pypi/fetchcode/json"), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.3.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.3.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", ), - ], - } + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), ], } input_purls = ["pkg:pypi/fetchcode"] - output = "" file = "" command_name = "metadata" - unique = False purl_metadata_data = purlcli.get_metadata_details( input_purls, output, file, - unique, command_name, ) @@ -432,195 +342,33 @@ def test_metadata_details(self, mock_check_metadata_purl, mock_collect_metadata) assert purl_metadata_data == expected_data - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_metadata_purl(self, mock_validate_purl): - mock_validate_purl.return_value = { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/bundler-sass", - } - input_purl = "pkg:rubygems/bundler-sass" - expected = "check_existence_not_supported" - purl_metadata = purlcli.check_metadata_purl(input_purl) - - assert purl_metadata == expected - - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_metadata_purl_multiple(self, mock_validate_purl): - mock_validate_purl.side_effect = [ - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode", - }, - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:gem/bundler-sass", - }, - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/bundler-sass", - }, - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx", - }, - { - "valid": True, - "exists": False, - "message": "The provided PackageURL is valid, but does not exist in the upstream repo.", - "purl": "pkg:pypi/zzzzz", - }, - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "pkg:pypi/?fetchcode", - }, - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "zzzzz", - }, - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:maven/axis/axis@1.0", - }, - ] - - input_purls_and_expected_states = [ - ["pkg:pypi/fetchcode", None], - ["pkg:gem/bundler-sass", "valid_but_not_supported"], - ["pkg:rubygems/bundler-sass", "check_existence_not_supported"], - ["pkg:nginx/nginx", "valid_but_not_supported"], - ["pkg:pypi/zzzzz", "not_in_upstream_repo"], - ["pkg:pypi/?fetchcode", "not_valid"], - ["zzzzz", "not_valid"], - ["pkg:maven/axis/axis@1.0", "valid_but_not_supported"], + def test_deduplicate_purls(self): + input_purls = [ + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", + "pkg:pypi/fetchcode@0.2.0", ] - - for input_purl, expected_state in input_purls_and_expected_states: - purl_metadata = purlcli.check_metadata_purl(input_purl) - assert purl_metadata == expected_state - - @pytest.mark.parametrize( - "test_input,expected_input_purls,expected_normalized_purls", - [ - ( - [["pkg:pypi/fetchcode"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode@1.2.3", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ( - [ - ( - "pkg:pypi/fetchcode@1.2.3?howistheweather=rainy", - "pkg:pypi/fetchcode", - ) - ] - ), - ), - ( - [["pkg:pypi/fetchcode?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode?howistheweather=rainy", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode#this/is/a/path"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode#this/is/a/path", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/?fetchcode"]], - (["pkg:pypi/"]), - ([("pkg:pypi/?fetchcode", "pkg:pypi/")]), - ), - ( - [ - [ - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - ] - ], - ( - [ - "pkg:pypi/fetchcode", - "pkg:pypi/dejacode", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - ] - ), - ( - [ - ("pkg:pypi/fetchcode@0.3.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@5.0.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/dejacode", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0?os=windows", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0os=windows", "pkg:pypi/dejacode"), - ( - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode", - ), - ("pkg:pypi/dejacode@5.0.0#how/are/you", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@10.0.0", "pkg:pypi/dejacode"), - ("pkg:cargo/banquo", "pkg:cargo/banquo"), - ("pkg:cargo/socksprox", "pkg:cargo/socksprox"), - ("pkg:nginx/nginx", "pkg:nginx/nginx"), - ("pkg:nginx/nginx@0.8.9?os=windows", "pkg:nginx/nginx"), - ] - ), - ), - ], - ) - def test_normalize_purls( - self, test_input, expected_input_purls, expected_normalized_purls - ): - unique = True - input_purls, normalized_purls = purlcli.normalize_purls(test_input[0], unique) - - assert input_purls == expected_input_purls - assert normalized_purls == expected_normalized_purls + actual_output = purlcli.deduplicate_purls(input_purls) + expected_output = ( + ["pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.2.0"] + ) + assert actual_output == expected_output @pytest.mark.parametrize( "test_input,expected", [ ( [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode@0.2.0", ], [ @@ -630,40 +378,33 @@ def test_normalize_purls( "--file": None, "--output": "", "--purl": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.2.0", + "pkg:pypi/fetchcode@0.2.0", ], "command": "metadata", }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], "tool_name": "purlcli", "tool_version": "0.1.0", - "warnings": [ - "'pkg:gem/bundler-sass' not supported with `metadata` command" - ], + "warnings": [], } ], ), ], ) - def test_construct_headers(self, test_input, expected): + def test_deduplicate_purls_construct_headers(self, test_input, expected): metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - normalized_purls=None, - unique=None, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, ) + cli_test_utils.streamline_headers(expected) cli_test_utils.streamline_headers(metadata_headers) @@ -691,43 +432,25 @@ def test_construct_headers(self, test_input, expected): "pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode@0.2.0", ], - "--unique": True, "command": "metadata", }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], "tool_name": "purlcli", "tool_version": "0.1.0", - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.1.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.2.0' normalized to 'pkg:pypi/fetchcode'", - "'pkg:gem/bundler-sass' not supported with `metadata` command", - ], + "warnings": [], } ], ), ], ) - def test_construct_headers_unique(self, test_input, expected): + def test_construct_headers(self, test_input, expected): metadata_headers = purlcli.construct_headers( test_input, output="", file="", command_name="metadata", head=None, - normalized_purls=[ - ("pkg:gem/bundler-sass", "pkg:gem/bundler-sass"), - ("pkg:pypi/fetchcode", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode"), - ], - unique=True, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, ) + cli_test_utils.streamline_headers(expected) cli_test_utils.streamline_headers(metadata_headers) @@ -746,8 +469,6 @@ def test_urls_cli_head(self, mock_make_head_request): {"head_request": "N/A"}, {"get_request": "N/A"}, {"head_request": "N/A"}, - {"get_request": "N/A"}, - {"head_request": "N/A"}, {"get_request": 200}, {"head_request": 200}, {"get_request": 200}, @@ -755,6 +476,7 @@ def test_urls_cli_head(self, mock_make_head_request): {"get_request": 200}, {"head_request": 200}, ] + expected_result_file = test_env.get_test_loc( "purlcli/expected_urls_output_head_mock.json" ) @@ -783,7 +505,6 @@ def test_urls_cli_head(self, mock_make_head_request): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -844,10 +565,7 @@ def test_urls_cli_no_input_sources(self): assert "Use either purls or file." in result.output assert result.exit_code == 2 - @mock.patch("purldb_toolkit.purlcli.check_urls_purl") - def test_urls_details(self, mock_check_urls_purl): - mock_check_urls_purl.return_value = "valid_but_not_fully_supported" - + def test_urls_details(self): expected_data = { "headers": [ { @@ -859,36 +577,19 @@ def test_urls_details(self, mock_check_urls_purl): "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command" - ], + "warnings": [], } ], "packages": [ { "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": None, - }, + "download_url": None, "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/", - } + "https://pypi.org/project/fetchcode/", ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/", - }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - }, + "repository_download_url": None, + "repository_homepage_url": "https://pypi.org/project/fetchcode/", }, ], } @@ -901,28 +602,12 @@ def test_urls_details(self, mock_check_urls_purl): file="", command_name="urls", head=False, - unique=False, ) cli_test_utils.streamline_headers(expected_data["headers"]) cli_test_utils.streamline_headers(purl_urls["headers"]) assert purl_urls == expected_data - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_urls_purl(self, mock_validate_purl): - mock_validate_purl.return_value = { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode", - } - - input_purl = "pkg:pypi/fetchcode" - expected = "valid_but_not_fully_supported" - purl_urls = purlcli.check_urls_purl(input_purl) - - assert purl_urls == expected - @mock.patch("requests.get") @mock.patch("requests.head") def test_validate_purl_mock_requests_get_and_head( @@ -964,108 +649,45 @@ def mock_requests_get_return_func(): results = purlcli.validate_purl(input_purl) assert mock_request_response == results - @mock.patch("requests.get") - def test_validate_purl_mock_requests_get_jsondecodeerror(self, mock_requests_get): - - def json_decode_failure_exception(**kwargs): - raise json.decoder.JSONDecodeError("test", "[{}]", 0) - - mock_requests_get.return_value.json = json_decode_failure_exception - input_purl = "pkg:pypi/fetchcode" - out = StringIO() - with mock.patch("requests.Response.json", json_decode_failure_exception): - with redirect_stdout(out): - purlcli.validate_purl(input_purl) - results = out.getvalue() - assert ( - "json.decoder.JSONDecodeError for 'pkg:pypi/fetchcode': test: line 1 column 1 (char 0)" - in results - ) - - @mock.patch("requests.get") - def test_validate_purl_mock_requests_get_exception(self, mock_requests_get): - - def raise_exception(**kwargs): - raise Exception - - mock_requests_get.return_value.json = raise_exception - input_purl = "pkg:pypi/fetchcode" - out = StringIO() - with mock.patch("requests.Response.json", raise_exception): - with redirect_stdout(out): - purlcli.validate_purl(input_purl) - results = out.getvalue() - assert "'validate' endpoint error for 'pkg:pypi/fetchcode': \n" in results - - @mock.patch("requests.get") - def test_validate_purl_mock_requests_none(self, mock_requests_get): - - def raise_exception(**kwargs): - raise Exception - - mock_requests_get.return_value.json = raise_exception - input_purl = None - out = StringIO() - with mock.patch("requests.Response.json", raise_exception): - with redirect_stdout(out): - purlcli.validate_purl(input_purl) - results = out.getvalue() - assert "'validate' endpoint error for 'None': \n" in results - class TestPURLCLI_versions(object): @mock.patch("purldb_toolkit.purlcli.collect_versions") - @mock.patch("purldb_toolkit.purlcli.check_versions_purl") - def test_versions_details_multiple( - self, - mock_check_versions_purl, - mock_collect_versions, - ): - - mock_check_versions_purl.side_effect = [ - None, - None, - "valid_but_not_supported", - "valid_but_not_supported", - None, - "not_valid", - ] - + def test_versions_details_multiple(self, mock_collect_versions): mock_collect_versions.side_effect = [ [ { - "purl": "pkg:pypi/fetchcode@0.1.0", + "purl": "pkg:pypi/fetchcode", "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", + "release_date": "2021-08-25", }, { - "purl": "pkg:pypi/fetchcode@0.2.0", + "purl": "pkg:pypi/fetchcode", "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", + "release_date": "2022-09-14", }, { - "purl": "pkg:pypi/fetchcode@0.3.0", + "purl": "pkg:pypi/fetchcode", "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", + "release_date": "2023-12-18", }, ], [ { - "purl": "pkg:gem/bundler-sass@0.1.2", + "purl": "pkg:gem/bundler-sass", "version": "0.1.2", - "release_date": "2013-12-11 00:27:10.097000+00:00", + "release_date": "2013-12-11", } ], [ { - "purl": "pkg:cargo/socksprox@0.1.1", - "release_date": "2024-02-07 23:29:58.801293+00:00", + "purl": "pkg:cargo/socksprox", + "release_date": "2024-02-07", "version": "0.1.1", }, { - "purl": "pkg:cargo/socksprox@0.1.0", - "release_date": "2024-02-07 23:21:05.242366+00:00", + "purl": "pkg:cargo/socksprox", + "release_date": "2024-02-07", "version": "0.1.0", }, ], @@ -1085,7 +707,6 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } @@ -1093,24 +714,19 @@ def test_versions_details_multiple( "packages": [ { "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", - }, - ], - } + "version": "0.1.0", + "release_date": "2021-08-25", + }, + { + "purl": "pkg:pypi/fetchcode", + "version": "0.2.0", + "release_date": "2022-09-14", + }, + { + "purl": "pkg:pypi/fetchcode", + "version": "0.3.0", + "release_date": "2023-12-18", + }, ], }, ], @@ -1127,7 +743,6 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:gem/bundler-sass"], "errors": [], "warnings": [], } @@ -1135,63 +750,12 @@ def test_versions_details_multiple( "packages": [ { "purl": "pkg:gem/bundler-sass", - "versions": [ - { - "purl": "pkg:gem/bundler-sass@0.1.2", - "version": "0.1.2", - "release_date": "2013-12-11 00:27:10.097000+00:00", - } - ], + "version": "0.1.2", + "release_date": "2013-12-11", } ], }, ], - [ - ["pkg:rubygems/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:rubygems/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "errors": [], - "warnings": [ - "'pkg:rubygems/bundler-sass' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ], - [ - ["pkg:nginx/nginx"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:nginx/nginx"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:nginx/nginx"], - "errors": [], - "warnings": [ - "'pkg:nginx/nginx' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ], [ ["pkg:cargo/socksprox"], { @@ -1205,7 +769,6 @@ def test_versions_details_multiple( "--file": None, "--output": "", }, - "purls": ["pkg:cargo/socksprox"], "errors": [], "warnings": [], } @@ -1213,41 +776,15 @@ def test_versions_details_multiple( "packages": [ { "purl": "pkg:cargo/socksprox", - "versions": [ - { - "purl": "pkg:cargo/socksprox@0.1.1", - "version": "0.1.1", - "release_date": "2024-02-07 23:29:58.801293+00:00", - }, - { - "purl": "pkg:cargo/socksprox@0.1.0", - "version": "0.1.0", - "release_date": "2024-02-07 23:21:05.242366+00:00", - }, - ], - } - ], - }, - ], - [ - ["pkg:pypi/?fetchcode"], - { - "headers": [ + "version": "0.1.1", + "release_date": "2024-02-07", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:pypi/?fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/?fetchcode"], - "errors": [], - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } + "purl": "pkg:cargo/socksprox", + "version": "0.1.0", + "release_date": "2024-02-07", + }, ], - "packages": [], }, ], ] @@ -1255,43 +792,37 @@ def test_versions_details_multiple( output = "" file = "" command_name = "versions" - unique = False for input_purl, expected_data in input_purls_and_expected_purl_data: purl_versions_data = purlcli.get_versions_details( - input_purl, output, file, unique, command_name + input_purl, + output, + file, + command_name, ) assert purl_versions_data == expected_data @mock.patch("purldb_toolkit.purlcli.collect_versions") - @mock.patch("purldb_toolkit.purlcli.check_versions_purl") - def test_versions_details( - self, - mock_check_versions_purl, - mock_collect_versions, - ): - + def test_versions_details(self, mock_collect_versions): mock_collect_versions.return_value = [ { - "purl": "pkg:pypi/fetchcode@0.1.0", + "purl": "pkg:pypi/fetchcode", "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", + "release_date": "2021-08-25", }, { - "purl": "pkg:pypi/fetchcode@0.2.0", + "purl": "pkg:pypi/fetchcode", "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", + "release_date": "2022-09-14", }, { - "purl": "pkg:pypi/fetchcode@0.3.0", + "purl": "pkg:pypi/fetchcode", "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", + "release_date": "2023-12-18", }, ] - mock_check_versions_purl.return_value = None - expected_data = { "headers": [ { @@ -1303,7 +834,6 @@ def test_versions_details( "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } @@ -1311,24 +841,19 @@ def test_versions_details( "packages": [ { "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", - }, - ], - } + "version": "0.1.0", + "release_date": "2021-08-25", + }, + { + "purl": "pkg:pypi/fetchcode", + "version": "0.2.0", + "release_date": "2022-09-14", + }, + { + "purl": "pkg:pypi/fetchcode", + "version": "0.3.0", + "release_date": "2023-12-18", + }, ], } @@ -1337,96 +862,15 @@ def test_versions_details( output = "" file = "" command_name = "versions" - unique = False purl_versions_data = purlcli.get_versions_details( input_purls, output, file, - unique, command_name, ) assert purl_versions_data == expected_data - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_versions_purl_multiple(self, mock_validate_purl): - mock_validate_purl.side_effect = [ - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:pypi/fetchcode", - }, - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:gem/bundler-sass", - }, - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/bundler-sass", - }, - { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:nginx/nginx", - }, - { - "valid": True, - "exists": False, - "message": "The provided PackageURL is valid, but does not exist in the upstream repo.", - "purl": "pkg:pypi/zzzzz", - }, - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "pkg:pypi/?fetchcode", - }, - { - "valid": False, - "exists": None, - "message": "The provided PackageURL is not valid.", - "purl": "zzzzz", - }, - { - "valid": True, - "exists": True, - "message": "The provided Package URL is valid, and the package exists in the upstream repo.", - "purl": "pkg:maven/axis/axis@1.0", - }, - ] - input_purls_and_expected_states = [ - ["pkg:pypi/fetchcode", None], - ["pkg:gem/bundler-sass", None], - ["pkg:rubygems/bundler-sass", "valid_but_not_supported"], - ["pkg:nginx/nginx", "valid_but_not_supported"], - ["pkg:pypi/zzzzz", "not_in_upstream_repo"], - ["pkg:pypi/?fetchcode", "not_valid"], - ["zzzzz", "not_valid"], - ["pkg:maven/axis/axis@1.0", None], - ] - for input_purl, expected_state in input_purls_and_expected_states: - purl_versions = purlcli.check_versions_purl(input_purl) - assert purl_versions == expected_state - - @mock.patch("purldb_toolkit.purlcli.validate_purl") - def test_check_versions_purl(self, mock_validate_purl): - mock_validate_purl.return_value = { - "valid": True, - "exists": None, - "message": "The provided PackageURL is valid, but `check_existence` is not supported for this package type.", - "purl": "pkg:rubygems/bundler-sass", - } - input_purl = "pkg:rubygems/bundler-sass" - purl_versions = purlcli.check_versions_purl(input_purl) - expected = "valid_but_not_supported" - assert purl_versions == expected - def streamline_metadata_packages(packages): """ diff --git a/purldb-toolkit/tests/test_purlcli_live.py b/purldb-toolkit/tests/test_purlcli_live.py index 929bc1af..ca14228d 100644 --- a/purldb-toolkit/tests/test_purlcli_live.py +++ b/purldb-toolkit/tests/test_purlcli_live.py @@ -14,7 +14,8 @@ import pytest from click.testing import CliRunner from commoncode.testcase import FileDrivenTesting -from purldb_toolkit import cli_test_utils, purlcli +from purldb_toolkit import cli_test_utils +from purldb_toolkit import purlcli test_env = FileDrivenTesting() test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data") @@ -49,11 +50,7 @@ def test_metadata_cli(self): "--purl", "pkg:cargo/banquo", "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:gem/rails", - "--purl", - "pkg:rubygems/rails", + "pkg:rubygems/pronto-goodcheck", "--output", actual_result_file, ] @@ -76,7 +73,6 @@ def test_metadata_cli(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -102,64 +98,379 @@ def test_metadata_cli(self): for output, expected in result_objects: assert output == expected - # NOTE: To avoid errors from the addition of new versions, we exclude - # "packages" from the result_objects list above and handle here. - # All other live-fetch tests are handled in a similar manner. + """ + NOTE: To avoid errors from the addition of new versions, we exclude + "packages" from the result_objects list above and handle here. All + other live-fetch tests are handled in a similar manner. + """ + compare_packages(expected_data, output_data) + + def test_metadata_cli_duplicate_input_sources(self): + """ + Test the `metadata` command with both `--purl` and `--file` inputs. + """ + options = [ + "--purl", + "pkg:pypi/minecode", + "--file", + test_env.get_test_loc("purlcli/metadata_input_purls.txt"), + "--output", + "-", + ] + runner = CliRunner() + result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + assert "Use either purls or file but not both." in result.output + assert result.exit_code == 2 + + def test_metadata_cli_no_input_sources(self): + """ + Test the `metadata` command with neither `--purl` nor `--file` inputs. + """ + options = [ + "--output", + "-", + ] + runner = CliRunner() + result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + assert "Use either purls or file." in result.output + assert result.exit_code == 2 + + def test_metadata_details(self): + expected_data = { + "headers": [ + { + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "metadata", + "--purl": ["pkg:pypi/fetchcode"], + "--file": None, + "--output": "", + }, + "errors": [], + "warnings": [], + } + ], + "packages": [ + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.1.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.1.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", + ), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.2.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.2.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", + ), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + OrderedDict( + [ + ("purl", "pkg:pypi/fetchcode@0.3.0"), + ("type", "pypi"), + ("namespace", None), + ("name", "fetchcode"), + ("version", "0.3.0"), + ("qualifiers", OrderedDict()), + ("subpath", None), + ("repository_homepage_url", None), + ("repository_download_url", None), + ("api_data_url", None), + ("primary_language", None), + ("description", None), + ("release_date", None), + ("parties", []), + ("keywords", []), + ("homepage_url", "https://github.com/nexB/fetchcode"), + ( + "download_url", + "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", + ), + ("api_url", "https://pypi.org/pypi/fetchcode/json"), + ("size", None), + ("sha1", None), + ("md5", None), + ("sha256", None), + ("sha512", None), + ("bug_tracking_url", None), + ("code_view_url", None), + ("vcs_url", None), + ("copyright", None), + ("license_expression", None), + ("declared_license", "Apache-2.0"), + ("notice_text", None), + ("root_path", None), + ("dependencies", []), + ("contains_source_code", None), + ("source_packages", []), + ] + ), + ], + } + + input_purls = ["pkg:pypi/fetchcode"] + output = "" + file = "" + command_name = "metadata" + + purl_metadata_data = purlcli.get_metadata_details( + input_purls, + output, + file, + command_name, + ) + + cli_test_utils.streamline_headers(purl_metadata_data["headers"]) + streamline_metadata_packages(purl_metadata_data["packages"]) + + cli_test_utils.streamline_headers(expected_data["headers"]) + streamline_metadata_packages(expected_data["packages"]) + + assert purl_metadata_data["headers"] == expected_data["headers"] + compare_packages(expected_data, purl_metadata_data) + + @pytest.mark.parametrize( + "test_input,expected", + [ + ( + [ + "pkg:gem/bundler-sass", + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", + ], + [ + { + "errors": [], + "options": { + "--file": None, + "--output": "", + "--purl": [ + "pkg:gem/bundler-sass", + "pkg:pypi/fetchcode", + "pkg:pypi/fetchcode@0.1.0", + "pkg:pypi/fetchcode@0.2.0", + ], + "command": "metadata", + }, + "tool_name": "purlcli", + "tool_version": "0.2.0", + "warnings": [], + } + ], + ), + ], + ) + def test_construct_headers(self, test_input, expected): + metadata_headers = purlcli.construct_headers( + test_input, + output="", + file="", + command_name="metadata", + head=None, + ) + cli_test_utils.streamline_headers(expected) + cli_test_utils.streamline_headers(metadata_headers) + + assert metadata_headers == expected - compare_packages(expected_data, output_data, "metadata") - def test_metadata_cli_unique(self): +class TestPURLCLI_urls(object): + def test_urls_cli(self): """ - Test the `metadata` command with actual and expected JSON output files - with the `--unique` flag included in the command. + Test the `urls` command with actual and expected JSON output files. """ expected_result_file = test_env.get_test_loc( - "purlcli/expected_metadata_output_unique.json" + "purlcli/expected_urls_output.json" ) - actual_result_file = test_env.get_temp_file("actual_metadata_output.json") + actual_result_file = test_env.get_temp_file("actual_urls_output.json") options = [ "--purl", "pkg:pypi/fetchcode", "--purl", "pkg:pypi/fetchcode@0.3.0", "--purl", - "pkg:pypi/fetchcode@0.3.0?os=windows", + "pkg:pypi/dejacode", "--purl", - "pkg:pypi/fetchcode@0.3.0os=windows", + "pkg:pypi/dejacode@5.0.0", "--purl", - "pkg:pypi/fetchcode@5.0.0", + "pkg:pypi/dejacode@5.0.0?os=windows", "--purl", "pkg:cargo/banquo", "--purl", - "pkg:nginx/nginx", + "pkg:cargo/socksprox", "--purl", - "pkg:gem/rails", + "pkg:gem/bundler-sass", "--purl", - "pkg:rubygems/rails", + "pkg:rubygems/bundler-sass", + "--purl", + "pkg:nuget/auth0-aspnet@1.1.0", "--output", actual_result_file, - "--unique", ] runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) + assert result.exit_code == 0 + + with open(actual_result_file) as f_output: + output_data = json.load(f_output) + cli_test_utils.streamline_headers(output_data["headers"]) + + with open(expected_result_file) as f_expected: + expected_data = json.load(f_expected) + cli_test_utils.streamline_headers(expected_data["headers"]) + + result_objects = [ + ( + output_data["headers"][0]["tool_name"], + expected_data["headers"][0]["tool_name"], + ), + ( + output_data["headers"][0]["warnings"], + expected_data["headers"][0]["warnings"], + ), + ( + output_data["headers"][0]["errors"], + expected_data["headers"][0]["errors"], + ), + ( + output_data["headers"][0]["options"]["command"], + expected_data["headers"][0]["options"]["command"], + ), + ( + output_data["headers"][0]["options"]["--purl"], + expected_data["headers"][0]["options"]["--purl"], + ), + ( + output_data["headers"][0]["options"]["--file"], + expected_data["headers"][0]["options"]["--file"], + ), + (output_data["packages"], expected_data["packages"]), + ] + + for output, expected in result_objects: + assert output == expected + + def test_urls_cli_head(self): + """ + Test the `urls` command with actual and expected JSON output files. + """ + expected_result_file = test_env.get_test_loc( + "purlcli/expected_urls_output_head.json" + ) + actual_result_file = test_env.get_temp_file("actual_urls_output_head.json") + options = [ + "--purl", + "pkg:pypi/fetchcode", + "--purl", + "pkg:pypi/fetchcode@0.3.0", + "--purl", + "pkg:pypi/dejacode", + "--purl", + "pkg:pypi/dejacode@5.0.0", + "--purl", + "pkg:pypi/dejacode@5.0.0?os=windows", + "--purl", + "pkg:cargo/banquo", + "--purl", + "pkg:cargo/socksprox", + "--purl", + "pkg:gem/bundler-sass", + "--purl", + "pkg:rubygems/bundler-sass", + "--purl", + "pkg:nuget/auth0-aspnet@1.1.0", + "--head", + "--output", + actual_result_file, + ] + runner = CliRunner() + result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) assert result.exit_code == 0 with open(actual_result_file) as f_output: output_data = json.load(f_output) cli_test_utils.streamline_headers(output_data["headers"]) - streamline_metadata_packages(output_data["packages"]) with open(expected_result_file) as f_expected: expected_data = json.load(f_expected) cli_test_utils.streamline_headers(expected_data["headers"]) - streamline_metadata_packages(expected_data["packages"]) result_objects = [ ( output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -181,19 +492,18 @@ def test_metadata_cli_unique(self): expected_data["headers"][0]["options"]["--file"], ), ( - output_data["headers"][0]["options"]["--unique"], - expected_data["headers"][0]["options"]["--unique"], + output_data["headers"][0]["options"]["--head"], + expected_data["headers"][0]["options"]["--head"], ), + (output_data["packages"], expected_data["packages"]), ] for output, expected in result_objects: assert output == expected - compare_packages(expected_data, output_data, "metadata") - - def test_metadata_cli_duplicate_input_sources(self): + def test_urls_cli_duplicate_input_sources(self): """ - Test the `metadata` command with both `--purl` and `--file` inputs. + Test the `urls` command with both `--purl` and `--file` inputs. """ options = [ "--purl", @@ -204,1448 +514,134 @@ def test_metadata_cli_duplicate_input_sources(self): "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) assert "Use either purls or file but not both." in result.output assert result.exit_code == 2 - def test_metadata_cli_no_input_sources(self): + def test_urls_cli_no_input_sources(self): """ - Test the `metadata` command with neither `--purl` nor `--file` inputs. + Test the `urls` command with neither `--purl` nor `--file` inputs. """ options = [ "--output", "-", ] runner = CliRunner() - result = runner.invoke(purlcli.get_metadata, options, catch_exceptions=False) + result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) assert "Use either purls or file." in result.output assert result.exit_code == 2 - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], + def test_urls_details(self): + input_purls = [ + "pkg:pypi/fetchcode@0.3.0", + "pkg:gem/bundler@2.3.23", + "pkg:github/istio/istio@1.20.2", + ] + output = "" + file = "" + command_name = "urls" + head = False + + purl_urls_data = purlcli.get_urls_details( + input_purls, + output, + file, + head, + command_name, + ) + + expected_data = { + "headers": [ { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "metadata", - "--purl": ["pkg:pypi/fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/fetchcode"], - "errors": [], - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "metadata": [ - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ("download_url", None), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.1.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.1.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ( - "download_url", - "https://files.pythonhosted.org/packages/19/a0/c90e5ba4d71ea1a1a89784f6d839ffb0dbf32d270cba04d5602188cb3713/fetchcode-0.1.0-py3-none-any.whl", - ), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.2.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.2.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ( - "download_url", - "https://files.pythonhosted.org/packages/d7/e9/96e9302e84e326b3c10a40c1723f21f4db96b557a17c6871e7a4c6336906/fetchcode-0.2.0-py3-none-any.whl", - ), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - OrderedDict( - [ - ("purl", "pkg:pypi/fetchcode@0.3.0"), - ("type", "pypi"), - ("namespace", None), - ("name", "fetchcode"), - ("version", "0.3.0"), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "https://github.com/nexB/fetchcode", - ), - ( - "download_url", - "https://files.pythonhosted.org/packages/8d/fb/e45da0abf63504c3f88ad02537dc9dc64ea5206b09ce29cfb8191420d678/fetchcode-0.3.0-py3-none-any.whl", - ), - ( - "api_url", - "https://pypi.org/pypi/fetchcode/json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", "Apache-2.0"), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ), - ], - } + "tool_name": "purlcli", + "tool_version": "0.2.0", + "options": { + "command": "urls", + "--purl": [ + "pkg:pypi/fetchcode@0.3.0", + "pkg:gem/bundler@2.3.23", + "pkg:github/istio/istio@1.20.2", + ], + "--file": None, + "--output": "", + }, + "errors": [], + "warnings": [], + } + ], + "packages": [ + { + "purl": "pkg:pypi/fetchcode@0.3.0", + "download_url": None, + "inferred_urls": [ + "https://pypi.org/project/fetchcode/0.3.0/", ], + "repository_download_url": None, + "repository_homepage_url": "https://pypi.org/project/fetchcode/0.3.0/", }, - ), - ( - ["pkg:gem/bundler-sass"], { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:gem/bundler-sass"], - "command": "metadata", - }, - "purls": ["pkg:gem/bundler-sass"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:gem/bundler-sass' not supported with `metadata` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:rubygems/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "options": { - "command": "metadata", - "--purl": ["pkg:rubygems/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "errors": [], - "warnings": [ - "'check_existence' is not supported for " - "'pkg:rubygems/bundler-sass'", - ], - } - ], - "packages": [ - { - "purl": "pkg:rubygems/bundler-sass", - "metadata": [ - OrderedDict( - [ - ("purl", "pkg:rubygems/bundler-sass"), - ("type", "rubygems"), - ("namespace", None), - ("name", "bundler-sass"), - ("version", None), - ("qualifiers", OrderedDict()), - ("subpath", None), - ("primary_language", None), - ("description", None), - ("release_date", None), - ("parties", []), - ("keywords", []), - ( - "homepage_url", - "http://github.com/vogelbek/bundler-sass", - ), - ( - "download_url", - "https://rubygems.org/gems/bundler-sass-0.1.2.gem", - ), - ( - "api_url", - "https://rubygems.org/api/v1/gems/bundler-sass.json", - ), - ("size", None), - ("sha1", None), - ("md5", None), - ("sha256", None), - ("sha512", None), - ("bug_tracking_url", None), - ("code_view_url", None), - ("vcs_url", None), - ("copyright", None), - ("license_expression", None), - ("declared_license", ["MIT"]), - ("notice_text", None), - ("root_path", None), - ("dependencies", []), - ("contains_source_code", None), - ("source_packages", []), - ("repository_homepage_url", None), - ("repository_download_url", None), - ("api_data_url", None), - ] - ) - ], - } - ], - }, - ), - ( - ["pkg:nginx/nginx"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:nginx/nginx"], - "command": "metadata", - }, - "purls": ["pkg:nginx/nginx"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:nginx/nginx' not supported with `metadata` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/zzzzz"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/zzzzz"], - "command": "metadata", - }, - "purls": ["pkg:pypi/zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:pypi/zzzzz' does not exist in the upstream repo", - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/?fetchcode"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/?fetchcode"], - "command": "metadata", - }, - "purls": ["pkg:pypi/?fetchcode"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } + "purl": "pkg:gem/bundler@2.3.23", + "download_url": "https://rubygems.org/downloads/bundler-2.3.23.gem", + "inferred_urls": [ + "https://rubygems.org/gems/bundler/versions/2.3.23", + "https://rubygems.org/downloads/bundler-2.3.23.gem", ], - "packages": [], + "repository_download_url": None, + "repository_homepage_url": "https://rubygems.org/gems/bundler/versions/2.3.23", }, - ), - ( - ["zzzzz"], { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["zzzzz"], - "command": "metadata", - }, - "purls": ["zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'zzzzz' not valid"], - } + "purl": "pkg:github/istio/istio@1.20.2", + "download_url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz", + "inferred_urls": [ + "https://github.com/istio/istio/tree/1.20.2", + "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz", ], - "packages": [], + "repository_download_url": "https://github.com/istio/istio/archive/refs/tags/1.20.2.tar.gz", + "repository_homepage_url": "https://github.com/istio/istio/tree/1.20.2", }, - ), - ], - ) - def test_metadata_details(self, test_input, expected): - """ - Test the `metadata` nested function, `get_metadata_details()`. - """ - purl_metadata = purlcli.get_metadata_details( - test_input, - output="", - file="", - command_name="metadata", - unique=False, - ) - cli_test_utils.streamline_headers(purl_metadata["headers"]) - streamline_metadata_packages(purl_metadata["packages"]) - - cli_test_utils.streamline_headers(expected["headers"]) - streamline_metadata_packages(expected["packages"]) - - assert purl_metadata["headers"] == expected["headers"] - - compare_packages(expected, purl_metadata, "metadata") - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], - None, - ), - ( - ["pkg:gem/bundler-sass"], - "valid_but_not_supported", - ), - ( - ["pkg:rubygems/bundler-sass"], - "check_existence_not_supported", - ), - ( - ["pkg:nginx/nginx"], - "valid_but_not_supported", - ), - ( - ["pkg:pypi/zzzzz"], - "not_in_upstream_repo", - ), - ( - ["pkg:pypi/?fetchcode"], - "not_valid", - ), - ( - ["zzzzz"], - "not_valid", - ), - ], - ) - def test_check_metadata_purl(self, test_input, expected): - purl_metadata = purlcli.check_metadata_purl(test_input[0]) - assert purl_metadata == expected - - @pytest.mark.parametrize( - "test_input,expected_input_purls,expected_normalized_purls", - [ - ( - [["pkg:pypi/fetchcode"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode@1.2.3", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode@1.2.3?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ( - [ - ( - "pkg:pypi/fetchcode@1.2.3?howistheweather=rainy", - "pkg:pypi/fetchcode", - ) - ] - ), - ), - ( - [["pkg:pypi/fetchcode?howistheweather=rainy"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode?howistheweather=rainy", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/fetchcode#this/is/a/path"]], - (["pkg:pypi/fetchcode"]), - ([("pkg:pypi/fetchcode#this/is/a/path", "pkg:pypi/fetchcode")]), - ), - ( - [["pkg:pypi/?fetchcode"]], - (["pkg:pypi/"]), - ([("pkg:pypi/?fetchcode", "pkg:pypi/")]), - ), - ( - [ - [ - "pkg:pypi/fetchcode@0.3.0", - "pkg:pypi/fetchcode@5.0.0", - "pkg:pypi/dejacode", - "pkg:pypi/dejacode@5.0.0", - "pkg:pypi/dejacode@5.0.0?os=windows", - "pkg:pypi/dejacode@5.0.0os=windows", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "pkg:pypi/dejacode@10.0.0", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - "pkg:nginx/nginx@0.8.9?os=windows", - ] - ], - ( - [ - "pkg:pypi/fetchcode", - "pkg:pypi/dejacode", - "pkg:cargo/banquo", - "pkg:cargo/socksprox", - "pkg:nginx/nginx", - ] - ), - ( - [ - ("pkg:pypi/fetchcode@0.3.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@5.0.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/dejacode", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0?os=windows", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@5.0.0os=windows", "pkg:pypi/dejacode"), - ( - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "pkg:pypi/dejacode", - ), - ("pkg:pypi/dejacode@5.0.0#how/are/you", "pkg:pypi/dejacode"), - ("pkg:pypi/dejacode@10.0.0", "pkg:pypi/dejacode"), - ("pkg:cargo/banquo", "pkg:cargo/banquo"), - ("pkg:cargo/socksprox", "pkg:cargo/socksprox"), - ("pkg:nginx/nginx", "pkg:nginx/nginx"), - ("pkg:nginx/nginx@0.8.9?os=windows", "pkg:nginx/nginx"), - ] - ), - ), - ], - ) - def test_normalize_purls( - self, test_input, expected_input_purls, expected_normalized_purls - ): - unique = True - input_purls, normalized_purls = purlcli.normalize_purls(test_input[0], unique) - - assert input_purls == expected_input_purls - assert normalized_purls == expected_normalized_purls - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - "command": "metadata", - }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:gem/bundler-sass' not supported with `metadata` command" - ], - } - ], - ), - ], - ) - def test_construct_headers(self, test_input, expected): - metadata_headers = purlcli.construct_headers( - test_input, - output="", - file="", - command_name="metadata", - head=None, - normalized_purls=None, - unique=None, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, - ) - cli_test_utils.streamline_headers(expected) - cli_test_utils.streamline_headers(metadata_headers) - - assert metadata_headers == expected - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - "--unique": True, - "command": "metadata", - }, - "purls": [ - "pkg:gem/bundler-sass", - "pkg:pypi/fetchcode", - "pkg:pypi/fetchcode@0.1.0", - "pkg:pypi/fetchcode@0.2.0", - ], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "input PURL: 'pkg:pypi/fetchcode@0.1.0' normalized to 'pkg:pypi/fetchcode'", - "input PURL: 'pkg:pypi/fetchcode@0.2.0' normalized to 'pkg:pypi/fetchcode'", - "'pkg:gem/bundler-sass' not supported with `metadata` command", - ], - } - ], - ), - ], - ) - def test_construct_headers_unique(self, test_input, expected): - metadata_headers = purlcli.construct_headers( - test_input, - output="", - file="", - command_name="metadata", - head=None, - normalized_purls=[ - ("pkg:gem/bundler-sass", "pkg:gem/bundler-sass"), - ("pkg:pypi/fetchcode", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.1.0", "pkg:pypi/fetchcode"), - ("pkg:pypi/fetchcode@0.2.0", "pkg:pypi/fetchcode"), ], - unique=True, - purl_warnings={"pkg:gem/bundler-sass": "valid_but_not_supported"}, - ) - cli_test_utils.streamline_headers(expected) - cli_test_utils.streamline_headers(metadata_headers) - - assert metadata_headers == expected - + } -class TestPURLCLI_urls(object): - def test_urls_cli(self): - """ - Test the `urls` command with actual and expected JSON output files. - """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_urls_output.json" - ) - actual_result_file = test_env.get_temp_file("actual_urls_output.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:pypi/dejacode", - "--purl", - "pkg:pypi/dejacode@5.0.0", - "--purl", - "pkg:pypi/dejacode@5.0.0?os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "--purl", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "--purl", - "pkg:pypi/dejacode@10.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:cargo/socksprox", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:nginx/nginx@0.8.9?os=windows", - "--purl", - "pkg:gem/bundler-sass", - "--purl", - "pkg:rubygems/bundler-sass", - "--purl", - "pkg:pypi/matchcode", - "--purl", - "abcdefg", - "--purl", - "pkg/abc", - "--purl", - "pkg:nuget/auth0-aspnet@1.1.0", - "--output", - actual_result_file, - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), - (output_data["packages"], expected_data["packages"]), - ] - - for output, expected in result_objects: - assert output == expected - - def test_urls_cli_unique(self): - """ - Test the `urls` command with actual and expected JSON output files. - """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_urls_output_unique.json" - ) - actual_result_file = test_env.get_temp_file("actual_urls_output_unique.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:pypi/dejacode", - "--purl", - "pkg:pypi/dejacode@5.0.0", - "--purl", - "pkg:pypi/dejacode@5.0.0?os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "--purl", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "--purl", - "pkg:pypi/dejacode@10.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:cargo/socksprox", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:nginx/nginx@0.8.9?os=windows", - "--purl", - "pkg:gem/bundler-sass", - "--purl", - "pkg:rubygems/bundler-sass", - "--purl", - "pkg:pypi/matchcode", - "--purl", - "abcdefg", - "--purl", - "pkg/abc", - "--purl", - "pkg:nuget/auth0-aspnet@1.1.0", - "--output", - actual_result_file, - "--unique", - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), - (output_data["packages"], expected_data["packages"]), - ] - - for output, expected in result_objects: - assert output == expected - - def test_urls_cli_head(self): - """ - Test the `urls` command with actual and expected JSON output files. - """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_urls_output_head.json" - ) - actual_result_file = test_env.get_temp_file("actual_urls_output_head.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:pypi/dejacode", - "--purl", - "pkg:pypi/dejacode@5.0.0", - "--purl", - "pkg:pypi/dejacode@5.0.0?os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0os=windows", - "--purl", - "pkg:pypi/dejacode@5.0.0?how_is_the_weather=rainy", - "--purl", - "pkg:pypi/dejacode@5.0.0#how/are/you", - "--purl", - "pkg:pypi/dejacode@10.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:cargo/socksprox", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:nginx/nginx@0.8.9?os=windows", - "--purl", - "pkg:gem/bundler-sass", - "--purl", - "pkg:rubygems/bundler-sass", - "--purl", - "pkg:pypi/matchcode", - "--purl", - "abcdefg", - "--purl", - "pkg/abc", - "--purl", - "pkg:nuget/auth0-aspnet@1.1.0", - "--head", - "--output", - actual_result_file, - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), - ( - output_data["headers"][0]["options"]["--head"], - expected_data["headers"][0]["options"]["--head"], - ), - (output_data["packages"], expected_data["packages"]), - ] - - for output, expected in result_objects: - assert output == expected - - def test_urls_cli_duplicate_input_sources(self): - """ - Test the `urls` command with both `--purl` and `--file` inputs. - """ - options = [ - "--purl", - "pkg:pypi/minecode", - "--file", - test_env.get_test_loc("purlcli/metadata_input_purls.txt"), - "--output", - "-", - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) - assert "Use either purls or file but not both." in result.output - assert result.exit_code == 2 - - def test_urls_cli_no_input_sources(self): - """ - Test the `urls` command with neither `--purl` nor `--file` inputs. - """ - options = [ - "--output", - "-", - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_urls, options, catch_exceptions=False) - assert "Use either purls or file." in result.output - assert result.exit_code == 2 - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "urls", - "--purl": ["pkg:pypi/fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/fetchcode"], - "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode' not fully supported with `urls` command" - ], - } - ], - "packages": [ - { - "purl": "pkg:pypi/fetchcode", - "download_url": { - "url": None, - }, - "inferred_urls": [ - { - "url": "https://pypi.org/project/fetchcode/", - } - ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://pypi.org/project/fetchcode/", - }, - "url": { - "url": "https://pypi.org/project/fetchcode/", - }, - }, - ], - }, - ), - ( - ["pkg:pypi/fetchcode@10.0.0"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "urls", - "--purl": ["pkg:pypi/fetchcode@10.0.0"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/fetchcode@10.0.0"], - "errors": [], - "warnings": [ - "'pkg:pypi/fetchcode@10.0.0' does not exist in the upstream repo", - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:gem/bundler-sass"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:gem/bundler-sass"], - "command": "urls", - }, - "purls": ["pkg:gem/bundler-sass"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:gem/bundler-sass", - "download_url": { - "url": None, - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass", - } - ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - }, - ], - }, - ), - ( - ["pkg:rubygems/bundler-sass"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:rubygems/bundler-sass"], - "command": "urls", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:rubygems/bundler-sass", - "download_url": { - "url": None, - }, - "inferred_urls": [ - { - "url": "https://rubygems.org/gems/bundler-sass", - } - ], - "repo_download_url": { - "url": None, - }, - "repo_download_url_by_package_type": { - "url": None, - }, - "repo_url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - "url": { - "url": "https://rubygems.org/gems/bundler-sass", - }, - }, - ], - }, - ), - ( - ["pkg:nginx/nginx"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:nginx/nginx"], - "command": "urls", - }, - "purls": ["pkg:nginx/nginx"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:nginx/nginx' not supported with `urls` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/zzzzz"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/zzzzz"], - "command": "urls", - }, - "purls": ["pkg:pypi/zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": [ - "'pkg:pypi/zzzzz' does not exist in the upstream repo", - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/?fetchcode"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["pkg:pypi/?fetchcode"], - "command": "urls", - }, - "purls": ["pkg:pypi/?fetchcode"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } - ], - "packages": [], - }, - ), - ( - ["zzzzz"], - { - "headers": [ - { - "errors": [], - "options": { - "--file": None, - "--output": "", - "--purl": ["zzzzz"], - "command": "urls", - }, - "purls": ["zzzzz"], - "tool_name": "purlcli", - "tool_version": "0.2.0", - "warnings": ["'zzzzz' not valid"], - } - ], - "packages": [], - }, - ), - ], - ) - def test_urls_details(self, test_input, expected): - """ - Test the `urls` nested function, `get_urls_details()`. - """ - purl_urls = purlcli.get_urls_details( - test_input, - output="", - file="", - command_name="urls", - head=False, - unique=False, - ) - cli_test_utils.streamline_headers(expected["headers"]) - cli_test_utils.streamline_headers(purl_urls["headers"]) - - assert purl_urls == expected - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], - "valid_but_not_fully_supported", - ), - ( - ["pkg:gem/bundler-sass"], - None, - ), - ( - ["pkg:rubygems/bundler-sass"], - None, - ), - ( - ["pkg:nginx/nginx"], - "valid_but_not_supported", - ), - ( - ["pkg:pypi/zzzzz"], - "not_in_upstream_repo", - ), - ( - ["pkg:pypi/?fetchcode"], - "not_valid", - ), - ( - ["zzzzz"], - "not_valid", - ), - ], - ) - def test_check_urls_purl(self, test_input, expected): - purl_urls = purlcli.check_urls_purl(test_input[0]) - assert purl_urls == expected - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["https://pypi.org/project/fetchcode/"], - {"get_request": 200, "head_request": 200}, - ), - ( - [None], - {"get_request": "N/A", "head_request": "N/A"}, - ), - ( - ["https://crates.io/crates/banquo"], - {"get_request": 404, "head_request": 404}, - ), - ( - ["https://crates.io/crates/socksprox"], - {"get_request": 404, "head_request": 404}, - ), - ( - ["https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0"], - {"get_request": 200, "head_request": 404}, - ), - ], - ) - def test_make_head_request(self, test_input, expected): - purl_status_code = purlcli.make_head_request(test_input[0]) - - assert purl_status_code == expected - - -class TestPURLCLI_validate(object): - def test_validate_cli(self): - """ - Test the `validate` command with actual and expected JSON output files. - """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_validate_output.json" - ) - actual_result_file = test_env.get_temp_file("actual_validate_output.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "--purl", - "pkg:pypi/fetchcode@0.3.0os=windows", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:gem/rails", - "--purl", - "pkg:rubygems/rails", - "--output", - actual_result_file, - ] - runner = CliRunner() - result = runner.invoke(purlcli.validate, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) + assert purl_urls_data == expected_data - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), + @pytest.mark.parametrize( + "test_input,expected", + [ ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], + ["https://pypi.org/project/fetchcode/"], + {"get_request": 200, "head_request": 200}, ), ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], + [None], + {"get_request": "N/A", "head_request": "N/A"}, ), ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], + ["https://crates.io/crates/banquo"], + {"get_request": 404, "head_request": 404}, ), ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], + ["https://crates.io/crates/socksprox"], + {"get_request": 404, "head_request": 404}, ), ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], + ["https://www.nuget.org/api/v2/package/auth0-aspnet/1.1.0"], + {"get_request": 200, "head_request": 404}, ), - (output_data["packages"], expected_data["packages"]), - ] + ], + ) + def test_make_head_request(self, test_input, expected): + purl_status_code = purlcli.make_head_request(test_input[0]) + + assert purl_status_code == expected - for output, expected in result_objects: - assert output == expected - def test_validate_cli_unique(self): +class TestPURLCLI_validate(object): + def test_validate_cli(self): """ - Test the `validate` command with actual and expected JSON output files - with the `--unique` flag included in the command. + Test the `validate` command with actual and expected JSON output files. """ expected_result_file = test_env.get_test_loc( - "purlcli/expected_validate_output_unique.json" + "purlcli/expected_validate_output.json" ) actual_result_file = test_env.get_temp_file("actual_validate_output.json") options = [ @@ -1669,7 +665,6 @@ def test_validate_cli_unique(self): "pkg:rubygems/rails", "--output", actual_result_file, - "--unique", ] runner = CliRunner() result = runner.invoke(purlcli.validate, options, catch_exceptions=False) @@ -1686,7 +681,6 @@ def test_validate_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -1707,10 +701,6 @@ def test_validate_cli_unique(self): output_data["headers"][0]["options"]["--file"], expected_data["headers"][0]["options"]["--file"], ), - ( - output_data["headers"][0]["options"]["--unique"], - expected_data["headers"][0]["options"]["--unique"], - ), (output_data["packages"], expected_data["packages"]), ] @@ -1891,86 +881,9 @@ def test_versions_cli(self): "--purl", "pkg:cargo/banquo", "--purl", - "pkg:nginx/nginx", - "--purl", - "pkg:hex/coherence@0.1.0", - "--output", - actual_result_file, - ] - runner = CliRunner() - result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) - assert result.exit_code == 0 - - with open(actual_result_file) as f_output: - output_data = json.load(f_output) - cli_test_utils.streamline_headers(output_data["headers"]) - - with open(expected_result_file) as f_expected: - expected_data = json.load(f_expected) - cli_test_utils.streamline_headers(expected_data["headers"]) - - result_objects = [ - ( - output_data["headers"][0]["tool_name"], - expected_data["headers"][0]["tool_name"], - ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), - ( - output_data["headers"][0]["warnings"], - expected_data["headers"][0]["warnings"], - ), - ( - output_data["headers"][0]["errors"], - expected_data["headers"][0]["errors"], - ), - ( - output_data["headers"][0]["options"]["command"], - expected_data["headers"][0]["options"]["command"], - ), - ( - output_data["headers"][0]["options"]["--purl"], - expected_data["headers"][0]["options"]["--purl"], - ), - ( - output_data["headers"][0]["options"]["--file"], - expected_data["headers"][0]["options"]["--file"], - ), - ] - - for output, expected in result_objects: - assert output == expected - - compare_packages(expected_data, output_data, "versions") - - def test_versions_cli_unique(self): - """ - Test the `versions` command with actual and expected JSON output files - with the `--unique` flag included in the command. - """ - expected_result_file = test_env.get_test_loc( - "purlcli/expected_versions_output_unique.json" - ) - actual_result_file = test_env.get_temp_file("actual_versions_output.json") - options = [ - "--purl", - "pkg:pypi/fetchcode", - "--purl", - "pkg:pypi/fetchcode@0.3.0", - "--purl", - "pkg:pypi/fetchcode@0.3.0?os=windows", - "--purl", - "pkg:pypi/fetchcode@0.3.0os=windows", - "--purl", - "pkg:pypi/fetchcode@5.0.0", - "--purl", - "pkg:cargo/banquo", - "--purl", - "pkg:nginx/nginx", - "--purl", "pkg:hex/coherence@0.1.0", "--output", actual_result_file, - "--unique", ] runner = CliRunner() result = runner.invoke(purlcli.get_versions, options, catch_exceptions=False) @@ -1989,7 +902,6 @@ def test_versions_cli_unique(self): output_data["headers"][0]["tool_name"], expected_data["headers"][0]["tool_name"], ), - (output_data["headers"][0]["purls"], expected_data["headers"][0]["purls"]), ( output_data["headers"][0]["warnings"], expected_data["headers"][0]["warnings"], @@ -2010,22 +922,22 @@ def test_versions_cli_unique(self): output_data["headers"][0]["options"]["--file"], expected_data["headers"][0]["options"]["--file"], ), - ( - output_data["headers"][0]["options"]["--unique"], - expected_data["headers"][0]["options"]["--unique"], - ), ] for output, expected in result_objects: assert output == expected - compare_packages(expected_data, output_data, "versions") + compare_packages(expected_data, output_data) @pytest.mark.parametrize( "test_input,expected", [ ( - ["pkg:pypi/fetchcode"], + [ + "pkg:pypi/fetchcode", + "pkg:gem/bundler-sass", + "pkg:pypi/zzzzz", + ], { "headers": [ { @@ -2033,11 +945,14 @@ def test_versions_cli_unique(self): "tool_version": "0.2.0", "options": { "command": "versions", - "--purl": ["pkg:pypi/fetchcode"], + "--purl": [ + "pkg:pypi/fetchcode", + "pkg:gem/bundler-sass", + "pkg:pypi/zzzzz", + ], "--file": None, "--output": "", }, - "purls": ["pkg:pypi/fetchcode"], "errors": [], "warnings": [], } @@ -2045,173 +960,30 @@ def test_versions_cli_unique(self): "packages": [ { "purl": "pkg:pypi/fetchcode", - "versions": [ - { - "purl": "pkg:pypi/fetchcode@0.1.0", - "version": "0.1.0", - "release_date": "2021-08-25 15:15:15.265015+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.2.0", - "version": "0.2.0", - "release_date": "2022-09-14 16:36:02.242182+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.3.0", - "version": "0.3.0", - "release_date": "2023-12-18 20:49:45.840364+00:00", - }, - { - "purl": "pkg:pypi/fetchcode@0.4.0", - "release_date": "2024-03-12 07:01:29.239299+00:00", - "version": "0.4.0", - }, - ], - } - ], - }, - ), - ( - ["pkg:gem/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:gem/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:gem/bundler-sass"], - "errors": [], - "warnings": [], - } - ], - "packages": [ - { - "purl": "pkg:gem/bundler-sass", - "versions": [ - { - "purl": "pkg:gem/bundler-sass@0.1.2", - "version": "0.1.2", - "release_date": "2013-12-11 00:27:10.097000+00:00", - } - ], - } - ], - }, - ), - ( - ["pkg:rubygems/bundler-sass"], - { - "headers": [ - { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:rubygems/bundler-sass"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:rubygems/bundler-sass"], - "errors": [], - "warnings": [ - "'pkg:rubygems/bundler-sass' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:nginx/nginx"], - { - "headers": [ + "version": "0.1.0", + "release_date": "2021-08-25", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:nginx/nginx"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:nginx/nginx"], - "errors": [], - "warnings": [ - "'pkg:nginx/nginx' not supported with `versions` command" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/zzzzz"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode", + "version": "0.2.0", + "release_date": "2022-09-14", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:pypi/zzzzz"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/zzzzz"], - "errors": [], - "warnings": [ - "'pkg:pypi/zzzzz' does not exist in the upstream repo" - ], - } - ], - "packages": [], - }, - ), - ( - ["pkg:pypi/?fetchcode"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode", + "version": "0.3.0", + "release_date": "2023-12-18", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["pkg:pypi/?fetchcode"], - "--file": None, - "--output": "", - }, - "purls": ["pkg:pypi/?fetchcode"], - "errors": [], - "warnings": ["'pkg:pypi/?fetchcode' not valid"], - } - ], - "packages": [], - }, - ), - ( - ["zzzzz"], - { - "headers": [ + "purl": "pkg:pypi/fetchcode", + "release_date": "2024-03-12", + "version": "0.4.0", + }, { - "tool_name": "purlcli", - "tool_version": "0.2.0", - "options": { - "command": "versions", - "--purl": ["zzzzz"], - "--file": None, - "--output": "", - }, - "purls": ["zzzzz"], - "errors": [], - "warnings": ["'zzzzz' not valid"], - } + "purl": "pkg:gem/bundler-sass", + "version": "0.1.2", + "release_date": "2013-12-11", + }, ], - "packages": [], }, ), ], @@ -2220,10 +992,12 @@ def test_versions_details(self, test_input, expected): output = "" file = "" command_name = "versions" - unique = False purl_versions = purlcli.get_versions_details( - test_input, output, file, unique, command_name + test_input, + output, + file, + command_name, ) cli_test_utils.streamline_headers(purl_versions["headers"]) @@ -2231,48 +1005,7 @@ def test_versions_details(self, test_input, expected): assert purl_versions["headers"] == expected["headers"] - compare_packages(expected, purl_versions, "versions") - - @pytest.mark.parametrize( - "test_input,expected", - [ - ( - ["pkg:pypi/fetchcode"], - None, - ), - ( - ["pkg:gem/bundler-sass"], - None, - ), - ( - ["pkg:rubygems/bundler-sass"], - "valid_but_not_supported", - ), - ( - ["pkg:nginx/nginx"], - "valid_but_not_supported", - ), - ( - ["pkg:pypi/zzzzz"], - "not_in_upstream_repo", - ), - ( - ["pkg:pypi/?fetchcode"], - "not_valid", - ), - ( - ["zzzzz"], - "not_valid", - ), - ( - ["pkg:maven/axis/axis@1.0"], - None, - ), - ], - ) - def test_check_versions_purl(self, test_input, expected): - purl_versions = purlcli.check_versions_purl(test_input[0]) - assert purl_versions == expected + compare_packages(expected, purl_versions) def streamline_metadata_packages(packages): @@ -2284,7 +1017,7 @@ def streamline_metadata_packages(packages): hle.pop("download_url", None) -def compare_packages(expected_data, actual_data, nested_key): +def compare_packages(expected_data, actual_data): """ Compare the expected and actual data nested inside the `packages` field returned from a live-fetch query and assert that expected data from prior @@ -2301,7 +1034,4 @@ def compare_packages(expected_data, actual_data, nested_key): actual.append(actual_pkg) for expected_entry in expected: - for actual_entry in actual: - if expected_entry["purl"] == actual_entry["purl"]: - for expected_sibling in expected_entry[nested_key]: - assert expected_sibling in actual_entry[nested_key] + assert expected_entry in actual