General Deprecation Warning Improvements (#11466)

Author: QMalcolm

Diff for: .changes/unreleased/Features-20250403-174659.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: Show summaries for deprecations and add ability to toggle seeing all deprecation
+  violation instances
+time: 2025-04-03T17:46:59.684525-05:00
+custom:
+  Author: QMalcolm
+  Issue: "11429"

Diff for: core/dbt/cli/main.py

@@ -129,6 +129,7 @@ def global_flags(func):
     @p.record_timing_info
     @p.send_anonymous_usage_stats
     @p.single_threaded
+    @p.show_all_deprecations
     @p.state
     @p.static_parser
     @p.target

Diff for: core/dbt/cli/params.py

@@ -595,6 +595,14 @@
     hidden=True,
 )
 
+show_all_deprecations = click.option(
+    "--show-all-deprecations/--no-show-all-deprecations",
+    envvar=None,
+    help="By default, each type of a deprecation warning is only shown once. Use this flag to show all deprecation warning instances.",
+    is_flag=True,
+    default=False,
+)
+
 skip_profile_setup = click.option(
     "--skip-profile-setup",
     "-s",

Diff for: core/dbt/cli/requires.py

@@ -16,6 +16,7 @@
 from dbt.config.runtime import UnsetProfile, load_profile, load_project
 from dbt.context.providers import generate_runtime_macro_context
 from dbt.context.query_header import generate_query_header_context
+from dbt.deprecations import show_all_deprecation_summaries
 from dbt.events.logging import setup_event_logger
 from dbt.events.types import (
     ArtifactUploadError,
@@ -176,6 +177,8 @@ def wrapper(*args, **kwargs):
             except Exception as e:
                 fire_event(ArtifactUploadError(msg=str(e)))
 
+            show_all_deprecation_summaries()
+
             if importlib.util.find_spec("resource") is not None:
                 import resource
 

Diff for: core/dbt/clients/registry.py

@@ -1,6 +1,6 @@
 import functools
 import os
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 import requests
 
@@ -34,9 +34,26 @@ def _get_url(name, registry_base_url=None):
     return "{}{}".format(registry_base_url, url)
 
 
-def _get_with_retries(package_name, registry_base_url=None):
+def _check_package_redirect(package_name: str, response: Dict[str, Any]) -> None:
+    # Either redirectnamespace or redirectname in the JSON response indicate a redirect
+    # redirectnamespace redirects based on package ownership
+    # redirectname redirects based on package name
+    # Both can be present at the same time, or neither. Fails gracefully to old name
+    if ("redirectnamespace" in response) or ("redirectname" in response):
+        use_namespace = response.get("redirectnamespace") or response["namespace"]
+        use_name = response.get("redirectname") or response["name"]
+
+        new_nwo = f"{use_namespace}/{use_name}"
+        deprecations.warn("package-redirect", old_name=package_name, new_name=new_nwo)
+
+
+def _get_package_with_retries(
+    package_name: str, registry_base_url: Optional[str] = None
+) -> Dict[str, Any]:
     get_fn = functools.partial(_get, package_name, registry_base_url)
-    return connection_exception_retry(get_fn, 5)
+    response: Dict[str, Any] = connection_exception_retry(get_fn, 5)
+    _check_package_redirect(package_name, response)
+    return response
 
 
 def _get(package_name, registry_base_url=None):
@@ -48,7 +65,7 @@ def _get(package_name, registry_base_url=None):
     resp.raise_for_status()
 
     # The response should always be a dictionary.  Anything else is unexpected, raise error.
-    # Raising this error will cause this function to retry (if called within _get_with_retries)
+    # Raising this error will cause this function to retry (if called within _get_package_with_retries)
     # and hopefully get a valid response.  This seems to happen when there's an issue with the Hub.
     # Since we control what we expect the HUB to return, this is safe.
     # See https://github.com/dbt-labs/dbt-core/issues/4577
@@ -96,29 +113,12 @@ def _get(package_name, registry_base_url=None):
     return response
 
 
-_get_cached = memoized(_get_with_retries)
+_get_cached = memoized(_get_package_with_retries)
 
 
 def package(package_name, registry_base_url=None) -> Dict[str, Any]:
     # returns a dictionary of metadata for all versions of a package
     response = _get_cached(package_name, registry_base_url)
-    # Either redirectnamespace or redirectname in the JSON response indicate a redirect
-    # redirectnamespace redirects based on package ownership
-    # redirectname redirects based on package name
-    # Both can be present at the same time, or neither. Fails gracefully to old name
-    if ("redirectnamespace" in response) or ("redirectname" in response):
-        if ("redirectnamespace" in response) and response["redirectnamespace"] is not None:
-            use_namespace = response["redirectnamespace"]
-        else:
-            use_namespace = response["namespace"]
-
-        if ("redirectname" in response) and response["redirectname"] is not None:
-            use_name = response["redirectname"]
-        else:
-            use_name = response["name"]
-
-        new_nwo = use_namespace + "/" + use_name
-        deprecations.warn("package-redirect", old_name=package_name, new_name=new_nwo)
     return response["versions"]
 
 

Diff for: core/dbt/deprecations.py

@@ -1,14 +1,18 @@
 import abc
-from typing import Callable, ClassVar, Dict, List, Optional, Set
+from collections import defaultdict
+from typing import Callable, ClassVar, DefaultDict, Dict, List, Optional
 
 import dbt.tracking
 from dbt.events import types as core_types
+from dbt.flags import get_flags
+from dbt_common.events.base_types import BaseEvent
 from dbt_common.events.functions import warn_or_error
 
 
 class DBTDeprecation:
     _name: ClassVar[Optional[str]] = None
     _event: ClassVar[Optional[str]] = None
+    _summary_event: ClassVar[Optional[str]] = None
 
     @property
     def name(self) -> str:
@@ -33,17 +37,45 @@ def event(self) -> abc.ABCMeta:
                 raise NameError(msg)
         raise NotImplementedError("event not implemented for {}".format(self._event))
 
+    @property
+    def summary_event(self) -> Optional[abc.ABCMeta]:
+        if self._summary_event is None:
+            return None
+        else:
+            module_path = core_types
+            class_name = self._summary_event
+
+            try:
+                return getattr(module_path, class_name)
+            except AttributeError:
+                msg = f"Event Class `{class_name}` is not defined in `{module_path}`"
+                raise NameError(msg)
+
     def show(self, *args, **kwargs) -> None:
-        if self.name not in active_deprecations:
+        flags = get_flags()
+        if self.name not in active_deprecations or flags.show_all_deprecations:
             event = self.event(**kwargs)
             warn_or_error(event)
             self.track_deprecation_warn()
-            active_deprecations.add(self.name)
+
+        active_deprecations[self.name] += 1
+
+    def show_summary(self) -> None:
+        event_class = self.summary_event
+        if self.name in active_deprecations and event_class is not None:
+            show_all_hint = (
+                not get_flags().show_all_deprecations and active_deprecations[self.name] > 1
+            )
+            event: BaseEvent = event_class(
+                occurrences=active_deprecations[self.name], show_all_hint=show_all_hint
+            )
+            warn_or_error(event)
 
 
 class PackageRedirectDeprecation(DBTDeprecation):
     _name = "package-redirect"
     _event = "PackageRedirectDeprecation"
+    _summary_event = "PackageRedirectDeprecationSummary"
 
 
 class PackageInstallPathDeprecation(DBTDeprecation):
@@ -163,10 +195,15 @@ def show_callback():
     buffered_deprecations.append(show_callback)
 
 
+def show_all_deprecation_summaries() -> None:
+    for deprecation in active_deprecations:
+        deprecations[deprecation].show_summary()
+
+
 # these are globally available
 # since modules are only imported once, active_deprecations is a singleton
 
-active_deprecations: Set[str] = set()
+active_deprecations: DefaultDict[str, int] = defaultdict(int)
 
 deprecations_list: List[DBTDeprecation] = [
     PackageRedirectDeprecation(),

Diff for: core/dbt/events/core_types.proto

@@ -470,6 +470,17 @@ message MicrobatchMacroOutsideOfBatchesDeprecationMsg {
     MicrobatchMacroOutsideOfBatchesDeprecation data = 2;
 }
 
+// D021
+message PackageRedirectDeprecationSummary {
+    int32 occurrences = 1;
+    bool show_all_hint = 2;
+}
+
+message PackageRedirectDeprecationSummaryMsg {
+    CoreEventInfo info = 1;
+    PackageRedirectDeprecationSummary data = 2;
+}
+
 // I065
 message DeprecatedModel {
     string model_name = 1;

Diff for: core/dbt/events/core_types_pb2.py

@@ -256,6 +256,22 @@ def message(self) -> str:
         return line_wrap_message(warning_tag(f"Deprecated functionality\n\n{description}"))
 
 
+class PackageRedirectDeprecationSummary(WarnLevel):
+    def code(self) -> str:
+        return "D021"
+
+    def message(self) -> str:
+        description = (
+            f"Found {pluralize(self.occurrences, 'package')} that {'has' if self.occurrences == 1 else 'have'} been deprecated in favor of new packages. Please "
+            f"update your `packages.yml` configuration to use the new packages instead."
+        )
+
+        if self.show_all_hint:
+            description += " To see all deprecated packages, run command again with the `--show-all-deprecations` flag."
+
+        return line_wrap_message(warning_tag(f"Deprecated functionality\n\n{description}"))
+
+
 class PackageInstallPathDeprecation(WarnLevel):
     def code(self) -> str:
         return "D002"

Diff for: core/dbt/events/types.py

@@ -1,4 +1,18 @@
+import pytest
+
 from tests.functional.fixtures.happy_path_fixture import (  # noqa:D
     happy_path_project,
     happy_path_project_files,
 )
+
+
+@pytest.fixture(scope="function", autouse=True)
+def clear_memoized_get_package_with_retries():
+    # This fixture is used to clear the memoized cache for _get_package_with_retries
+    # in dbt.clients.registry. This is necessary because the cache is shared across
+    # tests and can cause unexpected behavior if not cleared as some tests depend on
+    # the deprecation warning that _get_package_with_retries fires
+    yield
+    from dbt.clients.registry import _get_cached
+
+    _get_cached.cache = {}

Diff for: tests/functional/conftest.py

@@ -1,3 +1,5 @@
+from collections import defaultdict
+
 import pytest
 
 from dbt import deprecations
@@ -30,14 +32,13 @@ def project_config_update(self, unique_schema):
 
     def test_project_tests_config(self, project):
         deprecations.reset_deprecations()
-        assert deprecations.active_deprecations == set()
+        assert deprecations.active_deprecations == defaultdict(int)
         run_dbt(["parse"])
-        expected = set()
-        assert expected == deprecations.active_deprecations
+        assert deprecations.active_deprecations == defaultdict(int)
 
     def test_project_tests_config_fail(self, project):
         deprecations.reset_deprecations()
-        assert deprecations.active_deprecations == set()
+        assert deprecations.active_deprecations == defaultdict(int)
         with pytest.raises(CompilationError) as exc:
             run_dbt(["--warn-error", "--no-partial-parse", "parse"])
         exc_str = " ".join(str(exc.value).split())  # flatten all whitespace
@@ -60,14 +61,13 @@ def macros(self):
 
     def test_generic_tests_config(self, project):
         deprecations.reset_deprecations()
-        assert deprecations.active_deprecations == set()
+        assert deprecations.active_deprecations == defaultdict(int)
         run_dbt(["parse"])
-        expected = set()
-        assert expected == deprecations.active_deprecations
+        assert deprecations.active_deprecations == defaultdict(int)
 
     def test_generic_tests_fail(self, project):
         deprecations.reset_deprecations()
-        assert deprecations.active_deprecations == set()
+        assert deprecations.active_deprecations == defaultdict(int)
         run_dbt(["--warn-error", "--no-partial-parse", "parse"])
 
     def test_generic_data_test_parsing(self, project):
@@ -92,10 +92,9 @@ def seeds(self):
 
     def test_source_tests_config(self, project):
         deprecations.reset_deprecations()
-        assert deprecations.active_deprecations == set()
+        assert deprecations.active_deprecations == defaultdict(int)
         run_dbt(["parse"])
-        expected = set()
-        assert expected == deprecations.active_deprecations
+        assert deprecations.active_deprecations == defaultdict(int)
 
     def test_generic_data_tests(self, project):
         run_dbt(["seed"])