diff --git a/bigquery_etl/cli/monitoring.py b/bigquery_etl/cli/monitoring.py index 8eb57b62cab..534d7a45f0a 100644 --- a/bigquery_etl/cli/monitoring.py +++ b/bigquery_etl/cli/monitoring.py @@ -368,6 +368,8 @@ def _update_bigconfig( for collection in bigconfig.tag_deployments: for deployment in collection.deployments: for metric in deployment.metrics: + print("im here") + print(deployment) if metric.metric_type is None: err_message = f"""There appears to be an issue parsing \ a metric type definition for `{project}.{dataset}.{table}` \ diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/blogs_goals_v2/bigconfig.yml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/blogs_goals_v2/bigconfig.yml new file mode 100644 index 00000000000..537080abf95 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/blogs_goals_v2/bigconfig.yml @@ -0,0 +1,65 @@ +type: BIGCONFIG_FILE + +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.blogs_goals_v2.date + +saved_metric_definitions: + metrics: + - saved_metric_id: COUNT_DUPLICATES + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + group_by: + - date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + rct_overrides: + - date + - saved_metric_id: VISIT_IDENTIFIER_REGEX_CHECK + metric_type: + type: TEMPLATE + template_id: 947 + aggregation_type: COUNT + template_name: visit_identifier_regex_check + metric_name: COUNT of visit_identifier_regex_check + group_by: + - date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + parameters: + - key: column_name + string_value: visit_identifier + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + rct_overrides: + - date + +tag_deployments: +- collection: + name: blogs_goals_v2 + description: SDK Generated + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.blogs_goals_v2.visit_identifier + metrics: + - saved_metric_id: COUNT_DUPLICATES + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.blogs_goals_v2 + metrics: + - saved_metric_id: VISIT_IDENTIFIER_REGEX_CHECK diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/blogs_goals_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/blogs_goals_v2/metadata.yaml index c2042d8d0de..40c41fdc48c 100644 --- a/sql/moz-fx-data-shared-prod/mozilla_org_derived/blogs_goals_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/blogs_goals_v2/metadata.yaml @@ -20,4 +20,6 @@ bigquery: clustering: fields: - visit_identifier +monitoring: + enabled: true references: {} diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/downloads_with_attribution_v2/bigconfig.yml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/downloads_with_attribution_v2/bigconfig.yml new file mode 100644 index 00000000000..07904bb8b4d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/downloads_with_attribution_v2/bigconfig.yml @@ -0,0 +1,41 @@ +type: BIGCONFIG_FILE +saved_metric_definitions: + metrics: + - saved_metric_id: count_number_rows + metric_type: + type: PREDEFINED + predefined_metric: COUNT_ROWS + metric_name: downloads_with_attribution_v2_row_count + group_by: + - download_date + threshold: + type: CONSTANT + lower_bound: 50000.0 + rct_overrides: + - download_date +tag_deployments: +- collection: + name: Google Analytics + description: All GA related data + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.downloads_with_attribution_v2.* + metrics: + - saved_metric_id: count_number_rows + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.downloads_with_attribution_v2.* + metrics: + - metric_type: + type: PREDEFINED + predefined_metric: FRESHNESS + metric_name: FRESHNESS [warn] + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + - metric_type: + type: PREDEFINED + predefined_metric: VOLUME + metric_name: VOLUME [fail] + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/downloads_with_attribution_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/downloads_with_attribution_v2/metadata.yaml index bc9c6a8fe4c..65298c2b339 100644 --- a/sql/moz-fx-data-shared-prod/mozilla_org_derived/downloads_with_attribution_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/downloads_with_attribution_v2/metadata.yaml @@ -60,3 +60,6 @@ bigquery: expiration_days: null clustering: null references: {} +monitoring: + enabled: true + diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_clients_v2/bigconfig.yml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_clients_v2/bigconfig.yml new file mode 100644 index 00000000000..23f3e5d83dd --- /dev/null +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_clients_v2/bigconfig.yml @@ -0,0 +1,61 @@ +type: BIGCONFIG_FILE + +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.ga_clients_v2.first_seen_date + +saved_metric_definitions: + metrics: + - saved_metric_id: COUNT_DUPLICATES + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + rct_overrides: + - bigeye-no-rct + - saved_metric_id: COUNT_ROWS + metric_type: + type: PREDEFINED + predefined_metric: COUNT_ROWS + metric_name: Row count (#) + conditions: + - "first_seen_date >= '2024-01-01'\n and first_reported.country IN ('United States',\ + \ 'Canada')" + group_by: + - first_seen_date + - first_reported.country + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + rct_overrides: + - first_seen_date + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + +tag_deployments: +- collection: + name: Google Analytics + description: All checks related to GA tables + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.ga_clients_v2.ga_client_id + metrics: + - saved_metric_id: COUNT_DUPLICATES + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.ga_clients_v2.* + metrics: + - saved_metric_id: COUNT_ROWS diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_clients_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_clients_v2/metadata.yaml index 0476953d594..7c0767c60ba 100644 --- a/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_clients_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_clients_v2/metadata.yaml @@ -16,4 +16,6 @@ scheduling: bigquery: clustering: fields: ["first_seen_date"] +monitoring: + enabled: true references: {} diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_sessions_v2/bigconfig.yml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_sessions_v2/bigconfig.yml new file mode 100644 index 00000000000..a8e749beb2c --- /dev/null +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_sessions_v2/bigconfig.yml @@ -0,0 +1,60 @@ +type: BIGCONFIG_FILE + +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.ga_sessions_v2.session_date + +saved_metric_definitions: + metrics: + - saved_metric_id: PERCENT_NULL + metric_type: + type: PREDEFINED + predefined_metric: PERCENT_NULL + metric_name: Null (%) + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + rct_overrides: + - session_date + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + - saved_metric_id: COUNT_ROWS + metric_type: + type: PREDEFINED + predefined_metric: COUNT_ROWS + metric_name: Row count (#) + group_by: + - ga_session_id + - ga_client_id + threshold: + type: CONSTANT + upper_bound: 1.0 + lower_bound: 0.0 + rct_overrides: + - bigeye-no-rct + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC +tag_deployments: +- collection: + name: Google Analytics + description: All checks related to GA tables + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.ga_sessions_v2.session_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.ga_sessions_v2.ga_session_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.ga_sessions_v2.ga_client_id + metrics: + - saved_metric_id: PERCENT_NULL + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.ga_sessions_v2.* + metrics: + - saved_metric_id: COUNT_ROWS diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_sessions_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_sessions_v2/metadata.yaml index 85dda1d49f2..cd0bed3bbac 100644 --- a/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_sessions_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_sessions_v2/metadata.yaml @@ -31,3 +31,6 @@ workgroup_access: - workgroup:mozilla-confidential - workgroup:google-managed/external-ads-datafusion - workgroup:google-managed/external-ads-dataproc +monitoring: + enabled: true + diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/www_site_hits_v2/bigconfig.yml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/www_site_hits_v2/bigconfig.yml new file mode 100644 index 00000000000..fa6c09a96b4 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/www_site_hits_v2/bigconfig.yml @@ -0,0 +1,45 @@ +type: BIGCONFIG_FILE + +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.www_site_hits_v2.date + +saved_metric_definitions: + metrics: + - saved_metric_id: VISIT_IDENTIFIER_REGEX_CHECK + metric_type: + type: TEMPLATE + template_id: 947 + aggregation_type: COUNT + template_name: visit_identifier_regex_check + metric_name: mozilla_org_derived.www_site_hits_v2 - COUNT of visit_identifier_regex_check + group_by: + - date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + parameters: + - key: column_name + string_value: visit_identifier + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + rct_overrides: + - date + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + +tag_deployments: +- collection: + name: Google Analytics + description: All checks related to GA tables + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.mozilla_org_derived.www_site_hits_v2 + metrics: + - saved_metric_id: VISIT_IDENTIFIER_REGEX_CHECK diff --git a/sql/moz-fx-data-shared-prod/mozilla_org_derived/www_site_hits_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/mozilla_org_derived/www_site_hits_v2/metadata.yaml index b66c49e1b52..1385d3eb18a 100644 --- a/sql/moz-fx-data-shared-prod/mozilla_org_derived/www_site_hits_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/mozilla_org_derived/www_site_hits_v2/metadata.yaml @@ -22,4 +22,6 @@ bigquery: - country - language - event_name +monitoring: + enabled: true references: {}