From 58fffb0a9535ac43745d88461e2f125bf8d27042 Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Wed, 15 Jan 2025 21:58:20 -0500 Subject: [PATCH 1/3] DENG-7414_Set retention on FxA client level tables --- .../users_services_daily_v1/metadata.yaml | 6 ++++-- .../users_services_last_seen_v1/metadata.yaml | 6 ++++-- .../fxa_amplitude_export_v1/metadata.yaml | 1 + .../fxa_delete_events_v1/metadata.yaml | 2 +- .../fxa_delete_events_v2/metadata.yaml | 1 + .../fxa_log_device_command_events_v1/metadata.yaml | 1 + .../fxa_log_device_command_events_v2/metadata.yaml | 1 + .../fxa_users_daily_v1/metadata.yaml | 1 + .../fxa_users_daily_v2/metadata.yaml | 1 + .../fxa_users_last_seen_v1/metadata.yaml | 2 +- .../fxa_users_last_seen_v2/metadata.yaml | 2 +- .../fxa_users_services_daily_v1/metadata.yaml | 1 + .../fxa_users_services_daily_v2/metadata.yaml | 2 +- .../fxa_users_services_devices_daily_v1/metadata.yaml | 1 + .../fxa_users_services_last_seen_v1/metadata.yaml | 2 +- .../fxa_users_services_last_seen_v2/metadata.yaml | 1 + .../events_daily/templates/events_daily_v1/metadata.yaml | 2 ++ 17 files changed, 24 insertions(+), 9 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/metadata.yaml index e20946d8ef8..659a44d1345 100644 --- a/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/metadata.yaml @@ -5,7 +5,9 @@ owners: - ksiegler@mozilla.com labels: incremental: true - owner1: ksiegler@mozilla.com + owner1: akomar@mozilla.com + owner2: kik@mozilla.com + table_type: client_level scheduling: dag_name: bqetl_accounts_derived bigquery: @@ -13,7 +15,7 @@ bigquery: type: day field: submission_date require_partition_filter: true - expiration_days: null + expiration_days: 775 clustering: fields: [service, country] references: {} diff --git a/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_last_seen_v1/metadata.yaml index ac801f00dfb..1db1e24bf6d 100644 --- a/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_last_seen_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_last_seen_v1/metadata.yaml @@ -5,7 +5,9 @@ owners: - ksiegler@mozilla.com labels: incremental: true - owner1: ksiegler@mozilla.com + owner1: akomar@mozilla.com + owner2: kik@mozilla.com + table_type: client_level scheduling: dag_name: bqetl_accounts_derived bigquery: @@ -13,7 +15,7 @@ bigquery: type: day field: submission_date require_partition_filter: true - expiration_days: null + expiration_days: 775 clustering: fields: [service, country] references: {} diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/metadata.yaml index ca60095a733..b08191d8d9c 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/metadata.yaml @@ -15,6 +15,7 @@ bigquery: type: day field: submission_date_pacific require_partition_filter: false + expiration_days: 775 clustering: fields: - user_id diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/metadata.yaml index d1952f0e1d0..9447e45eb77 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/metadata.yaml @@ -17,6 +17,6 @@ bigquery: type: day field: submission_timestamp require_partition_filter: false - expiration_days: null + expiration_days: 775 clustering: null references: {} diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v2/metadata.yaml index 9ca0775207f..79df468f7ac 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v2/metadata.yaml @@ -17,3 +17,4 @@ bigquery: type: day field: submission_timestamp require_partition_filter: false + expiration_days: 775 diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/metadata.yaml index 42dd0f8e90c..233f08dc545 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/metadata.yaml @@ -22,6 +22,7 @@ bigquery: type: day field: timestamp require_partition_filter: false + expiration_days: 775 clustering: fields: - command diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/metadata.yaml index 8ee9790b99c..e9d6db305cc 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v2/metadata.yaml @@ -24,6 +24,7 @@ bigquery: type: day field: timestamp require_partition_filter: false + expiration_days: 775 clustering: fields: - command diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v1/metadata.yaml index 18fd1380a87..198231d5eac 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v1/metadata.yaml @@ -21,6 +21,7 @@ bigquery: type: day field: submission_date require_partition_filter: false + expiration_days: 775 clustering: fields: - user_id diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v2/metadata.yaml index 002d05856a7..ee93ad5441e 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_daily_v2/metadata.yaml @@ -28,6 +28,7 @@ bigquery: type: day field: submission_date require_partition_filter: true + expiration_days: 775 clustering: fields: - registered diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v1/metadata.yaml index 2b18d0be4f0..6855d84e74b 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v1/metadata.yaml @@ -25,7 +25,7 @@ bigquery: type: day field: submission_date require_partition_filter: true - expiration_days: null + expiration_days: 775 clustering: fields: - user_id diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v2/metadata.yaml index 7ec847a8372..4cd4cc76310 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_last_seen_v2/metadata.yaml @@ -25,4 +25,4 @@ bigquery: type: day field: submission_date require_partition_filter: true - expiration_days: null + expiration_days: 775 diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v1/metadata.yaml index 63b9c448606..574268a4674 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v1/metadata.yaml @@ -16,6 +16,7 @@ bigquery: type: day field: submission_date require_partition_filter: false + expiration_days: 775 clustering: fields: - service diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v2/metadata.yaml index 4054a45ed58..73551fb28cc 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_daily_v2/metadata.yaml @@ -16,7 +16,7 @@ bigquery: type: day field: submission_date require_partition_filter: false - + expiration_days: 775 clustering: fields: - service diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_devices_daily_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_devices_daily_v1/metadata.yaml index bc8591f35df..d1b45c3db6c 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_devices_daily_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_devices_daily_v1/metadata.yaml @@ -42,6 +42,7 @@ bigquery: type: day field: timestamp require_partition_filter: true + expiration_days: 775 clustering: fields: - service diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/metadata.yaml index 281b1621edb..58ccbccc4d9 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/metadata.yaml @@ -14,7 +14,7 @@ bigquery: type: day field: submission_date require_partition_filter: false - expiration_days: null + expiration_days: 775 clustering: fields: - service diff --git a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v2/metadata.yaml index a63b69f4801..bd717d0594b 100644 --- a/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v2/metadata.yaml @@ -18,6 +18,7 @@ bigquery: type: day field: submission_date require_partition_filter: false + expiration_days: 775 clustering: fields: - service diff --git a/sql_generators/events_daily/templates/events_daily_v1/metadata.yaml b/sql_generators/events_daily/templates/events_daily_v1/metadata.yaml index 4ec78a88ac3..7f08e628c97 100644 --- a/sql_generators/events_daily/templates/events_daily_v1/metadata.yaml +++ b/sql_generators/events_daily/templates/events_daily_v1/metadata.yaml @@ -9,6 +9,7 @@ labels: application: {{ dataset }} schedule: daily incremental: true + table_type: client_level scheduling: dag_name: {{ dag_name }} bigquery: @@ -16,6 +17,7 @@ bigquery: type: day field: submission_date require_partition_filter: true + expiration_days: 775 clustering: fields: - sample_id From fc3ea5c7dcc1e456ccac52b8f987f0b6544105ac Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Thu, 16 Jan 2025 09:49:56 -0500 Subject: [PATCH 2/3] Fix CI issue --- .../users_services_daily_v1/metadata.yaml | 1 - .../users_services_last_seen_v1/metadata.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/metadata.yaml index 659a44d1345..56064dc8d92 100644 --- a/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/metadata.yaml @@ -6,7 +6,6 @@ owners: labels: incremental: true owner1: akomar@mozilla.com - owner2: kik@mozilla.com table_type: client_level scheduling: dag_name: bqetl_accounts_derived diff --git a/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_last_seen_v1/metadata.yaml index 1db1e24bf6d..f5da5125f0f 100644 --- a/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_last_seen_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_last_seen_v1/metadata.yaml @@ -6,7 +6,6 @@ owners: labels: incremental: true owner1: akomar@mozilla.com - owner2: kik@mozilla.com table_type: client_level scheduling: dag_name: bqetl_accounts_derived From c1bdc3e3f055cd7babf39c9077652521b3a15d95 Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Thu, 16 Jan 2025 14:22:10 -0500 Subject: [PATCH 3/3] Remove file --- .../templates/events_daily_v1/metadata.yaml | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 sql_generators/events_daily/templates/events_daily_v1/metadata.yaml diff --git a/sql_generators/events_daily/templates/events_daily_v1/metadata.yaml b/sql_generators/events_daily/templates/events_daily_v1/metadata.yaml deleted file mode 100644 index 7f08e628c97..00000000000 --- a/sql_generators/events_daily/templates/events_daily_v1/metadata.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -friendly_name: '{{ name }} Events Daily' -description: > - Packed event representation with one-row per-client -owners: - - wlachance@mozilla.com - - akomar@mozilla.com -labels: - application: {{ dataset }} - schedule: daily - incremental: true - table_type: client_level -scheduling: - dag_name: {{ dag_name }} -bigquery: - time_partitioning: - type: day - field: submission_date - require_partition_filter: true - expiration_days: 775 - clustering: - fields: - - sample_id -