add dbt integration tests (#812)

* add dbt integration tests

* add gitignore and env_example files

* change the database creation

Author: osmazur

test/dbt_integration_tests/packages_integration_tests/.env_example

@@ -0,0 +1,23 @@
+
+
+export DBT_TARGET=embucket
+
+export EMBUCKET_HOST=localhost
+export EMBUCKET_PORT=3000
+export EMBUCKET_PROTOCOL=http
+export EMBUCKET_ACCOUNT=test
+export EMBUCKET_USER=test
+export EMBUCKET_PASSWORD=test
+export EMBUCKET_ROLE=SYSADMIN
+export EMBUCKET_DATABASE=embucket
+export EMBUCKET_WAREHOUSE=COMPUTE_WH
+export EMBUCKET_SCHEMA=public
+
+
+export SNOWFLAKE_ACCOUNT=
+export SNOWFLAKE_USER=
+export DBT_ENV_SECRET_SNOWFLAKE_PASS=
+export SNOWFLAKE_ROLE=
+export SNOWFLAKE_DATABASE=
+export SNOWFLAKE_WAREHOUSE=
+export SNOWFLAKE_SCHEMA=

test/dbt_integration_tests/packages_integration_tests/.gitignore

@@ -0,0 +1,7 @@
+.env
+.user.yml
+target/
+data/
+packages/
+logs/
+.DS_Store

test/dbt_integration_tests/packages_integration_tests/README.md

@@ -0,0 +1,21 @@
+# DBT integration tests
+
+1. Copy .env_example inside dbt_integration_tests/packages_integration_tests folder file and rename it to .env file.
+
+2. Set up a connection
+	1. Snowflake - replace Snowflake credentials in .env file from test to your credentials.
+	2. Embucket - launch Embucket locally, make sure connection parameters match Embucket launch parameters (if you have default settings, you don't need to change anything).
+	3. Set the target database DBT_TARGET env (embucket or snowflake) by default it will be embucket
+
+3. Make the sh file executable
+```sh
+chmod +x run_test.sh
+```
+
+4. Run integration test
+```sh
+./run_test.sh
+```
+
+In repos.yml there are packages with integrations tests. By default it runs integrations tests from the dbt-snowplow-web package.
+Feel free to test any package from the repos_full_list.yml file.

test/dbt_integration_tests/packages_integration_tests/profiles.yml

@@ -0,0 +1,28 @@
+default:
+  target: embucket
+  outputs:
+    snowflake:
+      type: snowflake
+      threads: 4
+      account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" 
+      user: "{{ env_var('SNOWFLAKE_USER') }}" 
+      password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" 
+      database: "{{ env_var('SNOWFLAKE_DATABASE') }}" 
+      role: "{{ env_var('SNOWFLAKE_ROLE') }}" 
+      warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" 
+      schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" 
+      client_session_keep_alive: True
+
+    embucket:
+      type: snowflake
+      host: "{{ env_var('EMBUCKET_HOST') }}"
+      port: 3000
+      protocol: "{{ env_var('EMBUCKET_PROTOCOL') }}"
+      account: "test"
+      user: "{{ env_var('EMBUCKET_USER') }}"
+      password: "{{ env_var('EMBUCKET_PASSWORD') }}"
+      role: "{{ env_var('EMBUCKET_ROLE') }}"
+      database: "embucket"
+      warehouse: "{{ env_var('EMBUCKET_WAREHOUSE') }}"
+      schema: "{{ env_var('EMBUCKET_SCHEMA') }}"
+      threads: 32

test/dbt_integration_tests/packages_integration_tests/repos.yml

@@ -0,0 +1,3 @@
+https://github.com/snowplow/dbt-snowplow-web.git
+#https://github.com/dbt-labs/dbt-utils.git
+

test/dbt_integration_tests/packages_integration_tests/repos_full_list.yml

@@ -0,0 +1,161 @@
+https://github.com/fivetran/dbt_ad_reporting.git
+https://github.com/fivetran/dbt_amazon_ads.git
+https://github.com/fivetran/dbt_amazon_ads_source.git
+https://github.com/fivetran/dbt_amazon_selling_partner.git
+https://github.com/fivetran/dbt_amplitude.git
+https://github.com/fivetran/dbt_amplitude_source.git
+https://github.com/fivetran/dbt_app_reporting.git
+https://github.com/fivetran/dbt_apple_search_ads.git
+https://github.com/fivetran/dbt_apple_search_ads_source.git
+https://github.com/fivetran/dbt_apple_store.git
+https://github.com/fivetran/dbt_apple_store_source.git
+https://github.com/fivetran/dbt_asana.git
+https://github.com/fivetran/dbt_asana_source.git
+https://github.com/fivetran/dbt_aws_cloud_cost.git
+https://github.com/fivetran/dbt_dynamics_365_crm.git
+https://github.com/fivetran/dbt_facebook_ads.git
+https://github.com/fivetran/dbt_facebook_ads_creative_history.git
+https://github.com/fivetran/dbt_facebook_ads_source.git
+https://github.com/fivetran/dbt_facebook_pages.git
+https://github.com/fivetran/dbt_facebook_pages_source.git
+https://github.com/fivetran/dbt_fivetran_log.git
+https://github.com/fivetran/dbt_fivetran_utils.git
+https://github.com/fivetran/dbt_ga4_export.git
+https://github.com/fivetran/dbt_github.git
+https://github.com/fivetran/dbt_github_source.git
+https://github.com/fivetran/dbt_google_ads.git
+https://github.com/fivetran/dbt_google_ads_source.git
+https://github.com/fivetran/dbt_google_play.git
+https://github.com/fivetran/dbt_google_play_source.git
+https://github.com/fivetran/dbt_greenhouse.git
+https://github.com/fivetran/dbt_greenhouse_source.git
+https://github.com/fivetran/dbt_hubspot.git
+https://github.com/fivetran/dbt_hubspot_source.git
+https://github.com/fivetran/dbt_instagram_business.git
+https://github.com/fivetran/dbt_instagram_business_source.git
+https://github.com/fivetran/dbt_intercom.git
+https://github.com/fivetran/dbt_intercom_source.git
+https://github.com/fivetran/dbt_iterable.git
+https://github.com/fivetran/dbt_iterable_source.git
+https://github.com/fivetran/dbt_jira.git
+https://github.com/fivetran/dbt_jira_source.git
+https://github.com/fivetran/dbt_klaviyo.git
+https://github.com/fivetran/dbt_klaviyo_source.git
+https://github.com/fivetran/dbt_lever.git
+https://github.com/fivetran/dbt_lever_source.git
+https://github.com/fivetran/dbt_linkedin.git
+https://github.com/fivetran/dbt_linkedin_pages.git
+https://github.com/fivetran/dbt_linkedin_pages_source.git
+https://github.com/fivetran/dbt_linkedin_source.git
+https://github.com/fivetran/dbt_mailchimp.git
+https://github.com/fivetran/dbt_mailchimp_source.git
+https://github.com/fivetran/dbt_marketo.git
+https://github.com/fivetran/dbt_marketo_source.git
+https://github.com/fivetran/dbt_microsoft_ads.git
+https://github.com/fivetran/dbt_microsoft_ads_source.git
+https://github.com/fivetran/dbt_mixpanel.git
+https://github.com/fivetran/dbt_netsuite.git
+https://github.com/fivetran/dbt_netsuite_source.git
+https://github.com/fivetran/dbt_pardot.git
+https://github.com/fivetran/dbt_pardot_source.git
+https://github.com/fivetran/dbt_pendo.git
+https://github.com/fivetran/dbt_pendo_source.git
+https://github.com/fivetran/dbt_pinterest.git
+https://github.com/fivetran/dbt_pinterest_source.git
+https://github.com/fivetran/dbt_qualtrics.git
+https://github.com/fivetran/dbt_qualtrics_source.git
+https://github.com/fivetran/dbt_quickbooks.git
+https://github.com/fivetran/dbt_quickbooks_source.git
+https://github.com/fivetran/dbt_recharge.git
+https://github.com/fivetran/dbt_recharge_source.git
+https://github.com/fivetran/dbt_recurly.git
+https://github.com/fivetran/dbt_recurly_source.git
+https://github.com/fivetran/dbt_reddit_ads.git
+https://github.com/fivetran/dbt_reddit_ads_source.git
+https://github.com/fivetran/dbt_sage_intacct.git
+https://github.com/fivetran/dbt_sage_intacct_source.git
+https://github.com/fivetran/dbt_salesforce.git
+https://github.com/fivetran/dbt_salesforce_formula_utils.git
+https://github.com/fivetran/dbt_salesforce_marketing_cloud.git
+https://github.com/fivetran/dbt_salesforce_source.git
+https://github.com/fivetran/dbt_sap.git
+https://github.com/fivetran/dbt_sap_source.git
+https://github.com/fivetran/dbt_servicenow.git
+https://github.com/fivetran/dbt_shopify.git
+https://github.com/fivetran/dbt_shopify_holistic_reporting.git
+https://github.com/fivetran/dbt_shopify_source.git
+https://github.com/fivetran/dbt_snapchat_ads.git
+https://github.com/fivetran/dbt_snapchat_ads_source.git
+https://github.com/fivetran/dbt_social_media_reporting.git
+https://github.com/fivetran/dbt_stripe.git
+https://github.com/fivetran/dbt_stripe_source.git
+https://github.com/fivetran/dbt_tiktok_ads.git
+https://github.com/fivetran/dbt_tiktok_ads_source.git
+https://github.com/fivetran/dbt_twilio.git
+https://github.com/fivetran/dbt_twilio_source.git
+https://github.com/fivetran/dbt_twitter.git
+https://github.com/fivetran/dbt_twitter_source.git
+https://github.com/fivetran/dbt_twitter_organic.git
+https://github.com/fivetran/dbt_twitter_organic_source.git
+https://github.com/fivetran/dbt_unified_rag.git
+https://github.com/fivetran/dbt_workday.git
+https://github.com/fivetran/dbt_xero.git
+https://github.com/fivetran/dbt_xero_source.git
+https://github.com/fivetran/dbt_youtube_analytics.git
+https://github.com/fivetran/dbt_youtube_analytics_source.git
+https://github.com/fivetran/dbt_zendesk.git
+https://github.com/fivetran/dbt_zendesk_source.git
+https://github.com/fivetran/dbt_zuora.git
+https://github.com/fivetran/dbt_zuora_source.git
+https://github.com/Fleetio/dbt-segment.git
+https://github.com/fullstorydev/dbt_fullstory.git
+https://github.com/get-select/dbt-snowflake-query-tags.git
+https://github.com/godatadriven/dbt-date.git
+https://github.com/google/fhir-dbt-utils.git
+https://github.com/infinitelambda/dbt-audit-helper-ext.git
+https://github.com/infinitelambda/dbt-data-diff.git
+https://github.com/infinitelambda/dbt-alert.git
+https://github.com/infinitelambda/dbt-tags.git
+https://github.com/infinitelambda/dq-tools.git
+https://github.com/infinitelambda/dq-vault.git
+https://github.com/jmangroup/data_profiler.git
+https://github.com/jpmmcneill/dbt-graph-theory.git
+https://github.com/kgmcquate/dbt-testgen.git
+https://github.com/LewisDavies/upstream-prod.git
+https://github.com/MaterializeInc/materialize-dbt-utils.git
+https://github.com/Matts52/dbt-ml-inline-preprocessing.git
+https://github.com/Matts52/dbt-set-similarity.git
+https://github.com/metaplane/dbt-expectations.git
+https://github.com/mjirv/dbt-datamocktool.git
+https://github.com/mjirv/dbt_product_analytics.git
+https://github.com/narratorai/dbt-activity-schema.git
+https://github.com/oleg-solovyev/dbt_facebook_ads.git
+https://github.com/omnata-labs/dbt-ml-preprocessing.git
+https://github.com/pvcy/dbt-privacy.git
+https://github.com/re-data/dbt-re-data.git
+https://github.com/rjh336/dbt-model-usage.git
+https://github.com/rlsalcido24/lakehouse_utils.git
+https://github.com/Snowflake-Labs/dbt_constraints.git
+https://github.com/snowplow/dbt-snowplow-attribution.git
+https://github.com/snowplow/dbt-snowplow-ecommerce.git
+https://github.com/snowplow/dbt-snowplow-fractribution.git
+https://github.com/snowplow/dbt-snowplow-media-player.git
+https://github.com/snowplow/dbt-snowplow-mobile.git
+https://github.com/snowplow/dbt-snowplow-normalize.git
+https://github.com/snowplow/dbt-snowplow-unified.git
+https://github.com/snowplow/dbt-snowplow-utils.git
+https://github.com/snowplow/dbt-snowplow-web.git
+https://github.com/starburstdata/dbt-trino-utils.git
+https://github.com/sutrolabs/dbt_census_utils.git
+https://github.com/TasmanAnalytics/tasman-dbt-mta.git
+https://github.com/TasmanAnalytics/tasman-dbt-revenuecat.git
+https://github.com/TasmanAnalytics/tasman-identity-graph.git
+https://github.com/techindicium/dbt-metalog.git
+https://github.com/tnightengale/dbt-activity-schema.git
+https://github.com/tnightengale/dbt-meta-testing.git
+https://github.com/tuva-health/the_tuva_project.git
+https://github.com/windsor-ai/dbt_facebook_ads.git
+https://github.com/yu-iskw/dbt-airflow-macros.git
+https://github.com/yu-iskw/dbt-gcp-billing.git
+https://github.com/yu-iskw/dbt-ops.git
+https://github.com/yu-iskw/dbt-unittest.git

test/dbt_integration_tests/packages_integration_tests/requirements.txt

@@ -0,0 +1,2 @@
+requests
+snowflake-connector-python