Skip to content

Commit 54560a4

Browse files
olemmazurolemmazur
committed
add dbt integration tests
1 parent bc19887 commit 54560a4

File tree

8 files changed

+1010
-0
lines changed

8 files changed

+1010
-0
lines changed

‎test/dbt_integration_tests/packages_integration_tests/README.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# DBT integration tests
2+
3+
1. Copy .env_example inside dbt_integration_tests/packages_integration_tests folder file and rename it to .env file.
4+
5+
2. Set up a connection
6+
1. Snowflake - replace Snowflake credentials in .env file from test to your credentials.
7+
2. Embucket - launch Embucket locally, make sure connection parameters match Embucket launch parameters (if you have default settings, you don't need to change anything).
8+
3. Set the target database DBT_TARGET env (embucket or snowflake) by default it will be embucket
9+
10+
3. Make the sh file executable
11+
```sh
12+
chmod +x run_test.sh
13+
```
14+
15+
4. Run integration test
16+
```sh
17+
./run_test.sh
18+
```
19+
20+
In repos.yml there are packages with integrations tests. By default it runs integrations tests from the dbt-snowplow-web package.
21+
Feel free to test any package from the repos_full_list.yml file.

‎test/dbt_integration_tests/packages_integration_tests/Web_Analytics_sample_events.csv

Lines changed: 82 additions & 0 deletions
Large diffs are not rendered by default.

‎test/dbt_integration_tests/packages_integration_tests/profiles.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
default:
2+
target: embucket
3+
outputs:
4+
snowflake:
5+
type: snowflake
6+
threads: 4
7+
account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
8+
user: "{{ env_var('SNOWFLAKE_USER') }}"
9+
password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}"
10+
database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
11+
role: "{{ env_var('SNOWFLAKE_ROLE') }}"
12+
warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"
13+
schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}"
14+
client_session_keep_alive: True
15+
16+
embucket:
17+
type: snowflake
18+
host: "{{ env_var('EMBUCKET_HOST') }}"
19+
port: 3000
20+
protocol: "{{ env_var('EMBUCKET_PROTOCOL') }}"
21+
account: "test"
22+
user: "{{ env_var('EMBUCKET_USER') }}"
23+
password: "{{ env_var('EMBUCKET_PASSWORD') }}"
24+
role: "{{ env_var('EMBUCKET_ROLE') }}"
25+
database: "embucket"
26+
warehouse: "{{ env_var('EMBUCKET_WAREHOUSE') }}"
27+
schema: "{{ env_var('EMBUCKET_SCHEMA') }}"
28+
threads: 32

‎test/dbt_integration_tests/packages_integration_tests/repos.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
https://github.com/snowplow/dbt-snowplow-web.git
2+
#https://github.com/dbt-labs/dbt-utils.git
3+

‎test/dbt_integration_tests/packages_integration_tests/repos_full_list.yml

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
https://github.com/fivetran/dbt_ad_reporting.git
2+
https://github.com/fivetran/dbt_amazon_ads.git
3+
https://github.com/fivetran/dbt_amazon_ads_source.git
4+
https://github.com/fivetran/dbt_amazon_selling_partner.git
5+
https://github.com/fivetran/dbt_amplitude.git
6+
https://github.com/fivetran/dbt_amplitude_source.git
7+
https://github.com/fivetran/dbt_app_reporting.git
8+
https://github.com/fivetran/dbt_apple_search_ads.git
9+
https://github.com/fivetran/dbt_apple_search_ads_source.git
10+
https://github.com/fivetran/dbt_apple_store.git
11+
https://github.com/fivetran/dbt_apple_store_source.git
12+
https://github.com/fivetran/dbt_asana.git
13+
https://github.com/fivetran/dbt_asana_source.git
14+
https://github.com/fivetran/dbt_aws_cloud_cost.git
15+
https://github.com/fivetran/dbt_dynamics_365_crm.git
16+
https://github.com/fivetran/dbt_facebook_ads.git
17+
https://github.com/fivetran/dbt_facebook_ads_creative_history.git
18+
https://github.com/fivetran/dbt_facebook_ads_source.git
19+
https://github.com/fivetran/dbt_facebook_pages.git
20+
https://github.com/fivetran/dbt_facebook_pages_source.git
21+
https://github.com/fivetran/dbt_fivetran_log.git
22+
https://github.com/fivetran/dbt_fivetran_utils.git
23+
https://github.com/fivetran/dbt_ga4_export.git
24+
https://github.com/fivetran/dbt_github.git
25+
https://github.com/fivetran/dbt_github_source.git
26+
https://github.com/fivetran/dbt_google_ads.git
27+
https://github.com/fivetran/dbt_google_ads_source.git
28+
https://github.com/fivetran/dbt_google_play.git
29+
https://github.com/fivetran/dbt_google_play_source.git
30+
https://github.com/fivetran/dbt_greenhouse.git
31+
https://github.com/fivetran/dbt_greenhouse_source.git
32+
https://github.com/fivetran/dbt_hubspot.git
33+
https://github.com/fivetran/dbt_hubspot_source.git
34+
https://github.com/fivetran/dbt_instagram_business.git
35+
https://github.com/fivetran/dbt_instagram_business_source.git
36+
https://github.com/fivetran/dbt_intercom.git
37+
https://github.com/fivetran/dbt_intercom_source.git
38+
https://github.com/fivetran/dbt_iterable.git
39+
https://github.com/fivetran/dbt_iterable_source.git
40+
https://github.com/fivetran/dbt_jira.git
41+
https://github.com/fivetran/dbt_jira_source.git
42+
https://github.com/fivetran/dbt_klaviyo.git
43+
https://github.com/fivetran/dbt_klaviyo_source.git
44+
https://github.com/fivetran/dbt_lever.git
45+
https://github.com/fivetran/dbt_lever_source.git
46+
https://github.com/fivetran/dbt_linkedin.git
47+
https://github.com/fivetran/dbt_linkedin_pages.git
48+
https://github.com/fivetran/dbt_linkedin_pages_source.git
49+
https://github.com/fivetran/dbt_linkedin_source.git
50+
https://github.com/fivetran/dbt_mailchimp.git
51+
https://github.com/fivetran/dbt_mailchimp_source.git
52+
https://github.com/fivetran/dbt_marketo.git
53+
https://github.com/fivetran/dbt_marketo_source.git
54+
https://github.com/fivetran/dbt_microsoft_ads.git
55+
https://github.com/fivetran/dbt_microsoft_ads_source.git
56+
https://github.com/fivetran/dbt_mixpanel.git
57+
https://github.com/fivetran/dbt_netsuite.git
58+
https://github.com/fivetran/dbt_netsuite_source.git
59+
https://github.com/fivetran/dbt_pardot.git
60+
https://github.com/fivetran/dbt_pardot_source.git
61+
https://github.com/fivetran/dbt_pendo.git
62+
https://github.com/fivetran/dbt_pendo_source.git
63+
https://github.com/fivetran/dbt_pinterest.git
64+
https://github.com/fivetran/dbt_pinterest_source.git
65+
https://github.com/fivetran/dbt_qualtrics.git
66+
https://github.com/fivetran/dbt_qualtrics_source.git
67+
https://github.com/fivetran/dbt_quickbooks.git
68+
https://github.com/fivetran/dbt_quickbooks_source.git
69+
https://github.com/fivetran/dbt_recharge.git
70+
https://github.com/fivetran/dbt_recharge_source.git
71+
https://github.com/fivetran/dbt_recurly.git
72+
https://github.com/fivetran/dbt_recurly_source.git
73+
https://github.com/fivetran/dbt_reddit_ads.git
74+
https://github.com/fivetran/dbt_reddit_ads_source.git
75+
https://github.com/fivetran/dbt_sage_intacct.git
76+
https://github.com/fivetran/dbt_sage_intacct_source.git
77+
https://github.com/fivetran/dbt_salesforce.git
78+
https://github.com/fivetran/dbt_salesforce_formula_utils.git
79+
https://github.com/fivetran/dbt_salesforce_marketing_cloud.git
80+
https://github.com/fivetran/dbt_salesforce_source.git
81+
https://github.com/fivetran/dbt_sap.git
82+
https://github.com/fivetran/dbt_sap_source.git
83+
https://github.com/fivetran/dbt_servicenow.git
84+
https://github.com/fivetran/dbt_shopify.git
85+
https://github.com/fivetran/dbt_shopify_holistic_reporting.git
86+
https://github.com/fivetran/dbt_shopify_source.git
87+
https://github.com/fivetran/dbt_snapchat_ads.git
88+
https://github.com/fivetran/dbt_snapchat_ads_source.git
89+
https://github.com/fivetran/dbt_social_media_reporting.git
90+
https://github.com/fivetran/dbt_stripe.git
91+
https://github.com/fivetran/dbt_stripe_source.git
92+
https://github.com/fivetran/dbt_tiktok_ads.git
93+
https://github.com/fivetran/dbt_tiktok_ads_source.git
94+
https://github.com/fivetran/dbt_twilio.git
95+
https://github.com/fivetran/dbt_twilio_source.git
96+
https://github.com/fivetran/dbt_twitter.git
97+
https://github.com/fivetran/dbt_twitter_source.git
98+
https://github.com/fivetran/dbt_twitter_organic.git
99+
https://github.com/fivetran/dbt_twitter_organic_source.git
100+
https://github.com/fivetran/dbt_unified_rag.git
101+
https://github.com/fivetran/dbt_workday.git
102+
https://github.com/fivetran/dbt_xero.git
103+
https://github.com/fivetran/dbt_xero_source.git
104+
https://github.com/fivetran/dbt_youtube_analytics.git
105+
https://github.com/fivetran/dbt_youtube_analytics_source.git
106+
https://github.com/fivetran/dbt_zendesk.git
107+
https://github.com/fivetran/dbt_zendesk_source.git
108+
https://github.com/fivetran/dbt_zuora.git
109+
https://github.com/fivetran/dbt_zuora_source.git
110+
https://github.com/Fleetio/dbt-segment.git
111+
https://github.com/fullstorydev/dbt_fullstory.git
112+
https://github.com/get-select/dbt-snowflake-query-tags.git
113+
https://github.com/godatadriven/dbt-date.git
114+
https://github.com/google/fhir-dbt-utils.git
115+
https://github.com/infinitelambda/dbt-audit-helper-ext.git
116+
https://github.com/infinitelambda/dbt-data-diff.git
117+
https://github.com/infinitelambda/dbt-alert.git
118+
https://github.com/infinitelambda/dbt-tags.git
119+
https://github.com/infinitelambda/dq-tools.git
120+
https://github.com/infinitelambda/dq-vault.git
121+
https://github.com/jmangroup/data_profiler.git
122+
https://github.com/jpmmcneill/dbt-graph-theory.git
123+
https://github.com/kgmcquate/dbt-testgen.git
124+
https://github.com/LewisDavies/upstream-prod.git
125+
https://github.com/MaterializeInc/materialize-dbt-utils.git
126+
https://github.com/Matts52/dbt-ml-inline-preprocessing.git
127+
https://github.com/Matts52/dbt-set-similarity.git
128+
https://github.com/metaplane/dbt-expectations.git
129+
https://github.com/mjirv/dbt-datamocktool.git
130+
https://github.com/mjirv/dbt_product_analytics.git
131+
https://github.com/narratorai/dbt-activity-schema.git
132+
https://github.com/oleg-solovyev/dbt_facebook_ads.git
133+
https://github.com/omnata-labs/dbt-ml-preprocessing.git
134+
https://github.com/pvcy/dbt-privacy.git
135+
https://github.com/re-data/dbt-re-data.git
136+
https://github.com/rjh336/dbt-model-usage.git
137+
https://github.com/rlsalcido24/lakehouse_utils.git
138+
https://github.com/Snowflake-Labs/dbt_constraints.git
139+
https://github.com/snowplow/dbt-snowplow-attribution.git
140+
https://github.com/snowplow/dbt-snowplow-ecommerce.git
141+
https://github.com/snowplow/dbt-snowplow-fractribution.git
142+
https://github.com/snowplow/dbt-snowplow-media-player.git
143+
https://github.com/snowplow/dbt-snowplow-mobile.git
144+
https://github.com/snowplow/dbt-snowplow-normalize.git
145+
https://github.com/snowplow/dbt-snowplow-unified.git
146+
https://github.com/snowplow/dbt-snowplow-utils.git
147+
https://github.com/snowplow/dbt-snowplow-web.git
148+
https://github.com/starburstdata/dbt-trino-utils.git
149+
https://github.com/sutrolabs/dbt_census_utils.git
150+
https://github.com/TasmanAnalytics/tasman-dbt-mta.git
151+
https://github.com/TasmanAnalytics/tasman-dbt-revenuecat.git
152+
https://github.com/TasmanAnalytics/tasman-identity-graph.git
153+
https://github.com/techindicium/dbt-metalog.git
154+
https://github.com/tnightengale/dbt-activity-schema.git
155+
https://github.com/tnightengale/dbt-meta-testing.git
156+
https://github.com/tuva-health/the_tuva_project.git
157+
https://github.com/windsor-ai/dbt_facebook_ads.git
158+
https://github.com/yu-iskw/dbt-airflow-macros.git
159+
https://github.com/yu-iskw/dbt-gcp-billing.git
160+
https://github.com/yu-iskw/dbt-ops.git
161+
https://github.com/yu-iskw/dbt-unittest.git

‎test/dbt_integration_tests/packages_integration_tests/requirements.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
requests
2+
snowflake-connector-python

‎test/dbt_integration_tests/packages_integration_tests/run_test.sh

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
#!/bin/bash
2+
3+
#set -e
4+
5+
# Set your DBT profiles directory relative to the script location
6+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
7+
export DBT_PROFILES_DIR="$SCRIPT_DIR"
8+
9+
# Skip if the repo alredy exists in target dir
10+
SKIP_EXISTING=false
11+
12+
# Set default DBT target (can be overridden by .env)
13+
DBT_TARGET="embucket"
14+
15+
# File with repo URLs
16+
REPOS_FILE="$SCRIPT_DIR/repos.yml"
17+
18+
19+
# Determine Python command
20+
if command -v python3 >/dev/null 2>&1; then
21+
PYTHON_CMD="python3"
22+
elif command -v python >/dev/null 2>&1; then
23+
PYTHON_CMD="python"
24+
else
25+
echo "Error: Neither python3 nor python found. Please install Python."
26+
exit 1
27+
fi
28+
29+
# Check Python version
30+
PYTHON_VERSION=$($PYTHON_CMD --version 2>&1)
31+
if [[ "$PYTHON_VERSION" =~ "Python 2" ]]; then
32+
echo "Error: Python 3 is required."
33+
exit 1
34+
fi
35+
36+
while IFS= read -r repo_url; do
37+
# Skip empty lines or comments
38+
[[ -z "$repo_url" || "$repo_url" =~ ^# ]] && continue
39+
40+
repo_name=$(basename "$repo_url" .git)
41+
repo_dir="packages/$repo_name"
42+
43+
# Setup logs directory and vars
44+
TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
45+
LOG_DIR="$SCRIPT_DIR/logs/$repo_name"
46+
mkdir -p "$LOG_DIR"
47+
LOG_FILE="$LOG_DIR/${TIMESTAMP}.log"
48+
49+
echo "###############################"
50+
echo ""
51+
52+
# Skip if already cloned
53+
if [ "$SKIP_EXISTING" = true ] && [ -d "$repo_dir" ]; then
54+
echo "Skipping $repo_name — already exists in target/"
55+
continue
56+
fi
57+
58+
echo "Cloning $repo_url..."
59+
git clone "$repo_url" "$repo_dir" >/dev/null 2>&1
60+
61+
repo_path="$repo_dir/integration_tests"
62+
echo ""
63+
echo "###############################"
64+
echo ""
65+
66+
if [ -d "$repo_path" ]; then
67+
echo "Setting up for $repo_name..."
68+
69+
cd "$repo_path"
70+
echo ""
71+
# Create and activate virtual env
72+
echo "###############################"
73+
echo ""
74+
echo "Creating virtual environment with $PYTHON_CMD ($PYTHON_VERSION)..."
75+
$PYTHON_CMD -m venv env
76+
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
77+
source env/Scripts/activate
78+
else
79+
source env/bin/activate
80+
fi
81+
echo ""
82+
83+
84+
echo "###############################"
85+
echo ""
86+
echo "Loading environment from .env..."
87+
source "$SCRIPT_DIR/.env"
88+
echo ""
89+
90+
cd "$SCRIPT_DIR"
91+
92+
93+
# Install requirements
94+
echo ""
95+
echo "###############################"
96+
echo ""
97+
echo "Installing the requirements"
98+
pip install -r $SCRIPT_DIR/requirements.txt >/dev/null 2>&1
99+
100+
echo ""
101+
102+
echo "###############################"
103+
echo ""
104+
echo "Creating embucket database"
105+
# Load data and create embucket catalog if the embucket is a target
106+
if [ "$DBT_TARGET" = "embucket" ]; then
107+
$PYTHON_CMD $SCRIPT_DIR/upload.py
108+
fi
109+
echo ""
110+
111+
cd "$repo_path"
112+
113+
DBT_TARGET="${DBT_TARGET:-snowflake}"
114+
115+
# Force profile name to 'default'
116+
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
117+
sed -i "s/^profile: .*/profile: 'default'/" dbt_project.yml
118+
else
119+
sed -i '' "s/^profile: .*/profile: 'default'/" dbt_project.yml
120+
fi
121+
122+
echo "###############################"
123+
124+
echo ""
125+
echo "Installing dbt core dbt-snowflake..."
126+
$PYTHON_CMD -m pip install --upgrade pip >/dev/null 2>&1
127+
$PYTHON_CMD -m pip install dbt-core dbt-snowflake > pip_install.log 2>&1
128+
echo ""
129+
130+
#set +e
131+
echo "###############################"
132+
echo ""
133+
echo "Running DBT tests for $repo_name with target '$DBT_TARGET'..."
134+
set -e
135+
dbt debug --target "$DBT_TARGET"
136+
set +e
137+
dbt clean --target "$DBT_TARGET"
138+
139+
if [ "$repo_name" == "snowplow_web" ]; then
140+
dbt deps --target "$DBT_TARGET"
141+
bash .scripts/integration_test.sh -d "$DBT_TARGET"
142+
elif [ "$repo_name" == "dbt-external-tables" ]; then
143+
dbt deps --target "$DBT_TARGET" | tee -a "$LOG_FILE"
144+
dbt seed --full-refresh --target "$DBT_TARGET" | tee -a "$LOG_FILE"
145+
dbt run-operation prep_external --target "$DBT_TARGET" | tee -a "$LOG_FILE"
146+
dbt run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target "$DBT_TARGET" | tee -a "$LOG_FILE"
147+
dbt run-operation dbt_external_tables.stage_external_sources --target "$DBT_TARGET" | tee -a "$LOG_FILE"
148+
dbt test --target "$DBT_TARGET" | tee -a "$LOG_FILE"
149+
else
150+
dbt deps --target "$DBT_TARGET" | tee -a "$LOG_FILE"
151+
dbt seed --target "$DBT_TARGET" --full-refresh | tee -a "$LOG_FILE"
152+
seed=$(grep -E 'Done\. PASS=.*TOTAL=.*' "$LOG_FILE" | tail -n 1)
153+
dbt run --target "$DBT_TARGET" --full-refresh | tee -a "$LOG_FILE"
154+
run=$(grep -E 'Done\. PASS=.*TOTAL=.*' "$LOG_FILE" | tail -n 1)
155+
dbt test --target "$DBT_TARGET" | tee -a "$LOG_FILE"
156+
test=$(grep -E 'Done\. PASS=.*TOTAL=.*' "$LOG_FILE" | tail -n 1)
157+
fi
158+
echo "Done with $repo_name"
159+
echo ""
160+
161+
deactivate
162+
cd "$SCRIPT_DIR"
163+
else
164+
echo "Skipping $repo_name — no integration_tests directory found"
165+
fi
166+
167+
echo ""
168+
done < "$REPOS_FILE"
169+
170+
echo "dbt seed - $seed"
171+
echo "dbt run - $run"
172+
echo "dbt test - $test"
173+
174+

0 commit comments

Comments
 (0)