Skip to content

Commit e95d4ee

Browse files
whummerwhummer
authored
add initial test setup running against LocalStack Snowflake (#1)
1 parent 9439854 commit e95d4ee

12 files changed

+433
-365
lines changed

‎.github/workflows/build_test.yml

+246-227
Large diffs are not rendered by default.

‎.github/workflows/changelog.yml

+20-20
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
1-
name: Changelog Check
2-
3-
on:
4-
pull_request:
5-
types: [opened, synchronize, labeled, unlabeled]
6-
branches:
7-
- main
8-
9-
jobs:
10-
check_change_log:
11-
runs-on: ubuntu-latest
12-
if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-CHANGELOG-UPDATES')}}
13-
steps:
14-
- name: Checkout
15-
uses: actions/checkout@v3
16-
with:
17-
fetch-depth: 0
18-
19-
- name: Ensure DESCRIPTION.md is updated
20-
run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "DESCRIPTION.md"
1+
#name: Changelog Check
2+
#
3+
#on:
4+
# pull_request:
5+
# types: [opened, synchronize, labeled, unlabeled]
6+
# branches:
7+
# - main
8+
#
9+
#jobs:
10+
# check_change_log:
11+
# runs-on: ubuntu-latest
12+
# if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-CHANGELOG-UPDATES')}}
13+
# steps:
14+
# - name: Checkout
15+
# uses: actions/checkout@v3
16+
# with:
17+
# fetch-depth: 0
18+
#
19+
# - name: Ensure DESCRIPTION.md is updated
20+
# run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "DESCRIPTION.md"

‎.github/workflows/cla_bot.yml

+29-29
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,29 @@
1-
name: "CLA Assistant"
2-
on:
3-
issue_comment:
4-
types: [created]
5-
pull_request_target:
6-
types: [opened,closed,synchronize]
7-
8-
jobs:
9-
CLAssistant:
10-
runs-on: ubuntu-latest
11-
permissions:
12-
actions: write
13-
contents: write
14-
pull-requests: write
15-
statuses: write
16-
steps:
17-
- name: "CLA Assistant"
18-
if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
19-
uses: contributor-assistant/github-action/@master
20-
env:
21-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
22-
PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
23-
with:
24-
path-to-signatures: 'signatures/version1.json'
25-
path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md'
26-
branch: 'main'
27-
allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa'
28-
remote-organization-name: 'snowflakedb'
29-
remote-repository-name: 'cla-db'
1+
#name: "CLA Assistant"
2+
#on:
3+
# issue_comment:
4+
# types: [created]
5+
# pull_request_target:
6+
# types: [opened,closed,synchronize]
7+
#
8+
#jobs:
9+
# CLAssistant:
10+
# runs-on: ubuntu-latest
11+
# permissions:
12+
# actions: write
13+
# contents: write
14+
# pull-requests: write
15+
# statuses: write
16+
# steps:
17+
# - name: "CLA Assistant"
18+
# if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
19+
# uses: contributor-assistant/github-action/@master
20+
# env:
21+
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
22+
# PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
23+
# with:
24+
# path-to-signatures: 'signatures/version1.json'
25+
# path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md'
26+
# branch: 'main'
27+
# allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa'
28+
# remote-organization-name: 'snowflakedb'
29+
# remote-repository-name: 'cla-db'

‎.github/workflows/semgrep.yml

+15-15
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
---
2-
name: Run semgrep checks
3-
4-
on:
5-
pull_request:
6-
branches: [main]
7-
8-
permissions:
9-
contents: read
10-
11-
jobs:
12-
run-semgrep-reusable-workflow:
13-
uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main
14-
secrets:
15-
token: ${{ secrets.SEMGREP_APP_TOKEN }}
1+
#---
2+
#name: Run semgrep checks
3+
#
4+
#on:
5+
# pull_request:
6+
# branches: [main]
7+
#
8+
#permissions:
9+
# contents: read
10+
#
11+
#jobs:
12+
# run-semgrep-reusable-workflow:
13+
# uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main
14+
# secrets:
15+
# token: ${{ secrets.SEMGREP_APP_TOKEN }}

‎.github/workflows/snyk-issue.yml

+33-33
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,33 @@
1-
name: Snyk Issue
2-
3-
on:
4-
schedule:
5-
- cron: '* */12 * * *'
6-
7-
permissions:
8-
contents: read
9-
issues: write
10-
pull-requests: write
11-
12-
concurrency: snyk-issue
13-
14-
jobs:
15-
snyk:
16-
runs-on: ubuntu-latest
17-
steps:
18-
- name: Checkout Action
19-
uses: actions/checkout@v3
20-
with:
21-
repository: snowflakedb/whitesource-actions
22-
token: ${{ secrets.whitesource_action_token }}
23-
path: whitesource-actions
24-
- name: Set Env
25-
run: echo "repo=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV
26-
- name: Jira Creation
27-
uses: ./whitesource-actions/snyk-issue
28-
with:
29-
snyk_org: ${{ secrets.snyk_org_id_public_repo }}
30-
snyk_token: ${{ secrets.snyk_github_integration_token_public_repo }}
31-
jira_token: ${{ secrets.jira_token_public_repo }}
32-
env:
33-
gh_token: ${{ secrets.github_token }}
1+
#name: Snyk Issue
2+
#
3+
#on:
4+
# schedule:
5+
# - cron: '* */12 * * *'
6+
#
7+
#permissions:
8+
# contents: read
9+
# issues: write
10+
# pull-requests: write
11+
#
12+
#concurrency: snyk-issue
13+
#
14+
#jobs:
15+
# snyk:
16+
# runs-on: ubuntu-latest
17+
# steps:
18+
# - name: Checkout Action
19+
# uses: actions/checkout@v3
20+
# with:
21+
# repository: snowflakedb/whitesource-actions
22+
# token: ${{ secrets.whitesource_action_token }}
23+
# path: whitesource-actions
24+
# - name: Set Env
25+
# run: echo "repo=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV
26+
# - name: Jira Creation
27+
# uses: ./whitesource-actions/snyk-issue
28+
# with:
29+
# snyk_org: ${{ secrets.snyk_org_id_public_repo }}
30+
# snyk_token: ${{ secrets.snyk_github_integration_token_public_repo }}
31+
# jira_token: ${{ secrets.jira_token_public_repo }}
32+
# env:
33+
# gh_token: ${{ secrets.github_token }}

‎.github/workflows/snyk-pr.yml

+37-37
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,37 @@
1-
name: Snyk PR
2-
on:
3-
pull_request:
4-
branches:
5-
- main
6-
7-
permissions:
8-
contents: read
9-
issues: write
10-
pull-requests: write
11-
12-
jobs:
13-
snyk:
14-
runs-on: ubuntu-latest
15-
if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }}
16-
steps:
17-
- name: Checkout
18-
uses: actions/checkout@v3
19-
with:
20-
ref: ${{ github.event.pull_request.head.ref }}
21-
fetch-depth: 0
22-
23-
- name: Checkout Action
24-
uses: actions/checkout@v3
25-
with:
26-
repository: snowflakedb/whitesource-actions
27-
token: ${{ secrets.whitesource_action_token }}
28-
path: whitesource-actions
29-
30-
- name: Snyk Pull Request Scan Check
31-
uses: ./whitesource-actions/snyk-pr
32-
env:
33-
pr_title: ${{ github.event.pull_request.title }}
34-
with:
35-
jira_token: ${{ secrets.jira_token_public_repo }}
36-
gh_token: ${{ secrets.github_token }}
37-
amend: false
1+
#name: Snyk PR
2+
#on:
3+
# pull_request:
4+
# branches:
5+
# - main
6+
#
7+
#permissions:
8+
# contents: read
9+
# issues: write
10+
# pull-requests: write
11+
#
12+
#jobs:
13+
# snyk:
14+
# runs-on: ubuntu-latest
15+
# if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }}
16+
# steps:
17+
# - name: Checkout
18+
# uses: actions/checkout@v3
19+
# with:
20+
# ref: ${{ github.event.pull_request.head.ref }}
21+
# fetch-depth: 0
22+
#
23+
# - name: Checkout Action
24+
# uses: actions/checkout@v3
25+
# with:
26+
# repository: snowflakedb/whitesource-actions
27+
# token: ${{ secrets.whitesource_action_token }}
28+
# path: whitesource-actions
29+
#
30+
# - name: Snyk Pull Request Scan Check
31+
# uses: ./whitesource-actions/snyk-pr
32+
# env:
33+
# pr_title: ${{ github.event.pull_request.title }}
34+
# with:
35+
# jira_token: ${{ secrets.jira_token_public_repo }}
36+
# gh_token: ${{ secrets.github_token }}
37+
# amend: false

‎setup.cfg

+1
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ console_scripts =
7979

8080
[options.extras_require]
8181
development =
82+
boto3
8283
Cython
8384
coverage
8485
more-itertools

‎test/integ/conftest.py

+29
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,35 @@ def print_help() -> None:
8383
)
8484

8585

86+
# START changes for LocalStack
87+
88+
89+
@pytest.fixture(scope="session", autouse=True)
90+
def create_external_volume():
91+
import boto3
92+
93+
# create S3 bucket in LocalStack
94+
s3_client = boto3.client(
95+
"s3",
96+
endpoint_url="http://localhost:4566",
97+
aws_access_key_id="test",
98+
aws_secret_access_key="test",
99+
)
100+
s3_client.create_bucket(Bucket="test")
101+
102+
# create a local external volume which is required by some of the tests
103+
query = (
104+
"CREATE EXTERNAL VOLUME IF NOT EXISTS python_connector_iceberg_exvol STORAGE_LOCATIONS = "
105+
"((NAME = 'sl1' STORAGE_PROVIDER='S3' STORAGE_BASE_URL='s3://test' "
106+
"STORAGE_AWS_ROLE_ARN='arn:aws:iam::000000000000:role/test'))"
107+
)
108+
connection = create_connection("default")
109+
connection.cursor().execute(query)
110+
111+
112+
# END changes for LocalStack
113+
114+
86115
@pytest.fixture(scope="session")
87116
def is_public_test() -> bool:
88117
return is_public_testaccount()

‎test/integ/test_arrow_result.py

+7-3
Original file line numberDiff line numberDiff line change
@@ -172,9 +172,12 @@
172172
RUNNING_ON_GH = os.getenv("GITHUB_ACTIONS") == "true"
173173

174174
ICEBERG_SUPPORTED = CLOUD in ICEBERG_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev"
175-
STRUCTURED_TYPES_SUPPORTED = (
176-
CLOUD in STRUCTRED_TYPE_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev"
177-
)
175+
# STRUCTURED_TYPES_SUPPORTED = (
176+
# CLOUD in STRUCTRED_TYPE_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev"
177+
# )
178+
# Note whummer (2024-12-29): setting this value to False, as otherwise invalid tests are being generated,
179+
# using, e.g., `CREATE TABLE(c1 ARRAY(FLOAT))` which is invalid in standard case (verified against real SF)
180+
STRUCTURED_TYPES_SUPPORTED = False
178181

179182
# Generate all valid test cases. By using pytest.param with an id you can
180183
# run a specific test case easier like so:
@@ -937,6 +940,7 @@ def test_select_date(conn_cnx):
937940
finish(conn_cnx, table)
938941

939942

943+
@pytest.mark.skip(reason="LocalStack - currently incompatible with Snowflake emulator")
940944
@pytest.mark.parametrize("scale", range(10))
941945
@pytest.mark.parametrize("type", ["timestampntz", "timestampltz", "timestamptz"])
942946
def test_select_timestamp_with_scale(conn_cnx, scale, type):

‎test/integ/test_cursor.py

+3
Original file line numberDiff line numberDiff line change
@@ -1556,6 +1556,9 @@ def test_resultbatch(
15561556
assert total_rows == rowcount
15571557

15581558

1559+
@pytest.mark.skip(
1560+
reason="Multi-batches and large result sets currently not yet working in LocalStack"
1561+
)
15591562
@pytest.mark.skipolddriver(reason="new feature in v2.5.0")
15601563
@pytest.mark.parametrize(
15611564
"result_format,patch_path",

‎test/parameters.py

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#
2+
# Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
3+
#
4+
5+
CONNECTION_PARAMETERS = {
6+
"account": "test",
7+
"user": "test",
8+
"password": "test",
9+
"database": "test",
10+
"schema": "public",
11+
"host": "snowflake.localhost.localstack.cloud",
12+
}

‎tox.ini

+1-1
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ deps = flake8
131131
commands = flake8 {posargs}
132132

133133
[testenv:fix_lint]
134-
basepython = python3.8
134+
# basepython = python3.8
135135
description = format the code base to adhere to our styles, and complain about what we cannot do automatically
136136
passenv =
137137
PROGRAMDATA

0 commit comments

Comments
 (0)