Skip to content

Commit 5059366

Browse files
toddbirchardtoddbirchard
committed
minor tweks
1 parent 045e7f6 commit 5059366

13 files changed

+38
-159
lines changed

‎Pipfile

+1-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@ verify_ssl = true
99
SQLAlchemy = "*"
1010
PyBigQuery = "*"
1111
PyMySQL = "*"
12-
Psycopg2-Binary = "*"
1312
Loguru = "*"
1413

1514
[requires]
16-
python_version = "3.7"
15+
python_version = "3.8"

‎Pipfile.lock

+2-40
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎README.md

+4-5
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
# BigQuery SQLAlchemy Tutorial
22

3-
![Python](https://img.shields.io/badge/Python-v3.7-blue.svg?logo=python&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
4-
![Google Cloud BigQuery](https://img.shields.io/badge/Google--Cloud--BigQuery-v1.11.2-blue.svg?logo=Google&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
5-
![PyBigQuery](https://img.shields.io/badge/PyBigQuery-v0.4.11-blue.svg?logo=Google&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
6-
![SQLAlchemy](https://img.shields.io/badge/SQLAlchemy-v1.3.1-red.svg?longCache=true&style=flat-square&logo=scala&logoColor=white&colorA=4c566a&colorB=bf616a)
3+
![Python](https://img.shields.io/badge/Python-v^3.7-blue.svg?logo=python&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
4+
![Google Cloud BigQuery](https://img.shields.io/badge/Google--Cloud--BigQuery-v1.24.0-blue.svg?logo=Google&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
5+
![PyBigQuery](https://img.shields.io/badge/PyBigQuery-v0.4.13-blue.svg?logo=Google&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
6+
![SQLAlchemy](https://img.shields.io/badge/SQLAlchemy-v1.3.13-red.svg?longCache=true&style=flat-square&logo=scala&logoColor=white&colorA=4c566a&colorB=bf616a)
77
![PyMySQL](https://img.shields.io/badge/PyMySQL-v0.9.3-red.svg?longCache=true&style=flat-square&logo=mysql&logoColor=white&colorA=4c566a&colorB=bf616a)
8-
![Psycopg2-Binary](https://img.shields.io/badge/Psycopg2--Binary-v2.8.4-red.svg?longCache=true&style=flat-square&logo=postgresql&logoColor=white&colorA=4c566a&colorB=bf616a)
98
![GitHub Last Commit](https://img.shields.io/github/last-commit/google/skia.svg?style=flat-square&colorA=4c566a&colorB=a3be8c)
109
[![GitHub Issues](https://img.shields.io/github/issues/hackersandslackers/bigquery-sqlalchemy-tutorial.svg?style=flat-square&colorA=4c566a&colorB=ebcb8b)](https://github.com/hackersandslackers/bigquery-python-tutorial/issues)
1110
[![GitHub Stars](https://img.shields.io/github/stars/hackersandslackers/bigquery-sqlalchemy-tutorial.svg?style=flat-square&colorB=ebcb8b&colorA=4c566a)](https://github.com/hackersandslackers/bigquery-python-tutorial/stargazers)

‎biquery_sql_etl/__init__.py

+14-5
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,27 @@
1+
from loguru import logger
12
from biquery_sql_etl.engines import bigquery_engine, rdbms_engine
23
from biquery_sql_etl.queries import sql_queries
34
from biquery_sql_etl.client import DataClient
4-
from loguru import logger
55
from config import bigquery_table
66

77

88
logger.add('logs/queries.log', format="{time} {message}", level="INFO")
99

1010

11-
def main():
12-
"""Move data between sources."""
13-
bqc = DataClient(bigquery_engine)
14-
dbc = DataClient(rdbms_engine)
11+
def init_pipeline():
12+
"""Move data between Bigquery and MySQL."""
13+
total_rows_affected = 0
14+
bqc, dbc = data_sources()
1515
for table_name, query in sql_queries.items():
1616
fetched_rows = bqc.fetch_rows(query, table=bigquery_table)
1717
inserted_rows = dbc.insert_rows(fetched_rows, table_name, replace=True)
1818
logger.info(inserted_rows)
19+
total_rows_affected += len(fetched_rows)
20+
logger.info(f"Completed migration of {total_rows_affected} rows from BigQuery to MySQL.")
21+
22+
23+
def data_sources():
24+
"""Construct datasources."""
25+
bqc = DataClient(bigquery_engine)
26+
dbc = DataClient(rdbms_engine)
27+
return bqc, dbc

‎biquery_sql_etl/client.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
"""Base Data Client."""
1+
"""Generic Client for interacting with data sources."""
22
from sqlalchemy import MetaData, Table
33

44

‎biquery_sql_etl/engines/bigquery.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
"""SQL Database Engine."""
2-
1+
"""BigQuery Engine."""
32
from sqlalchemy.engine import create_engine
43
from config import (gcp_credentials,
54
bigquery_uri)
65

76

8-
bigquery_engine = create_engine(bigquery_uri, credentials_path=gcp_credentials)
7+
bigquery_engine = create_engine(bigquery_uri,
8+
credentials_path=gcp_credentials)

‎biquery_sql_etl/queries.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66

77
def get_local_sql_files():
8-
"""Fetch all SQL query files in folder."""
8+
"""Fetch all .sql files from local folder."""
99
files = [local_sql_folder + '/' + f for f in listdir(local_sql_folder) if isfile(join(local_sql_folder, f)) if '.sql' in f]
1010
return files
1111

‎config.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,5 @@
1515
rdbms_name = environ.get('DATABASE_NAME')
1616
rdbms_uri = f'mysql+pymysql://{rdbms_user}:{rdbms_pass}@{rdbms_host}:{rdbms_port}/{rdbms_name}'
1717

18-
# Local
18+
# Locally stored queries
1919
local_sql_folder = 'sql'

‎main.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""Application entry point."""
2-
from biquery_sql_etl import main
2+
from biquery_sql_etl import init_pipeline
33

44
if __name__ == '__main__':
5-
main()
5+
init_pipeline()

‎poetry.lock

+2-90
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml

+4-5
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
[tool.poetry]
22
name = "biquery_sql_etl"
3-
version = "0.1.0"
4-
description = "Simple Python project built with Poetry."
3+
version = "1.0.0"
4+
description = "Light ETL script to migrate data from BigQuery to SQL and vice-versa."
55
authors = ["Todd Birchard <[email protected]>"]
66
maintainers = ["Todd Birchard <[email protected]>"]
77
license = "MIT"
88
readme = "README.md"
9-
homepage = ""
9+
homepage = "https://hackersandslackers.com/bigquery-and-sql-databases/"
1010
repository = "https://github.com/hackersandslackers/bigquery-sqlalchemy-tutorial/"
1111
documentation = "https://hackersandslackers.com/bigquery-and-sql-databases/"
1212
keywords = ["BigQuery",
@@ -16,11 +16,10 @@ keywords = ["BigQuery",
1616
"Google Cloud"]
1717

1818
[tool.poetry.dependencies]
19-
Python = "^3.7"
19+
Python = "^3.8"
2020
SQLAlchemy = "*"
2121
PyBigQuery = "*"
2222
PyMySQL = "*"
23-
Psycopg2-Binary = "*"
2423
Loguru = "*"
2524

2625
[tool.poetry.dev-dependencies]

‎requirements.txt

-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
bigquery-sqlalchemy-tutorial==1.0.0
21
cachetools==4.0.0
32
certifi==2019.11.28
43
chardet==3.0.4
@@ -12,7 +11,6 @@ googleapis-common-protos==1.51.0
1211
idna==2.8
1312
loguru==0.4.1
1413
protobuf==3.11.3
15-
psycopg2-binary==2.8.4
1614
pyasn1==0.4.8
1715
pyasn1-modules==0.2.8
1816
pybigquery==0.4.13

‎setup.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,14 @@
2222
'Development Status :: 3 - Alpha',
2323
'Intended Audience :: Developers',
2424
'Programming Language :: Python :: 3.7',
25+
'Programming Language :: Python :: 3.8',
2526
],
26-
keywords='SQLAlchemy Google BigQuery SQL ETL',
27+
keywords='SQLAlchemy GCP Google BigQuery SQL ETL RDBMS',
2728
packages=find_packages(),
2829
install_requires=['SQLAlchemy',
2930
'PyBigQuery',
3031
'PyMySQL',
31-
'Psycopg2-Binary'],
32+
'Loguru'],
3233
extras_require={
3334
'dev': ['check-manifest'],
3435
'test': ['coverage'],

0 commit comments

Comments
 (0)