Skip to content

Commit 138b9c0

Browse files
committed
Merge branch 'master' into activity-creation
2 parents 29a70a1 + 81a6269 commit 138b9c0

File tree

192 files changed

+25181
-20772
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

192 files changed

+25181
-20772
lines changed

.github/actions/comment_action/action.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ inputs:
1919
runs:
2020
using: "composite"
2121
steps:
22-
- uses: actions/github-script@v5
22+
- uses: actions/github-script@v7
2323
with:
2424
github-token: ${{ inputs.token }}
2525
script: |

.github/workflows/build_deploy_frontend.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ jobs:
3737

3838
- uses: unstructuredstudio/zubhub/.github/actions/scp_action@master
3939
with:
40-
host: ${{ vars.DO_BACKEND_HOST }}
41-
username: ${{ vars.DO_BACKEND_USERNAME }}
40+
host: ${{ vars.DO_FRONTEND_HOST }}
41+
username: ${{ vars.DO_FRONTEND_USERNAME }}
4242
key: ${{ secrets.DO_SSHKEY }}
4343
source: "."
4444
target: "/home/zubhub-frontend/zubhub"
@@ -65,7 +65,7 @@ jobs:
6565
script: |
6666
6767
docker system prune -a -f
68-
68+
6969
volumes=$(docker volume ls -q)
7070
# Loop through the volumes
7171
while IFS= read -r volume; do

.github/workflows/create_destroy_test_vm.yaml

+5-3
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ jobs:
3838
- name: Get checkout commit ref and repo
3939
run: |
4040
41-
# for context read this https://github.com/Arelle/Arelle/pull/938/files#r1377926250
41+
# for context read this https://github.com/Arelle/Arelle/pull/938/files#r1377926250
4242
merge_commit_sha=${{ github.event.pull_request.merge_commit_sha }}
4343
head_ref=${{ github.event.pull_request.head.ref }}
4444
target_repo=${{ github.repository }}
@@ -70,7 +70,7 @@ jobs:
7070
script: |
7171
7272
# This script will be executed inside a composite action.
73-
# rename GITHUB_OUTPUT so that it's clear that it's a
73+
# rename GITHUB_OUTPUT so that it's clear that it's a
7474
# composite output and should be handled differently from $GITHUB_OUTPUT
7575
COMPOSITE_OUTPUT=$GITHUB_OUTPUT
7676
NEW_DROPLET_NAME=zubhub-test-${{ github.event.pull_request.number }}
@@ -279,6 +279,8 @@ jobs:
279279
Test VM is ready ✅✅✅
280280
You can access it with the url:
281281
https://${{env.FRONTEND_DOMAIN}}.unstructured.studio
282+
Default username: dummy
283+
default password: dummy_password
282284
######################################################################################
283285

284286
#################################################################################
@@ -304,7 +306,7 @@ jobs:
304306
script: |
305307
306308
# This script will be executed inside a composite action.
307-
# rename GITHUB_OUTPUT so that it's clear that it's a
309+
# rename GITHUB_OUTPUT so that it's clear that it's a
308310
# composite output and should be handled differently from $GITHUB_OUTPUT
309311
COMPOSITE_OUTPUT=$GITHUB_OUTPUT
310312
# for events like unlabelled, github.event.pull_request.number is available

.github/workflows/pre_commit.yml

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
name: pre-commit
2+
on:
3+
pull_request:
4+
5+
jobs:
6+
pre-commit:
7+
runs-on: ubuntu-latest
8+
steps:
9+
- name: Set up Docker Buildx
10+
uses: docker/setup-buildx-action@v3
11+
12+
- name: Set up Docker Compose
13+
run: |
14+
sudo curl -L "https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-$(uname -s)-$(uname -m)" \
15+
-o /usr/local/bin/docker-compose
16+
sudo chmod +x /usr/local/bin/docker-compose
17+
18+
- name: Get number of PR commits + 1
19+
run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> $GITHUB_ENV
20+
21+
- uses: unstructuredstudio/zubhub/.github/actions/checkout@master
22+
with:
23+
ref: ${{ github.head_ref }}
24+
fetch_depth: ${{ env.PR_FETCH_DEPTH }}
25+
26+
- name: Undo all PR commits and keep changes in staging area
27+
run: |
28+
29+
# some of our pre-commit hooks uses `git diff --name-only --cached` to get the list
30+
# changed files. This step undos the most recent commits in the checkout,
31+
# leaving the changes in the staging area for our `git diff ..` to find.
32+
git reset --soft HEAD~$(( ${{ env.PR_FETCH_DEPTH }} - 1 ))
33+
34+
- run: |
35+
36+
# Running pre-commit on the following files:
37+
target_files=$(git diff --name-only --cached)
38+
echo "Running pre-commit on the following files:"
39+
echo "$target_files"
40+
41+
- uses: actions/setup-python@v3
42+
43+
- uses: actions/setup-node@v4
44+
with:
45+
node-version: 16
46+
cache: 'npm'
47+
cache-dependency-path: zubhub_frontend/zubhub/package-lock.json
48+
49+
- run: npm ci --prefix ./zubhub_frontend/zubhub/
50+
51+
- uses: pre-commit/[email protected]
52+
id: pre_commit
53+
with:
54+
# by default pre-commit/action runs `pre-commit run --all-files ...`.
55+
# Here we use empty extra_args to force it to run `pre-commit run ...` instead.
56+
# this is important because we want it to only run on the files that have changed
57+
extra_args:
58+
continue-on-error: true
59+
60+
- name: Success comment
61+
if: steps.pre_commit.outcome == 'success'
62+
uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
63+
with:
64+
token: ${{ secrets.GITHUB_TOKEN }}
65+
issue_number: ${{ github.event.pull_request.number }}
66+
owner: ${{ github.repository_owner }}
67+
repo: ${{ github.event.repository.name }}
68+
message: |
69+
All pre-commit checks passed ✅✅✅
70+
71+
- name: Failure comment
72+
if: steps.pre_commit.outcome != 'success'
73+
uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
74+
with:
75+
token: ${{ secrets.GITHUB_TOKEN }}
76+
issue_number: ${{ github.event.pull_request.number }}
77+
owner: ${{ github.repository_owner }}
78+
repo: ${{ github.event.repository.name }}
79+
message: |
80+
pre-commit checks failed ❌❌❌
81+
make sure you have pre-commit set up locally (see the README.md for instructions).
82+
If you have set up pre-commit locally, you may need to undo this commit and
83+
run "pre-commit" locally to see what the errors are.
84+
85+
- name: Failure exit
86+
if: steps.pre_commit.outcome != 'success'
87+
run: exit 1

.pre-commit-config.yaml

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
default_language_version:
2+
python: python3
3+
4+
repos:
5+
- repo: https://github.com/psf/black
6+
rev: 23.1.0
7+
hooks:
8+
- id: black
9+
args: ["-l 88"]
10+
files: zubhub_backend/.*\.py$
11+
12+
- repo: https://github.com/PyCQA/isort
13+
rev: 5.11.5 # do not update this until our CI runner uses Python >=3.8.1
14+
hooks:
15+
- id: isort
16+
files: zubhub_backend/.*\.py$
17+
18+
- repo: https://github.com/PyCQA/flake8
19+
rev: 5.0.4 # do not update this until our CI runner uses Python >=3.8.1
20+
hooks:
21+
- id: flake8
22+
# use same max-line-length as black
23+
# E121, E123, E126, E133, E226, E241, E242, E704 and W503 are default ignores
24+
# E124: closing bracket does not match visual indentation
25+
# E203: non-pep8 compliant brainfart of flake8
26+
args:
27+
[
28+
"--max-line-length=88",
29+
"--ignore=E121,E123,E126,E133,E203,E226,E241,E242,E704,W503,E124",
30+
]
31+
files: zubhub_backend/.*\.py$
32+
33+
# - repo: https://github.com/pre-commit/mirrors-mypy
34+
# rev: v1.0.0
35+
# hooks:
36+
# - id: mypy
37+
# exclude: ^tests/
38+
# verbose: true
39+
# args: [--show-error-codes, --ignore-missing-imports]
40+
# additional_dependencies:
41+
# ["types-requests", "types-PyYAML", "types-flask", "toolforge-weld"]
42+
43+
- repo: https://github.com/pre-commit/pre-commit-hooks
44+
rev: v4.4.0
45+
hooks:
46+
- id: trailing-whitespace
47+
- id: end-of-file-fixer
48+
- id: check-yaml
49+
- id: check-shebang-scripts-are-executable
50+
- id: check-executables-have-shebangs
51+
- id: check-merge-conflict
52+
53+
- repo: https://github.com/pre-commit/mirrors-eslint
54+
rev: v7.32.0
55+
hooks:
56+
- id: eslint
57+
entry: ./zubhub_frontend/zubhub/pre-commit-eslint
58+
pass_filenames: false
59+
always_run: true
60+
61+
- repo: https://github.com/pre-commit/mirrors-prettier
62+
rev: v3.1.0
63+
hooks:
64+
- id: prettier
65+
files: zubhub_frontend/.*\.[jt]sx?$
66+
exclude: ^zubhub_frontend/zubhub/node_modules/
67+
args: ["--config", "./zubhub_frontend/zubhub/.prettierrc.yaml"]
68+
69+
- repo: local
70+
hooks:
71+
- id: zubhub-test
72+
name: zubhub-test
73+
entry: ./zubhub_backend/pre-commit-zubhub-test
74+
language: system
75+
files: zubhub_backend/zubhub/.*\.py$
76+
pass_filenames: false
77+
78+
- id: media-test
79+
name: media-test
80+
entry: ./zubhub_backend/pre-commit-media-test
81+
language: system
82+
files: zubhub_backend/media/.*\.py$
83+
pass_filenames: false
84+
85+
- id: react-test
86+
name: react-test
87+
entry: npm --prefix ./zubhub_frontend/zubhub/ run test -- --coverage --watchAll=false
88+
language: node
89+
files: zubhub_frontend/.*\.[jt]sx?$
90+
exclude: ^zubhub_frontend/zubhub/node_modules/
91+
pass_filenames: false

README.md

+26
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Follow the instructions below to set up your local development environment
3030
6. Install make. ( [jump to section](#install-make) )
3131
7. Setup Backend. ( [jump to section](#setup-backend) )
3232
8. Setup Frontend. ( [jump to section](#setup-frontend) )
33+
9. Tests
34+
10. Making changes and Commiting
3335

3436
<br/>
3537
<br/>
@@ -202,6 +204,30 @@ Visit **localhost:3000** on your browser to access the frontend.
202204
<br/>
203205
<br/>
204206

207+
# Tests
208+
Before you get started making changes and commiting, you should setup pre-commit on your development machine:
209+
- Run `pip install pre-commit`
210+
- Run `pre-commit --version` to verify that pre-commit installation was successful
211+
- Run `cd <path to cloned zubhub repository>`
212+
- Run `pre-commit install` to install the git hook scripts
213+
214+
To verify that your setup was successful, run `pre-commit` in the root of your cloned zubhub repo. You should see this running a number of checks.
215+
216+
## Frontend
217+
You can also test the frontend seperately by going to the root of the frontend project (the place you have package.json) and running `npm run test -- --coverage --watchAll=false`
218+
219+
## Backend
220+
To test the backend seperately,
221+
- Run `cd ./zubhub_backend`
222+
- Run `make zubhub-test` to run the test suites of the web django app
223+
- Run `make media-test` to run the test suites of the media django app
224+
225+
# Making changes and Commiting
226+
Before making changes, make sure that you've set up pre-commit as described in the previous step.
227+
After that just `git add .` and `git commit` your changes. pre-commit will automatically run the different tests and will fail to commit until you fix all the errors.
228+
229+
> **_NOTE:_** If you fail to setup pre-commit on your local machine before making pull requests, our pre-commit action on github will run the same tests on your code and if it fails to pass, your code won't be merged.
230+
205231
# Deployment
206232

207233
ZubHub is currently deployed on its main website using Github Actions that act as our build and deployment tooling. If you are interested in deploying Zubhub on your VM for testing and hosting purposes, follow the [Single VM Deployment](./single_vm_deployment/DEPLOYMENT.md) instructions.

zubhub_backend/.gitignore

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
media_store/
2-
media/
2+
zubhub/media/
33
# migrations/
44
__pycache__/
55
*.pyc
@@ -11,3 +11,5 @@ cadvisor.htpasswd
1111
.ssl-data
1212
celerybeat-schedule
1313
celerybeat.pid
14+
venv/
15+
.idea/

zubhub_backend/Makefile

+6-2
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,13 @@ migrate: ## Run `manage.py migrate`
8686
docker-compose -f docker-compose.yml exec web bash -c "python zubhub/manage.py migrate"
8787
.PHONY: migrate
8888

89-
test: ## Run `manage.py test`
89+
zubhub-test: ## Run `manage.py test` on the web container
9090
docker-compose -f docker-compose.yml exec web bash -c "cd zubhub && python manage.py test"
91-
.PHONY: test
91+
.PHONY: zubhub-test
92+
93+
media-test: ## Run `manage.py test` on the media container
94+
docker-compose -f docker-compose.yml exec media bash -c "cd media && python manage.py test"
95+
.PHONY: media-test
9296

9397
admin-user:
9498
docker-compose -f docker-compose.yml exec web bash -c "python zubhub/manage.py create_dummy_admin_user"

zubhub_backend/compose/web/dev/start

100644100755
+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ python /zubhub_backend/zubhub/manage.py collectstatic --noinput
1212
#python /zubhub_backend/zubhub/manage.py makemigrations creators
1313
#python /zubhub_backend/zubhub/manage.py makemigrations projects
1414
#python /zubhub_backend/zubhub/manage.py makemigrations notifications
15-
#python /zubhub_backend/zubhub/manage.py migrate
15+
python /zubhub_backend/zubhub/manage.py migrate
1616

1717
python /zubhub_backend/zubhub/manage.py createcachetable
1818

0 commit comments

Comments
 (0)