Release 2.0.0 (#220)

* fixed sorting issue, pagination issue still seen

* fixed server table and authentication stuff

* added better admin settings

* added redis UI and python linting part 1

* flake8 linted

* added google analytics

* ready to deploy

* ready

* flip flag on

Author: amthorn

.github/workflows/release.yml

@@ -30,7 +30,7 @@ jobs:
           docker login -u publisher -p ${DOCKER_TOKEN} ghcr.io
           
           # Build all containers
-          QUTEX_VERSION=$(echo ${GITHUB_REF##*/}) docker compose -f docker-compose.build.yml build
-          QUTEX_VERSION=$(echo ${GITHUB_REF##*/}) docker compose -f docker-compose.build.yml push
+          QUTEX_VERSION=$(echo ${GITHUB_REF##*/}) docker compose -f docker-compose.yml -f docker-compose.build.yml build
+          QUTEX_VERSION=$(echo ${GITHUB_REF##*/}) docker compose -f docker-compose.yml -f docker-compose.build.yml push
         env:
           DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -19,3 +19,6 @@ docs/.sass-cache
 docs/.jekyll-cache
 docs/.jekyll-metadata
 docs/vendor
+
+# App stuff
+services/ui/public/env.js

.local.env

@@ -3,6 +3,7 @@ AUTH_SERVICE_HOST=http://auth:4000
 FLASK_ENV=development
 FQDN=http://localhost
 NODE_ENV=development
+SUPER_ADMINS=["Y2lzY29zcGFyazovL3VzL1BFT1BMRS9kODRkZjI1MS1iYmY3LTRlZTEtOTM1OS00Y2I0MGIyOTBhN2I"]
 
 # UI
 DANGEROUSLY_DISABLE_HOST_CHECK=true

.production.env

@@ -12,4 +12,4 @@ MONGO_INITDB_DATABASE=qutex
 MONGO_INITDB_ROOT_PASSWORD_FILE=/run/secrets/mongoPassword
 
 # FEATURE FLAGS
-BOT_2_0_0=false
+BOT_2_0_0=true

Dockerfile.mongoexpress

@@ -0,0 +1,3 @@
+FROM mongo-express
+
+RUN sed -i 's/req.adminDb = mongo.adminDb;/req.adminDb = mongo.mainClient.adminDb || undefined;/g' lib/router.js

Makefile

@@ -1,10 +1,10 @@
 .PHONY: build $(SERVICE)
 build:
-	docker compose -f docker-compose.yml -f docker-compose.dev.yml build $(SERVICE)
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.build.yml build $(SERVICE)
 
-.PHONY: up
+.PHONY: up $(SERVICE)
 up:
-	docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build -d
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.build.yml  up --build $(SERVICE) -d
 
 .PHONY: deploy $(VERSION)
 deploy:
@@ -18,15 +18,24 @@ logs:
 down:
 	docker compose down
 
-.PHONY: test
-test:
+.PHONY: test-qutex
+test-qutex:
 	export QUTEX_TESTING=true && \
 	yarn --cwd services/bot test --verbose && \
 	unset QUTEX_TESTING
 
+.PHONY: test-nginx
+test-nginx:
+	pytest services/nginx/tests
+
+.PHONY: test
+test:
+	${MAKE} test-qutex
+	${MAKE} test-nginx
+
 .PHONY: lint
 lint:
 	yarn --cwd services/bot lint
 	yarn --cwd services/ui lint
-	docker run -it -v $(PWD)services:/apps/services alpine/flake8 /apps
+	flake8 services
 

docker-compose.build.yml

@@ -1,35 +1,36 @@
 version: '3.9'
 services:
     nginx:
-        image: ghcr.io/amthorn/qutex/qutex_nginx:${QUTEX_VERSION:-latest}
         build:
             context: services/nginx
     bot:
-        image: ghcr.io/amthorn/qutex/qutex_bot:${QUTEX_VERSION:-latest}
         build:
             context: ./services/bot
     ui:
-        image: ghcr.io/amthorn/qutex/qutex_ui:${QUTEX_VERSION:-latest}
         build:
             context: services/ui/
     projects:
-        image: ghcr.io/amthorn/qutex/qutex_projects:${QUTEX_VERSION:-latest}
         build:
             context: .
             dockerfile: services/_api_service_template/Dockerfile
             args:
                 SERVICE_PREFIX: projects
     users:
-        image: ghcr.io/amthorn/qutex/qutex_users:${QUTEX_VERSION:-latest}
         build:
             context: .
             dockerfile: services/_api_service_template/Dockerfile
             args:
                 SERVICE_PREFIX: users
     auth:
-        image: ghcr.io/amthorn/qutex/qutex_auth:${QUTEX_VERSION:-latest}
         build:
             context: .
             dockerfile: services/_api_service_template/Dockerfile
             args:
-                SERVICE_PREFIX: auth
+                SERVICE_PREFIX: auth
+    mongo_ui:
+        build:
+            context: .
+            dockerfile: Dockerfile.mongoexpress
+    mongo_backup:
+        build:
+            context: services/mongo_backup

docker-compose.dev.yml

@@ -8,13 +8,13 @@ x-interactive: &interactive
 services:
     nginx:
         image: qutex_nginx:latest
-        build:
-            context: services/nginx
         environment:
-            CERTBOT_EMAIL: [email protected]
-            STAGING: "1"
-            DEBUG: "1"
-            RENEWAL_INTERVAL: 8d
+            STAGING: 1
+            # DEBUG: 1
+        volumes:
+            - ./services/nginx/dev_default.conf:/etc/nginx/user_conf.d/default.conf
+        ports:
+            - 80:80
     bot:
         <<: *interactive
         image: qutex_bot:latest
@@ -79,23 +79,14 @@ services:
         build:
             context: ./services/bot
         env_file: *env_files
+        volumes: 
+            - ./services/bot:/app
+    mongo_backup:
+        image: qutex_mongo_backup:latest
     mongo:
         env_file: *env_files
         ports:
             - 27017:27017
-    ####################
-    ## -- DEV ONLY -- ## ( FOR NOW )
-    ####################
-    mongo_ui:
-        image: mongo-express:latest
-        environment:
-            ME_CONFIG_MONGODB_SERVER: mongo
-            ME_CONFIG_MONGODB_ADMINUSERNAME: root
-            ME_CONFIG_MONGODB_ADMINPASSWORD_FILE: /run/secrets/mongoPassword
-        ports:
-            - 8081:8081
-        secrets:
-            - mongoPassword
 volumes:
     # ignore all css from the docker container and do not mount to my local dir
     # This is because the CSS files shouldn't be modified. Only the sass files

docker-compose.yml

@@ -95,6 +95,43 @@ services:
             - mongo_volume:/data/db
         secrets:
             - mongoPassword
+    mongo_ui:
+        image: mongo-express:1.0.0-alpha
+        depends_on:
+            - mongo
+        environment:
+            ME_CONFIG_MONGODB_SERVER: mongo
+            ME_CONFIG_MONGODB_ADMINUSERNAME: root
+            ME_CONFIG_MONGODB_ADMINPASSWORD_FILE: /run/secrets/mongoPassword
+            ME_CONFIG_SITE_BASEURL: /admin/mongo/
+            ME_CONFIG_OPTIONS_NO_DELETE: "true"
+            ME_CONFIG_OPTIONS_READONLY: "true"
+            ME_CONFIG_MONGODB_ENABLE_ADMIN: "true"
+        secrets:
+            - mongoPassword
+    mongo_backup:
+        image: ghcr.io/amthorn/qutex/qutex_mongo_backup:${QUTEX_VERSION:-latest}
+        environment:
+            MONGODB_HOST: mongo
+            MONGODB_PORT: 27017
+            MONGODB_USER: root
+            MONGODB_PASS_FILE: /run/secrets/mongoPassword
+            CRON_TIME: 0 0 * * *
+            EXTRA_OPTS: --authenticationDatabase admin --db qutex --gzip --forceTableScan
+            MAX_BACKUPS: 14
+            INIT_BACKUP: 1
+        volumes: 
+            - ./mongo_backups:/backup
+        secrets:
+            - mongoPassword
+    redis_ui:
+        image: rediscommander/redis-commander
+        environment:
+            REDIS_HOST: redis
+            REDIS_PORT: "6379"
+            URL_PREFIX: /admin/redis
+        ports:
+            - 8081:8081
 secrets:
     token:
         file: secrets/prod/token

package.json

@@ -1,6 +1,7 @@
 {
   "dependencies": {
     "eslint": "^7.32.0",
-    "qutex_web": "file:services/ui"
+    "qutex_web": "file:services/ui",
+    "react-dotenv": "^0.1.3"
   }
 }

services/_api_service_template/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-alpine
+FROM python:3.10-alpine
 LABEL MAINTAINER="Ava Thorn" EMAIL="[email protected]"
 ARG SERVICE_PREFIX
 

services/_api_service_template/src/main.py

@@ -13,6 +13,7 @@
 app.config['SERVICE_PREFIX'] = os.environ.get('SERVICE_PREFIX')
 app.config['AUTH_SERVICE_TOKEN_CHECK_ROUTE'] = os.environ['AUTH_SERVICE_TOKEN_CHECK_ROUTE']
 app.config['AUTH_SERVICE_HOST'] = os.environ['AUTH_SERVICE_HOST']
+app.config['SUPER_ADMINS'] = json.loads(os.environ['SUPER_ADMINS'])
 app.config['TOKEN_COOKIE_NAME'] = 'qutexToken'
 app.config['FQDN'] = os.environ.get('FQDN', 'http://localhost')
 app.config['DEFAULT_PAGE_LENGTH'] = 50
@@ -22,11 +23,12 @@
 
 
 class CustomJSONEncoder(flask.json.JSONEncoder):
-    def default(self, o):
+    def default(self, o: object) -> str:
         if isinstance(o, ObjectId):
             return str(o)
         return super().default(o)
 
+
 app.config["RESTX_JSON"] = {"cls": CustomJSONEncoder}
 
 ########################
@@ -35,34 +37,33 @@ def default(self, o):
 try:
     with open('/run/secrets/privateKey') as f:
         app.secret_key = f.read()
-except Exception as e:
+except Exception:
     print("WARNING: No privateKey secret provided for Flask application")
 
 try:
     with open('/run/secrets/token') as f:
         app.config['WEBEX_TEAMS_ACCESS_TOKEN'] = f.read().strip()
-except Exception as e:
+except Exception:
     print("WARNING: No webex teams access token provided for Flask application")
 
 try:
     with open('/run/secrets/mongoPassword') as f:
         app.config['MONGO_PASSWORD'] = f.read()
-except Exception as e:
+except Exception:
     print("WARNING:  No mongo password provided for Flask application")
 
 ##########
 # MODELS #
 ##########
-import setup_db
+import setup_db  # noqa
 
 ##########
 #  APIS  #
 ##########
 from setup_api import v1  # noqa
 
-
-import documents
-import api
+import documents  # noqa
+import api  # noqa
 
 
 @v1.errorhandler(Exception)
@@ -73,16 +74,24 @@ def handle_exception(e: Exception) -> flask.Response:
         return dump_data(e, flask.make_response(), e.messages, 422)
     elif isinstance(e, werkzeug.exceptions.Unauthorized):
         response = dump_data(e, e.get_response(), {e.name: e.description}, e.code)
-        return response[0], response[1], {**response[2], 'Set-Cookie': f'{app.config.get("TOKEN_COOKIE_NAME")}=deleted; path=/; expires=Thu, 01 Jan 1970 00:00:00 GMT'}
+        return response[0], response[1], {
+            **response[2],
+            'Set-Cookie': app.config.get("TOKEN_COOKIE_NAME") +
+            '=deleted; path=/; expires=Thu, 01 Jan 1970 00:00:00 GMT'
+        }
     elif isinstance(e, werkzeug.exceptions.HTTPException):
         return dump_data(e, e.get_response(), {e.name: e.description}, e.code)
     return dump_data(e, flask.make_response(), {str(e.__class__.__name__): str(e)}, 500)
 
 
 @app.before_request
-def authenticate():
+def authenticate() -> None:
     # if its not an unauthenticated route
-    if not flask.request.path.startswith('/api/v1/auth/') and not flask.request.path.endswith('/healthcheck'):
+    unauthenticated = [
+        '/api/v1/auth/',
+        'healthcheck'
+    ]
+    if not any([flask.request.path.startswith(i) for i in unauthenticated]):
         # Will throw 401 if not authenticated
         result = requests.get(
             f"{app.config['AUTH_SERVICE_HOST']}/api/v1/auth/token/check",

services/_api_service_template/src/setup_db.py

@@ -1,5 +1,3 @@
-import json
-
 from app import app
 from flask_mongoengine import MongoEngine
 
@@ -10,16 +8,12 @@
 db = MongoEngine(app)
 app.db = db
 
-class SerializerMixin:
-    def as_dict(self):
-        ugly = json.loads(self.to_json())
-        
 
 class TimestampMixin():
     created_at = db.DateTimeField(auto_now_add=True, auto_now=False)
     updated_at = db.DateTimeField(auto_now_add=False, auto_now=True)
     deleted_at = db.DateTimeField(required=False)
 
 
-class BaseMixin(TimestampMixin, SerializerMixin):
+class BaseMixin(TimestampMixin):
     id = db.StringField(primary_key=True)

services/auth/api/__init__.py

@@ -1 +1 @@
-from api import v1
+from api import v1  # noqa

services/auth/api/v1/__init__.py

@@ -1,2 +1,2 @@
-from api.v1 import auth
-from api.v1 import token
+from api.v1 import auth  # noqa
+from api.v1 import token  # noqa

services/auth/api/v1/auth/__init__.py

@@ -1,3 +1,3 @@
-from api.v1.auth import login
-from api.v1.auth import logout
-from api.v1.auth import register
+from api.v1.auth import login  # noqa
+from api.v1.auth import logout  # noqa
+from api.v1.auth import register  # noqa

services/auth/api/v1/auth/login.py

@@ -1,14 +1,12 @@
 from app import app
 from setup_api import v1
 
-from blacklist_handler import BlacklistHandler
 from encoder import JWTEncoder
 from flask import jsonify, request
-from flask_restx import Api, Resource
+from flask_restx import Resource
 from marshmallow import Schema, fields
 from documents.person import PersonDocument
 from werkzeug.exceptions import Unauthorized
-from typing import Union
 
 
 class LoginSchema(Schema):
@@ -25,6 +23,7 @@ def post(self) -> dict[str, dict[str, str]]:
         if not user or user.passwordHash != PersonDocument._hash(data['password']):
             raise Unauthorized('Username or password incorrect')
         else:
+            # TODO: roles?
             token = JWTEncoder().encode(userId=user.id)
             response = jsonify({
                 'data': {'token': token},