From 082c94d31e616598e49aa4ea0f2f3421cc8d91da Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 14 May 2024 20:38:24 +0000 Subject: [PATCH 1/6] Update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5b25c77..f952df9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # RAG on PostgreSQL -[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](placeholder) -[![Open in Dev Containers](https://img.shields.io/static/v1?style=for-the-badge&label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](placeholder) +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/Azure-Samples/rag-postgres-openai-python) +[![Open in Dev Containers](https://img.shields.io/static/v1?style=for-the-badge&label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/azure-samples/rag-postgres-openai-python) This project creates a web-based chat application with an API backend that can use OpenAI chat models to answer questions about the items in a PostgreSQL database table. The frontend is built with React and FluentUI, while the backend is written with Python and FastAPI. From 802655b9003aa18471f62a191166b64dbccaf887 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 14 May 2024 20:50:55 +0000 Subject: [PATCH 2/6] More workflows --- .devcontainer/devcontainer.json | 2 +- .../workflows/{tests.yaml => app-tests.yaml} | 11 ++- .github/workflows/bicep-audit.yaml | 34 ---------- ...lidation.yaml => bicep-security-scan.yaml} | 9 ++- .github/workflows/python-code-quality.yaml | 25 +++++++ CONTRIBUTING.md | 68 +++++++++++++++---- README.md | 2 +- infra/main.bicep | 19 +++--- infra/web.bicep | 5 +- 9 files changed, 105 insertions(+), 70 deletions(-) rename .github/workflows/{tests.yaml => app-tests.yaml} (70%) delete mode 100644 .github/workflows/bicep-audit.yaml rename .github/workflows/{bicep-validation.yaml => bicep-security-scan.yaml} (81%) create mode 100644 .github/workflows/python-code-quality.yaml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 37f8efb..532b2fd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: // https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/python-3 { - "name": "RAG on database", + "name": "rag-postgres-openai-python", "dockerComposeFile": "docker-compose.yaml", "service": "app", "workspaceFolder": "/workspace", diff --git a/.github/workflows/tests.yaml b/.github/workflows/app-tests.yaml similarity index 70% rename from .github/workflows/tests.yaml rename to .github/workflows/app-tests.yaml index 52da6db..a4f0f79 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -1,4 +1,4 @@ -name: Python check +name: App Tests on: push: @@ -33,4 +33,11 @@ jobs: architecture: x64 - name: Install dependencies run: | - python3 -m pip install -e src + python -m pip install -r requirements-dev.txt + - name: Install app as editable app + run: | + python -m pip install -e src + - name: Setup local database with seed data + cp .env.sample .env + python ./src/fastapi_app/setup_postgres_database.py + python ./src/fastapi_app/setup_postgres_seeddata.py diff --git a/.github/workflows/bicep-audit.yaml b/.github/workflows/bicep-audit.yaml deleted file mode 100644 index cce79cb..0000000 --- a/.github/workflows/bicep-audit.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Analyze AZD Template for Security Issues -on: - push: - branches: [ main ] - paths: - - "infra/**" - pull_request: - branches: [ main ] - paths: - - "infra/**" - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - permissions: - security-events: write - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Run Microsoft Security DevOps Analysis - uses: microsoft/security-devops-action@preview - id: msdo - continue-on-error: true - with: - tools: templateanalyzer - - - name: Upload alerts to Security tab - uses: github/codeql-action/upload-sarif@v3 - if: github.repository_owner == 'Azure-Samples' - with: - - sarif_file: ${{ steps.msdo.outputs.sarifFile }} \ No newline at end of file diff --git a/.github/workflows/bicep-validation.yaml b/.github/workflows/bicep-security-scan.yaml similarity index 81% rename from .github/workflows/bicep-validation.yaml rename to .github/workflows/bicep-security-scan.yaml index 5100934..f7f2eb7 100644 --- a/.github/workflows/bicep-validation.yaml +++ b/.github/workflows/bicep-security-scan.yaml @@ -1,4 +1,4 @@ -name: Validate AZD template +name: Bicep Security Scan on: push: branches: [ main ] @@ -10,7 +10,6 @@ on: - "infra/**" workflow_dispatch: - jobs: build: runs-on: ubuntu-latest @@ -21,12 +20,12 @@ jobs: uses: actions/checkout@v4 - name: Build Bicep for linting - uses: azure/CLI@v1 + uses: azure/CLI@v2 with: inlineScript: az config set bicep.use_binary_from_path=false && az bicep build -f infra/main.bicep --stdout - name: Run Microsoft Security DevOps Analysis - uses: microsoft/security-devops-action@v1 + uses: microsoft/security-devops-action@preview id: msdo continue-on-error: true with: @@ -34,6 +33,6 @@ jobs: - name: Upload alerts to Security tab uses: github/codeql-action/upload-sarif@v3 - if: github.repository == 'Azure-Samples/langfuse-on-azure' + if: github.repository == 'Azure-Samples/azure-search-openai-demo' with: sarif_file: ${{ steps.msdo.outputs.sarifFile }} diff --git a/.github/workflows/python-code-quality.yaml b/.github/workflows/python-code-quality.yaml new file mode 100644 index 0000000..0026bfd --- /dev/null +++ b/.github/workflows/python-code-quality.yaml @@ -0,0 +1,25 @@ +name: Python code quality + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + - name: Lint with ruff + run: ruff . + - name: Check formatting with black + run: black . --check --verbose diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1391df3..9284002 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to [project-title] +# Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us @@ -15,7 +15,9 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio - [Code of Conduct](#coc) - [Issues and Bugs](#issue) - [Feature Requests](#feature) - - [Submission Guidelines](#submit) + - [Submitting a PR](#submit-pr) + - [Running Tests](#tests) + - [Code Style](#style) ## Code of Conduct Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). @@ -51,26 +53,64 @@ chances of your issue being dealt with quickly: * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be causing the problem (line of code or commit) -You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/Azure-Samples/rag-postgres-openai-python/issues/new]. +You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/Azure-samples/rag-postgres-openai-python/issues/new]. ### Submitting a Pull Request (PR) Before you submit your Pull Request (PR) consider the following guidelines: -* Search the repository (https://github.com/Azure-Samples/rag-postgres-openai-python/pulls) for an open or closed PR +* Search the repository (https://github.com/Azure-samples/rag-postgres-openai-python/pulls) for an open or closed PR that relates to your submission. You don't want to duplicate effort. - * Make your changes in a new git fork +* Follow [Code style conventions](#style) +* [Run the tests](#tests) (and write new ones, if needed) * Commit your changes using a descriptive commit message * Push your fork to GitHub -* In GitHub, create a pull request -* If we suggest changes then: - * Make the required updates. - * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): +* In GitHub, create a pull request to the `main` branch of the repository +* Ask a maintainer to review your PR and address any comments they might have + +## Setting up the development environment + +Install the development dependencies: + +``` +python3 -m pip install -r requirements-dev.txt +``` + +Install the pre-commit hooks: + +``` +pre-commit install +``` + +Compile the JavaScript: + +``` +( cd ./app/frontend ; npm install ; npm run build ) +``` + +## Code Style + +This codebase includes several languages: TypeScript, Python, Bicep, Powershell, and Bash. +Code should follow the standard conventions of each language. + +For Python, you can enforce the conventions using `ruff` and `black`. + +Install the development dependencies: + +``` +python3 -m pip install -r requirements-dev.txt +``` + +Run `ruff` to lint a file: + +``` +python3 -m ruff +``` - ```shell - git rebase master -i - git push -f - ``` +Run `black` to format a file: -That's it! Thank you for your contribution! +``` +python3 -m black +``` +If you followed the steps above to install the pre-commit hooks, then you can just wait for those hooks to run `ruff` and `black` for you. diff --git a/README.md b/README.md index f952df9..e70b01a 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ You can run this template virtually by using GitHub Codespaces. The button will 3. Sign in to your Azure account: ```shell - azd auth login --use-device-code + azd auth login ``` 4. Provision the resources and deploy the code: diff --git a/infra/main.bicep b/infra/main.bicep index 2fbba05..28e572a 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -246,7 +246,6 @@ module web 'web.bicep' = { } } - resource openAiResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = if (!empty(openAiResourceGroupName)) { name: !empty(openAiResourceGroupName) ? openAiResourceGroupName : resourceGroup.name @@ -293,15 +292,16 @@ module openAi 'core/ai/cognitiveservices.bicep' = { } // USER ROLES -module openAiRoleUser 'core/security/role.bicep' = if (empty(runningOnGh)) { - scope: openAiResourceGroup - name: 'openai-role-user' - params: { - principalId: principalId - roleDefinitionId: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' - principalType: 'User' +module openAiRoleUser 'core/security/role.bicep' = + if (empty(runningOnGh)) { + scope: openAiResourceGroup + name: 'openai-role-user' + params: { + principalId: principalId + roleDefinitionId: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' + principalType: 'User' + } } -} // Backend roles module openAiRoleBackend 'core/security/role.bicep' = { @@ -314,7 +314,6 @@ module openAiRoleBackend 'core/security/role.bicep' = { } } - output AZURE_LOCATION string = location output APPLICATIONINSIGHTS_NAME string = monitoring.outputs.applicationInsightsName diff --git a/infra/web.bicep b/infra/web.bicep index f376693..6a6c414 100644 --- a/infra/web.bicep +++ b/infra/web.bicep @@ -9,13 +9,11 @@ param identityName string param serviceName string = 'web' param environmentVariables array = [] - resource webIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { name: identityName location: location } - module app 'core/host/container-app-upsert.bicep' = { name: '${serviceName}-container-app-module' params: { @@ -26,7 +24,8 @@ module app 'core/host/container-app-upsert.bicep' = { exists: exists containerAppsEnvironmentName: containerAppsEnvironmentName containerRegistryName: containerRegistryName - env: union(environmentVariables, + env: union( + environmentVariables, [ { name: 'APP_IDENTITY_ID' From 687227acc0717af22b97e8fd2008e4f3755653d8 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 14 May 2024 20:55:43 +0000 Subject: [PATCH 3/6] Security updates --- src/frontend/package-lock.json | 14 ++++---- src/frontend/package.json | 2 +- src/pyproject.toml | 5 ++- src/requirements.txt | 61 +++++++++++++++++++++++++--------- 4 files changed, 55 insertions(+), 27 deletions(-) diff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.json index 3e2cf15..7a25073 100644 --- a/src/frontend/package-lock.json +++ b/src/frontend/package-lock.json @@ -31,7 +31,7 @@ "@vitejs/plugin-react": "^4.1.1", "prettier": "^3.0.3", "typescript": "^5.2.2", - "vite": "^4.5.2" + "vite": "^4.5.3" }, "engines": { "node": ">=14.0.0" @@ -3522,9 +3522,9 @@ } }, "node_modules/vite": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz", - "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==", + "version": "4.5.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.3.tgz", + "integrity": "sha512-kQL23kMeX92v3ph7IauVkXkikdDRsYMGTVl5KY2E9OY4ONLvkHf04MDTbnfo6NKxZiDLWzVpP5oTa8hQD8U3dg==", "dev": true, "dependencies": { "esbuild": "^0.18.10", @@ -5802,9 +5802,9 @@ "requires": {} }, "vite": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz", - "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==", + "version": "4.5.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.3.tgz", + "integrity": "sha512-kQL23kMeX92v3ph7IauVkXkikdDRsYMGTVl5KY2E9OY4ONLvkHf04MDTbnfo6NKxZiDLWzVpP5oTa8hQD8U3dg==", "dev": true, "requires": { "esbuild": "^0.18.10", diff --git a/src/frontend/package.json b/src/frontend/package.json index 632d80a..d6ae12e 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -35,6 +35,6 @@ "prettier": "^3.0.3", "typescript": "^5.2.2", "@types/react-syntax-highlighter": "^15.5.7", - "vite": "^4.5.2" + "vite": "^4.5.3" } } diff --git a/src/pyproject.toml b/src/pyproject.toml index 41aa005..a4bb0de 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -15,10 +15,9 @@ dependencies = [ "pgvector", "openai", "tiktoken", - "openai-messages-token-helper", - "rich" + "openai-messages-token-helper" ] [build-system] requires = ["flit_core<4"] -build-backend = "flit_core.buildapi" \ No newline at end of file +build-backend = "flit_core.buildapi" diff --git a/src/requirements.txt b/src/requirements.txt index c4998ae..4ae104d 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -36,25 +36,35 @@ cffi==1.16.0 charset-normalizer==3.3.2 # via requests click==8.1.7 - # via uvicorn -cryptography==42.0.6 + # via + # typer + # uvicorn +cryptography==42.0.7 # via # azure-identity # msal # pyjwt distro==1.9.0 # via openai +dnspython==2.6.1 + # via email-validator +email-validator==2.1.1 + # via fastapi environs==11.0.0 # via fastapi_app (pyproject.toml) -fastapi==0.110.3 - # via fastapi_app (pyproject.toml) +fastapi==0.111.0 + # via + # fastapi-cli + # fastapi_app (pyproject.toml) +fastapi-cli==0.0.3 + # via fastapi frozenlist==1.4.1 # via # aiohttp # aiosignal greenlet==3.0.3 # via sqlalchemy -gunicorn==20.1.0 +gunicorn==22.0.0 # via fastapi_app (pyproject.toml) h11==0.14.0 # via @@ -65,15 +75,22 @@ httpcore==1.0.5 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via openai + # via + # fastapi + # openai idna==3.7 # via # anyio + # email-validator # httpx # requests # yarl +jinja2==3.1.4 + # via fastapi markdown-it-py==3.0.0 # via rich +markupsafe==2.1.5 + # via jinja2 marshmallow==3.21.2 # via environs mdurl==0.1.2 @@ -90,14 +107,17 @@ multidict==6.0.5 # yarl numpy==1.26.4 # via pgvector -openai==1.25.2 +openai==1.30.1 # via # fastapi_app (pyproject.toml) # openai-messages-token-helper -openai-messages-token-helper==0.1.3 +openai-messages-token-helper==0.1.4 # via fastapi_app (pyproject.toml) +orjson==3.10.3 + # via fastapi packaging==24.0 # via + # gunicorn # marshmallow # msal-extensions pgvector==0.2.5 @@ -125,9 +145,11 @@ python-dotenv==1.0.1 # environs # fastapi_app (pyproject.toml) # uvicorn +python-multipart==0.0.9 + # via fastapi pyyaml==6.0.1 # via uvicorn -regex==2024.4.28 +regex==2024.5.10 # via tiktoken requests==2.31.0 # via @@ -135,7 +157,9 @@ requests==2.31.0 # msal # tiktoken rich==13.7.1 - # via fastapi_app (pyproject.toml) + # via typer +shellingham==1.5.4 + # via typer six==1.16.0 # via azure-core sniffio==1.3.1 @@ -147,12 +171,14 @@ sqlalchemy[asyncio]==2.0.30 # via fastapi_app (pyproject.toml) starlette==0.37.2 # via fastapi -tiktoken==0.6.0 +tiktoken==0.7.0 # via # fastapi_app (pyproject.toml) # openai-messages-token-helper tqdm==4.66.4 # via openai +typer==0.12.3 + # via fastapi-cli typing-extensions==4.11.0 # via # azure-core @@ -161,10 +187,16 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # sqlalchemy + # typer +ujson==5.10.0 + # via fastapi urllib3==2.2.1 # via requests -uvicorn[standard]==0.23.2 - # via fastapi_app (pyproject.toml) +uvicorn[standard]==0.29.0 + # via + # fastapi + # fastapi-cli + # fastapi_app (pyproject.toml) uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 @@ -173,6 +205,3 @@ websockets==12.0 # via uvicorn yarl==1.9.4 # via aiohttp - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From b32311ec10bb5980b40e11bb266b8487675a7414 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 14 May 2024 21:00:24 +0000 Subject: [PATCH 4/6] Ruff and black --- .github/workflows/python-code-quality.yaml | 2 +- pyproject.toml | 3 ++- src/fastapi_app/query_rewriter.py | 2 +- src/fastapi_app/rag_advanced.py | 17 ++++++++++------- src/fastapi_app/setup_postgres_seeddata.py | 19 +++++++++++-------- src/gunicorn.conf.py | 2 +- 6 files changed, 26 insertions(+), 19 deletions(-) diff --git a/.github/workflows/python-code-quality.yaml b/.github/workflows/python-code-quality.yaml index 0026bfd..987b2e1 100644 --- a/.github/workflows/python-code-quality.yaml +++ b/.github/workflows/python-code-quality.yaml @@ -20,6 +20,6 @@ jobs: python -m pip install --upgrade pip pip install -r requirements-dev.txt - name: Lint with ruff - run: ruff . + run: ruff check . - name: Check formatting with black run: black . --check --verbose diff --git a/pyproject.toml b/pyproject.toml index 89e677b..68ed78b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,10 @@ [tool.ruff] line-length = 120 target-version = "py311" + +[tool.ruff.lint] select = ["E", "F", "I", "UP"] ignore = ["D203"] -show-source = true [tool.ruff.lint.isort] known-first-party = ["fastapi_app"] diff --git a/src/fastapi_app/query_rewriter.py b/src/fastapi_app/query_rewriter.py index a02157e..99a8624 100644 --- a/src/fastapi_app/query_rewriter.py +++ b/src/fastapi_app/query_rewriter.py @@ -26,7 +26,7 @@ def build_search_function() -> list[ChatCompletionToolParam]: "properties": { "comparison_operator": { "type": "string", - "description": "Operator to compare the column value, either '>', '<', '>=', '<=', '=='", + "description": "Operator to compare the column value, either '>', '<', '>=', '<=', '=='", # noqa }, "value": { "type": "number", diff --git a/src/fastapi_app/rag_advanced.py b/src/fastapi_app/rag_advanced.py index d6bdc4d..e40fa80 100644 --- a/src/fastapi_app/rag_advanced.py +++ b/src/fastapi_app/rag_advanced.py @@ -17,7 +17,6 @@ class AdvancedRAGChat: - def __init__( self, *, @@ -26,7 +25,8 @@ def __init__( chat_model: str, chat_deployment: str | None, # Not needed for non-Azure OpenAI openai_embed_client: AsyncOpenAI, - embed_deployment: str | None, # Not needed for non-Azure OpenAI or for retrieval_mode="text" + embed_deployment: str + | None, # Not needed for non-Azure OpenAI or for retrieval_mode="text" embed_model: str, embed_dimensions: int, ): @@ -46,7 +46,6 @@ def __init__( async def run( self, messages: list[dict], overrides: dict[str, Any] = {} ) -> dict[str, Any] | AsyncGenerator[dict[str, Any], None]: - text_search = overrides.get("retrieval_mode") in ["text", "hybrid", None] vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] top = overrides.get("top", 3) @@ -61,7 +60,8 @@ async def run( system_prompt=self.query_prompt_template, new_user_content=original_user_query, past_messages=past_messages, - max_tokens=self.chat_token_limit - query_response_token_limit, # TODO: count functions + max_tokens=self.chat_token_limit + - query_response_token_limit, # TODO: count functions fallback_to_default=True, ) @@ -70,7 +70,7 @@ async def run( # Azure OpenAI takes the deployment name as the model name model=self.chat_deployment if self.chat_deployment else self.chat_model, temperature=0.0, # Minimize creativity for search query generation - max_tokens=query_response_token_limit, # Setting too low risks malformed JSON, setting too high may affect performance + max_tokens=query_response_token_limit, # Setting too low risks malformed JSON, too high risks performance n=1, tools=build_search_function(), tool_choice="auto", @@ -93,14 +93,17 @@ async def run( results = await self.searcher.search(query_text, vector, top, filters) - sources_content = [f"[{(item.id)}]:{item.to_str_for_rag()}\n\n" for item in results] + sources_content = [ + f"[{(item.id)}]:{item.to_str_for_rag()}\n\n" for item in results + ] content = "\n".join(sources_content) # Generate a contextual and content specific answer using the search results and chat history response_token_limit = 1024 messages = build_messages( model=self.chat_model, - system_prompt=overrides.get("prompt_template") or self.answer_prompt_template, + system_prompt=overrides.get("prompt_template") + or self.answer_prompt_template, new_user_content=original_user_query + "\n\nSources:\n" + content, past_messages=past_messages, max_tokens=self.chat_token_limit - response_token_limit, diff --git a/src/fastapi_app/setup_postgres_seeddata.py b/src/fastapi_app/setup_postgres_seeddata.py index 4ee31df..98f50e5 100644 --- a/src/fastapi_app/setup_postgres_seeddata.py +++ b/src/fastapi_app/setup_postgres_seeddata.py @@ -9,33 +9,38 @@ from sqlalchemy import select, text from sqlalchemy.ext.asyncio import async_sessionmaker -from fastapi_app.postgres_engine import create_postgres_engine_from_args, create_postgres_engine_from_env +from fastapi_app.postgres_engine import ( + create_postgres_engine_from_args, + create_postgres_engine_from_env, +) from fastapi_app.postgres_models import Item logger = logging.getLogger("ragapp") async def seed_data(engine): - # Check if Item table exists async with engine.begin() as conn: result = await conn.execute( text( - "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'items')" + "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'items')" # noqa ) ) if not result.scalar(): - logger.error("Items table does not exist. Please run the database setup script first.") + logger.error( + "Items table does not exist. Please run the database setup script first." + ) return async with async_sessionmaker(engine, expire_on_commit=False)() as session: - # Insert the items from the JSON file into the database current_dir = os.path.dirname(os.path.realpath(__file__)) with open(os.path.join(current_dir, "seed_data.json")) as f: catalog_items = json.load(f) for catalog_item in catalog_items: - item = await session.execute(select(Item).filter(Item.id == catalog_item["Id"])) + item = await session.execute( + select(Item).filter(Item.id == catalog_item["Id"]) + ) if item.scalars().first(): continue item = Item( @@ -57,7 +62,6 @@ async def seed_data(engine): async def main(): - parser = argparse.ArgumentParser(description="Create database schema") parser.add_argument("--host", type=str, help="Postgres host") parser.add_argument("--username", type=str, help="Postgres username") @@ -78,7 +82,6 @@ async def main(): if __name__ == "__main__": - logging.basicConfig(level=logging.WARNING) logger.setLevel(logging.INFO) load_dotenv(override=True) diff --git a/src/gunicorn.conf.py b/src/gunicorn.conf.py index c6bc3ba..03df0f7 100644 --- a/src/gunicorn.conf.py +++ b/src/gunicorn.conf.py @@ -8,4 +8,4 @@ worker_class = "uvicorn.workers.UvicornWorker" -timeout = 600 \ No newline at end of file +timeout = 600 From ed4fcc6369895ec262f8b6740582fcc9ee569882 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 14 May 2024 21:01:33 +0000 Subject: [PATCH 5/6] Actions versions --- .github/workflows/app-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index a4f0f79..4775aff 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -25,9 +25,9 @@ jobs: # needed because the postgres container does not provide a healthcheck options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} architecture: x64 From 3e42e82e5e83eea96c113e55718121df20dcf7e9 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 14 May 2024 21:07:28 +0000 Subject: [PATCH 6/6] Move to ruff format --- .devcontainer/devcontainer.json | 3 +-- .github/PULL_REQUEST_TEMPLATE.md | 3 +-- .github/workflows/python-code-quality.yaml | 4 ++-- .pre-commit-config.yaml | 10 +++++----- CONTRIBUTING.md | 10 +++++----- pyproject.toml | 4 ---- requirements-dev.txt | 3 +-- src/fastapi_app/api_routes.py | 1 - src/fastapi_app/openai_clients.py | 1 - src/fastapi_app/postgres_engine.py | 1 - src/fastapi_app/postgres_searcher.py | 2 -- src/fastapi_app/rag_advanced.py | 13 ++++--------- src/fastapi_app/rag_simple.py | 2 -- src/fastapi_app/setup_postgres_azurerole.py | 3 --- src/fastapi_app/setup_postgres_database.py | 2 -- src/fastapi_app/setup_postgres_seeddata.py | 8 ++------ 16 files changed, 21 insertions(+), 49 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 532b2fd..06d7ef0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -29,7 +29,6 @@ "ms-python.python", "ms-python.vscode-pylance", "charliermarsh.ruff", - "ms-python.black-formatter", "mtxr.sqltools", "mtxr.sqltools-driver-pg", "ms-vscode.vscode-node-azure-pack", @@ -45,7 +44,7 @@ "editor.codeActionsOnSave": { "source.fixAll": "explicit" }, - "editor.defaultFormatter": "ms-python.black-formatter" + "editor.defaultFormatter": "charliermarsh.ruff" }, "files.exclude": { ".ruff_cache": true, diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 52c8c39..5e79659 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -32,5 +32,4 @@ See [CONTRIBUTING.md](https://github.com/Azure-Samples/rag-postgres-openai-pytho - [ ] I added tests that prove my fix is effective or that my feature works - [ ] I ran `python -m pytest --cov` to verify 100% coverage of added lines - [ ] I ran `python -m mypy` to check for type errors -- [ ] I either used the pre-commit hooks or ran `ruff` and `black` manually on my code. - +- [ ] I either used the pre-commit hooks or ran `ruff` manually on my code. diff --git a/.github/workflows/python-code-quality.yaml b/.github/workflows/python-code-quality.yaml index 987b2e1..56191b1 100644 --- a/.github/workflows/python-code-quality.yaml +++ b/.github/workflows/python-code-quality.yaml @@ -21,5 +21,5 @@ jobs: pip install -r requirements-dev.txt - name: Lint with ruff run: ruff check . - - name: Check formatting with black - run: black . --check --verbose + - name: Check formatting with ruff + run: ruff format --check . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 839d4fc..48296dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,8 +8,8 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.0 hooks: - - id: ruff -- repo: https://github.com/psf/black - rev: 23.9.1 - hooks: - - id: black + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9284002..5528a7b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,7 +93,7 @@ Compile the JavaScript: This codebase includes several languages: TypeScript, Python, Bicep, Powershell, and Bash. Code should follow the standard conventions of each language. -For Python, you can enforce the conventions using `ruff` and `black`. +For Python, you can enforce the conventions using `ruff`. Install the development dependencies: @@ -104,13 +104,13 @@ python3 -m pip install -r requirements-dev.txt Run `ruff` to lint a file: ``` -python3 -m ruff +python3 -m ruff check ``` -Run `black` to format a file: +Run `ruff` to format a file: ``` -python3 -m black +python3 -m ruff format ``` -If you followed the steps above to install the pre-commit hooks, then you can just wait for those hooks to run `ruff` and `black` for you. +If you followed the steps above to install the pre-commit hooks, then you can just wait for those hooks to run `ruff` for you. diff --git a/pyproject.toml b/pyproject.toml index 68ed78b..d6d7928 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,3 @@ ignore = ["D203"] [tool.ruff.lint.isort] known-first-party = ["fastapi_app"] - -[tool.black] -line-length = 120 -target-version = ["py311"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 87fda3e..0924b72 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,4 @@ -r src/requirements.txt ruff -black pre-commit -pip-tools \ No newline at end of file +pip-tools diff --git a/src/fastapi_app/api_routes.py b/src/fastapi_app/api_routes.py index 0153250..c3f7f10 100644 --- a/src/fastapi_app/api_routes.py +++ b/src/fastapi_app/api_routes.py @@ -11,7 +11,6 @@ @router.post("/chat") async def chat_handler(chat_request: ChatRequest): - messages = [message.model_dump() for message in chat_request.messages] overrides = chat_request.context.get("overrides", {}) diff --git a/src/fastapi_app/openai_clients.py b/src/fastapi_app/openai_clients.py index 26c4a3f..c3d4fd2 100644 --- a/src/fastapi_app/openai_clients.py +++ b/src/fastapi_app/openai_clients.py @@ -38,7 +38,6 @@ async def create_openai_chat_client(azure_credential): async def create_openai_embed_client(azure_credential): - OPENAI_EMBED_HOST = os.getenv("OPENAI_EMBED_HOST") if OPENAI_EMBED_HOST == "azure": token_provider = azure.identity.aio.get_bearer_token_provider( diff --git a/src/fastapi_app/postgres_engine.py b/src/fastapi_app/postgres_engine.py index fd3a295..08a2f7a 100644 --- a/src/fastapi_app/postgres_engine.py +++ b/src/fastapi_app/postgres_engine.py @@ -8,7 +8,6 @@ async def create_postgres_engine(*, host, username, database, password, sslmode, azure_credential) -> AsyncEngine: - if host.endswith(".database.azure.com"): logger.info("Authenticating to Azure Database for PostgreSQL using Azure Identity...") if azure_credential is None: diff --git a/src/fastapi_app/postgres_searcher.py b/src/fastapi_app/postgres_searcher.py index 0d9cfab..1765a23 100644 --- a/src/fastapi_app/postgres_searcher.py +++ b/src/fastapi_app/postgres_searcher.py @@ -6,7 +6,6 @@ class PostgresSearcher: - def __init__(self, engine): self.async_session_maker = async_sessionmaker(engine, expire_on_commit=False) @@ -30,7 +29,6 @@ async def search( query_top: int = 5, filters: list[dict] | None = None, ): - filter_clause_where, filter_clause_and = self.build_filter_clause(filters) vector_query = f""" diff --git a/src/fastapi_app/rag_advanced.py b/src/fastapi_app/rag_advanced.py index e40fa80..81a1fd5 100644 --- a/src/fastapi_app/rag_advanced.py +++ b/src/fastapi_app/rag_advanced.py @@ -25,8 +25,7 @@ def __init__( chat_model: str, chat_deployment: str | None, # Not needed for non-Azure OpenAI openai_embed_client: AsyncOpenAI, - embed_deployment: str - | None, # Not needed for non-Azure OpenAI or for retrieval_mode="text" + embed_deployment: str | None, # Not needed for non-Azure OpenAI or for retrieval_mode="text" embed_model: str, embed_dimensions: int, ): @@ -60,8 +59,7 @@ async def run( system_prompt=self.query_prompt_template, new_user_content=original_user_query, past_messages=past_messages, - max_tokens=self.chat_token_limit - - query_response_token_limit, # TODO: count functions + max_tokens=self.chat_token_limit - query_response_token_limit, # TODO: count functions fallback_to_default=True, ) @@ -93,17 +91,14 @@ async def run( results = await self.searcher.search(query_text, vector, top, filters) - sources_content = [ - f"[{(item.id)}]:{item.to_str_for_rag()}\n\n" for item in results - ] + sources_content = [f"[{(item.id)}]:{item.to_str_for_rag()}\n\n" for item in results] content = "\n".join(sources_content) # Generate a contextual and content specific answer using the search results and chat history response_token_limit = 1024 messages = build_messages( model=self.chat_model, - system_prompt=overrides.get("prompt_template") - or self.answer_prompt_template, + system_prompt=overrides.get("prompt_template") or self.answer_prompt_template, new_user_content=original_user_query + "\n\nSources:\n" + content, past_messages=past_messages, max_tokens=self.chat_token_limit - response_token_limit, diff --git a/src/fastapi_app/rag_simple.py b/src/fastapi_app/rag_simple.py index 00dc97b..fc0864a 100644 --- a/src/fastapi_app/rag_simple.py +++ b/src/fastapi_app/rag_simple.py @@ -13,7 +13,6 @@ class SimpleRAGChat: - def __init__( self, *, @@ -41,7 +40,6 @@ def __init__( async def run( self, messages: list[dict], overrides: dict[str, Any] = {} ) -> dict[str, Any] | AsyncGenerator[dict[str, Any], None]: - text_search = overrides.get("retrieval_mode") in ["text", "hybrid", None] vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] top = overrides.get("top", 3) diff --git a/src/fastapi_app/setup_postgres_azurerole.py b/src/fastapi_app/setup_postgres_azurerole.py index 9406932..a73e382 100644 --- a/src/fastapi_app/setup_postgres_azurerole.py +++ b/src/fastapi_app/setup_postgres_azurerole.py @@ -11,7 +11,6 @@ async def assign_role_for_webapp(engine, app_identity_name): - async with engine.begin() as conn: identities = await conn.execute( text(f"select * from pgaadauth_list_principals(false) WHERE rolname = '{app_identity_name}'") @@ -39,7 +38,6 @@ async def assign_role_for_webapp(engine, app_identity_name): async def main(): - parser = argparse.ArgumentParser(description="Create database schema") parser.add_argument("--host", type=str, help="Postgres host") parser.add_argument("--username", type=str, help="Postgres username") @@ -64,7 +62,6 @@ async def main(): if __name__ == "__main__": - logging.basicConfig(level=logging.WARNING) logger.setLevel(logging.INFO) load_dotenv(override=True) diff --git a/src/fastapi_app/setup_postgres_database.py b/src/fastapi_app/setup_postgres_database.py index 54df1d1..f39abf5 100644 --- a/src/fastapi_app/setup_postgres_database.py +++ b/src/fastapi_app/setup_postgres_database.py @@ -22,7 +22,6 @@ async def create_db_schema(engine): async def main(): - parser = argparse.ArgumentParser(description="Create database schema") parser.add_argument("--host", type=str, help="Postgres host") parser.add_argument("--username", type=str, help="Postgres username") @@ -45,7 +44,6 @@ async def main(): if __name__ == "__main__": - logging.basicConfig(level=logging.WARNING) logger.setLevel(logging.INFO) load_dotenv(override=True) diff --git a/src/fastapi_app/setup_postgres_seeddata.py b/src/fastapi_app/setup_postgres_seeddata.py index 98f50e5..e8719c9 100644 --- a/src/fastapi_app/setup_postgres_seeddata.py +++ b/src/fastapi_app/setup_postgres_seeddata.py @@ -27,9 +27,7 @@ async def seed_data(engine): ) ) if not result.scalar(): - logger.error( - "Items table does not exist. Please run the database setup script first." - ) + logger.error("Items table does not exist. Please run the database setup script first.") return async with async_sessionmaker(engine, expire_on_commit=False)() as session: @@ -38,9 +36,7 @@ async def seed_data(engine): with open(os.path.join(current_dir, "seed_data.json")) as f: catalog_items = json.load(f) for catalog_item in catalog_items: - item = await session.execute( - select(Item).filter(Item.id == catalog_item["Id"]) - ) + item = await session.execute(select(Item).filter(Item.id == catalog_item["Id"])) if item.scalars().first(): continue item = Item(