diff --git a/.vscode/launch.json b/.vscode/launch.json index 64d934d5..0dcd3c27 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,22 @@ { "version": "0.2.0", "configurations": [ + { + "name": "Debug Pytest (current file)", + "type": "python", + "request": "launch", + "python": "${command:python.interpreterPath}", + // run pytest as a module + "module": "pytest", + "args": [ + "--maxfail=1", + "--disable-warnings", + "-q", + "${file}" + ], + "console": "integratedTerminal", + "justMyCode": false, + }, { "name": "rag_backend", "type": "python", diff --git a/.vscode/settings.json b/.vscode/settings.json index 94ff2e2f..729db8f9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,6 +6,9 @@ "./rag-core-library/rag-core-api/src", "./rag-core-library/rag-core-lib/src", "./rag-core-library/extractor-api-lib/src", + "./admin-backend", + "./rag-backend", + "./document-extractor" ], "[yaml]": { "editor.tabSize": 2, @@ -13,4 +16,12 @@ "editor.formatOnType": true, "editor.autoIndent": "advanced" }, + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.pytestArgs": ["--import-mode","importlib"], + "python.testing.autoTestDiscoverOnSaveEnabled": true, + "python.envFile": "${workspaceFolder}/.env", + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python-envs.pythonProjects": [], } diff --git a/admin-backend/Dockerfile b/admin-backend/Dockerfile index 69e997c7..449a9932 100644 --- a/admin-backend/Dockerfile +++ b/admin-backend/Dockerfile @@ -19,7 +19,7 @@ COPY admin-backend/pyproject.toml admin-backend/poetry.lock ./ RUN mkdir log && chmod 700 log RUN touch /app/admin-backend/log/logfile.log && chmod 600 /app/admin-backend/log/logfile.log -RUN poetry config virtualenvs.create false &&\ +RUN poetry config virtualenvs.create false && \ if [ "$dev" = "1" ]; then \ poetry install --no-interaction --no-ansi --no-root --with dev; \ else \ diff --git a/admin-backend/poetry.lock b/admin-backend/poetry.lock index d2ac5526..d1ef2ece 100644 --- a/admin-backend/poetry.lock +++ b/admin-backend/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "admin-api-lib" @@ -1823,21 +1823,21 @@ tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<10" [[package]] name = "langchain-core" -version = "0.3.58" +version = "0.3.63" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "langchain_core-0.3.58-py3-none-any.whl", hash = "sha256:266f90d2a079fe9510190ad3be88bd993baad43e6cee0f822a883767a4bfdd5b"}, - {file = "langchain_core-0.3.58.tar.gz", hash = "sha256:6ee2282b02fa65bf4ee1afa869d431505536757ff2f1f9f0b432d8ca755d66c6"}, + {file = "langchain_core-0.3.63-py3-none-any.whl", hash = "sha256:f91db8221b1bc6808f70b2e72fded1a94d50ee3f1dff1636fb5a5a514c64b7f5"}, + {file = "langchain_core-0.3.63.tar.gz", hash = "sha256:e2e30cfbb7684a5a0319f6cbf065fc3c438bfd1060302f085a122527890fb01e"}, ] [package.dependencies] jsonpatch = ">=1.33,<2.0" -langsmith = ">=0.1.125,<0.4" +langsmith = ">=0.1.126,<0.4" packaging = ">=23.2,<25" -pydantic = {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""} +pydantic = ">=2.7.4" PyYAML = ">=5.3" tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<10.0.0" typing-extensions = ">=4.7" @@ -2933,6 +2933,7 @@ deprecated = "^1.2.18" flashrank = "^0.2.10" langchain = "^0.3.25" langchain-community = "0.3.23" +langchain-core = "0.3.63" langfuse = "^2.60.4" oauthlib = "^3.2.2" openai = "^1.77.0" diff --git a/admin-backend/tests/dummy_test.py b/admin-backend/tests/dummy1_test.py similarity index 100% rename from admin-backend/tests/dummy_test.py rename to admin-backend/tests/dummy1_test.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..e2772f2b --- /dev/null +++ b/conftest.py @@ -0,0 +1,15 @@ +import sys +import os +from pathlib import Path + +# Add project root and specific directories to Python path +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) +sys.path.insert(0, str(project_root / "admin-backend")) +sys.path.insert(0, str(project_root / "rag-backend")) +sys.path.insert(0, str(project_root / "document-extractor")) + +# point at each rag-core library's src folder so their packages (admin_api_lib, rag_core_api, etc.) are importable +lib_root = project_root / "rag-core-library" +for lib in ["admin-api-lib", "rag-core-api", "rag-core-lib", "extractor-api-lib"]: + sys.path.insert(0, str(lib_root / lib / "src")) diff --git a/document-extractor/poetry.lock b/document-extractor/poetry.lock index b30c99c3..5fc0622d 100644 --- a/document-extractor/poetry.lock +++ b/document-extractor/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "aiofiles" @@ -1064,7 +1064,7 @@ fastapi = "^0.115.12" fasttext = {git = "https://github.com/cfculhane/fastText", rev = "main"} html5lib = "^1.1" langchain-community = "^0.3.23" -langchain-core = "^0.3.58" +langchain-core = "0.3.63" lxml = "^5.4.0" markdownify = "^1.1.0" numpy = "^2.2.5" @@ -1979,21 +1979,21 @@ tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<10" [[package]] name = "langchain-core" -version = "0.3.58" +version = "0.3.63" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "langchain_core-0.3.58-py3-none-any.whl", hash = "sha256:266f90d2a079fe9510190ad3be88bd993baad43e6cee0f822a883767a4bfdd5b"}, - {file = "langchain_core-0.3.58.tar.gz", hash = "sha256:6ee2282b02fa65bf4ee1afa869d431505536757ff2f1f9f0b432d8ca755d66c6"}, + {file = "langchain_core-0.3.63-py3-none-any.whl", hash = "sha256:f91db8221b1bc6808f70b2e72fded1a94d50ee3f1dff1636fb5a5a514c64b7f5"}, + {file = "langchain_core-0.3.63.tar.gz", hash = "sha256:e2e30cfbb7684a5a0319f6cbf065fc3c438bfd1060302f085a122527890fb01e"}, ] [package.dependencies] jsonpatch = ">=1.33,<2.0" -langsmith = ">=0.1.125,<0.4" +langsmith = ">=0.1.126,<0.4" packaging = ">=23.2,<25" -pydantic = {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""} +pydantic = ">=2.7.4" PyYAML = ">=5.3" tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<10.0.0" typing-extensions = ">=4.7" diff --git a/document-extractor/tests/dummy_test.py b/document-extractor/tests/dummy2_test.py similarity index 100% rename from document-extractor/tests/dummy_test.py rename to document-extractor/tests/dummy2_test.py diff --git a/frontend/libs/admin-app/data-access/+state/documents.store.ts b/frontend/libs/admin-app/data-access/+state/documents.store.ts index e73151c5..383da5d7 100644 --- a/frontend/libs/admin-app/data-access/+state/documents.store.ts +++ b/frontend/libs/admin-app/data-access/+state/documents.store.ts @@ -3,7 +3,7 @@ import { ref } from 'vue'; import { DocumentModel } from "../../models/document.model.ts"; import { ErrorType } from "../../models/error-type"; import { UploadedDocument, mapToUploadDocument } from "../../models/uploaded-document.model"; -import { DocumentAPI } from "../document.api"; +import { DocumentAPI, ConfluenceConfig } from "../document.api"; export const useDocumentsStore = defineStore('chat', () => { const uploadedDocuments = ref([]); @@ -52,11 +52,12 @@ export const useDocumentsStore = defineStore('chat', () => { } }; - const loadConfluence = async () => { + const loadConfluence = async (config: ConfluenceConfig) => { isLoadingConfluence.value = true; error.value = null; try { - await DocumentAPI.loadConfluence(); + // provide confluence configuration from frontend + await DocumentAPI.loadConfluence(config); await loadDocuments(); // Refresh the document list after uploading } catch(err) { if (err.response && err.response.status === 501) { diff --git a/frontend/libs/admin-app/data-access/document.api.ts b/frontend/libs/admin-app/data-access/document.api.ts index ef058ab1..44656bf6 100644 --- a/frontend/libs/admin-app/data-access/document.api.ts +++ b/frontend/libs/admin-app/data-access/document.api.ts @@ -7,6 +7,15 @@ axios.defaults.auth = { password: import.meta.env.VITE_AUTH_PASSWORD }; +// confluence configuration interface +export interface ConfluenceConfig { + spaceKey: string; + token: string; + url: string; + maxPages: number; + name: string; +} + export class DocumentAPI { static async loadDocuments(): Promise { try { @@ -20,9 +29,9 @@ export class DocumentAPI { static async uploadDocument(file: File, onUploadProgress: (progressEvent: AxiosProgressEvent) => void): Promise { try { const formData = new FormData(); - formData.append('body', file); + formData.append('file', file); - const response = await axios.post('/upload_documents', formData, { + const response = await axios.post('/upload_file', formData, { headers: { 'Content-Type': 'multipart/form-data' }, @@ -35,9 +44,19 @@ export class DocumentAPI { } } - static async loadConfluence(): Promise { + static async loadConfluence(config: ConfluenceConfig): Promise { try { - await axios.post('/load_confluence'); + // convert config to list of key/value items for backend + const payload = [ + { key: 'url', value: config.url }, + { key: 'token', value: config.token }, + { key: 'space_key', value: config.spaceKey }, + { key: 'max_pages', value: String(config.maxPages) } + ]; + // include required query parameters + await axios.post('/upload_source', payload, { + params: { source_type: 'confluence', name: config.name } + }); } catch(error) { this.handleError(error); } diff --git a/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue b/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue index 949e976b..846afdf8 100644 --- a/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue +++ b/frontend/libs/admin-app/feature-document/DocumentUploadContainer.vue @@ -15,6 +15,14 @@ const isInvalidFileType = ref(false); const allowedFileTypes = ['application/pdf', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'text/xml']; const uploadMethod = ref<'file' | 'confluence'>('file'); + +// confluence configuration refs +const confluenceName = ref(''); +const spaceKey = ref(''); +const confluenceToken = ref(''); +const confluenceUrl = ref(''); +const maxPages = ref(); + const error = computed(() => store.error); const uploadDocuments = (files: File[]) => { @@ -55,7 +63,14 @@ const onRemoveDocument = (documentId: string) => { } const handleConfluenceUpload = () => { - store.loadConfluence(); + // send configured parameters to backend + store.loadConfluence({ + name: confluenceName.value, + spaceKey: spaceKey.value, + token: confluenceToken.value, + url: confluenceUrl.value, + maxPages: maxPages.value + }); } const clearError = () => { @@ -113,7 +128,7 @@ const getErrorMessage = (errorType: string) => {
@@ -130,10 +145,23 @@ const getErrorMessage = (errorType: string) => {
+ class="flex flex-col m-auto justify-center items-center w-full h-112 bg-base-100 rounded-box border border-base-300">

{{ t('documents.confluenceLoadTitle') }}

+ +
+ + + + + + + + + + +

{{ t('documents.confluenceLoadDescription') }}