From 43bae3ed7413f58f9a392b8e055f5f69bcf8946d Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 29 Apr 2026 13:37:26 -0700 Subject: [PATCH 1/8] For testing/prototyping labkey documentation RAG in non-distributed module --- .../org/labkey/devtools/DevtoolsModule.java | 2 + .../org/labkey/devtools/TestController.java | 123 +++++++++++++++++- 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/devtools/src/org/labkey/devtools/DevtoolsModule.java b/devtools/src/org/labkey/devtools/DevtoolsModule.java index d391f62baa0..178b254ca33 100644 --- a/devtools/src/org/labkey/devtools/DevtoolsModule.java +++ b/devtools/src/org/labkey/devtools/DevtoolsModule.java @@ -18,6 +18,7 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.exp.property.Domain; +import org.labkey.api.mcp.McpService; import org.labkey.api.module.CodeOnlyModule; import org.labkey.api.module.ModuleContext; import org.labkey.api.security.AuthenticationManager; @@ -71,6 +72,7 @@ protected void init() @Override public void doStartup(ModuleContext moduleContext) { + McpService.get().register(new TestController.DocumentationMCP()); } @Override diff --git a/devtools/src/org/labkey/devtools/TestController.java b/devtools/src/org/labkey/devtools/TestController.java index 5c8cf796240..ea0776edc9d 100644 --- a/devtools/src/org/labkey/devtools/TestController.java +++ b/devtools/src/org/labkey/devtools/TestController.java @@ -19,6 +19,8 @@ import jakarta.servlet.http.HttpServletResponse; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; +import org.json.JSONArray; +import org.json.JSONObject; import org.labkey.api.action.ApiResponse; import org.labkey.api.action.ApiSimpleResponse; import org.labkey.api.action.ConfirmAction; @@ -28,8 +30,13 @@ import org.labkey.api.action.SimpleResponse; import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.announcements.CommSchema; +import org.labkey.api.collections.LabKeyCollectors; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; import org.labkey.api.mcp.AbstractAgentAction; import org.labkey.api.mcp.McpService; import org.labkey.api.security.CSRF; @@ -52,6 +59,7 @@ import org.labkey.api.util.HtmlString; import org.labkey.api.util.HtmlStringBuilder; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; import org.labkey.api.util.URLHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.HtmlView; @@ -64,7 +72,11 @@ import org.labkey.api.view.template.ClientDependency; import org.labkey.api.view.template.PageConfig; import org.labkey.api.wiki.WikiService; +import org.springframework.ai.chat.model.ToolContext; import org.springframework.ai.document.Document; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.ai.vectorstore.SimpleVectorStore; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.dao.PessimisticLockingFailureException; @@ -76,14 +88,17 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Gatherers; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.labkey.api.util.DOM.Attribute.name; import static org.labkey.api.util.DOM.Attribute.src; import static org.labkey.api.util.DOM.Attribute.style; @@ -1382,11 +1397,11 @@ public boolean handlePost(Object o, BindException errors) count.incrementAndGet(); var metadata = Map.of( "Content-Type", "text/html", - "filename", wiki.name() + ".html", + "filename", wiki.name() + ".html", // CONSIDER add path information "title", (Object)wiki.title(), "source", wikiBase.clone().addParameter("name",wiki.name()).getURIString() ); - return new Document(wiki.entityId(), wiki.html().toString(), metadata); + return new Document("documentation/"+wiki.name(), wiki.html().toString(), metadata); }) .gather(Gatherers.windowFixed(50)) .forEach(vs); @@ -1404,4 +1419,108 @@ public boolean handlePost(Object o, BindException errors) } } } + + public static class DocumentationMCP implements McpService.McpImpl + { + @Tool(description = "List of available documents from the LabKey user and administration manuals.") + @RequiresNoPermission + JSONObject listDocuments(ToolContext toolContext) + { + Container documentsContainer = ContainerManager.getForPath("/Documentation"); + if (null == documentsContainer) + return new JSONObject(Map.of("error","There is no /Documentation project on this server")); + + // CONSIDER include hierarchy or paths + // TODO WikiService doesn't expose this, just do a query for now (even though this info is cached) + TableInfo currentWikiVersions = CommSchema.getInstance().getSchema().getTable("CurrentWikiVersions"); + SimpleFilter filter = SimpleFilter.createContainerFilter(documentsContainer); + Collection> rows = new TableSelector(currentWikiVersions, Set.of("Name","Title","RowId","Parent"), filter, null).getMapCollection(); + + JSONArray array = new JSONArray(); + for (var row : rows) + { + array.put(new JSONObject(row)); + } + var ret = new JSONObject(); + ret.put("Version", "26.3"); + ret.put("Documents", array); + return ret; + } + + @Tool(description = "Return the entire document from the LabKey documentation using the `id` as returned by `searchDocumentation`.") + @RequiresNoPermission + JSONObject retrieveDocument( + ToolContext context, + @ToolParam(description = "Id of the document to return") String id) + { + WikiService service = Objects.requireNonNull(WikiService.get()); + Container documentsContainer = ContainerManager.getForPath("/Documentation"); + if (null == documentsContainer) + return new JSONObject(Map.of("error","There is not /Documentation project on this server")); + + ActionURL wikiBase = new ActionURL("wiki","page",documentsContainer); + var path = Path.parse(id); + var name = path.getName(); + var wiki = service.getRenderedWiki(documentsContainer, name); + if (null == wiki) + throw new NotFoundException(); + + var ret = new JSONObject(); + ret.put("Content-Type", "text/html"); + ret.put("filename", wiki.name() + ".html"); + ret.put("id", "documentation/" + wiki.name()); + ret.put("title", wiki.title()); + ret.put("source", wikiBase.clone().addParameter("name",wiki.name()).getURIString()); + ret.put("contents", wiki.html().toString()); + return ret; + } + + @Tool(description = "Search the LabKey documentation for documents semantically similar to a natural language query. " + + "Returns matching documents with their content, metadata (title, source URL, content type), and similarity scores.") + @RequiresNoPermission + JSONObject searchDocumentation( + ToolContext context, + @ToolParam(description = "Natural language search query describing what you're looking for") String query, + @ToolParam(required = false, description = "Maximum number of results to return, defaults to 5") String topK) + { + VectorStore vs = McpService.get().getVectorStore(); + if (vs == null) + throw new IllegalStateException("Vector store is not available. An embedding model may not be configured."); + + int k = 5; + if (isNotBlank(topK)) + { + try { k = Math.clamp(Integer.parseInt(topK), 1, 20); } + catch (NumberFormatException ignored) {} + } + + SearchRequest request = SearchRequest.builder() + .query(query) + .topK(k) + .build(); + + List results = vs.similaritySearch(request); + + var docs = results.stream() + .map(doc -> { + var obj = new JSONObject(); + obj.put("id", doc.getId()); + String text = doc.getText(); + if (text != null && text.length() > 2000) + text = text.substring(0, 2000) + "..."; + obj.put("content", text); + obj.put("metadata", new JSONObject(doc.getMetadata())); + if (doc.getScore() != null) + obj.put("score", doc.getScore()); + return obj; + }) + .collect(LabKeyCollectors.toJSONArray()); + + return new JSONObject(Map.of( + "query", query, + "resultCount", results.size(), + "results", docs + )); + } + } } From 9b9b79b90cd2ce7c7fdacad884ddc1ffcc45deeb Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Mon, 11 May 2026 16:14:39 -0700 Subject: [PATCH 2/8] CoreMCP listModules() --- core/src/org/labkey/core/CoreMcp.java | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/core/src/org/labkey/core/CoreMcp.java b/core/src/org/labkey/core/CoreMcp.java index fa18a2a55c1..ceff5a4df00 100644 --- a/core/src/org/labkey/core/CoreMcp.java +++ b/core/src/org/labkey/core/CoreMcp.java @@ -4,11 +4,13 @@ import io.modelcontextprotocol.spec.McpSchema.ReadResourceResult; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; import org.json.JSONObject; import org.labkey.api.collections.LabKeyCollectors; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.mcp.McpService; +import org.labkey.api.module.ModuleLoader; import org.labkey.api.security.RequiresNoPermission; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.User; @@ -130,6 +132,27 @@ String setContainer(ToolContext context, @ToolParam(description = "Container pat return message; } + + // TODO replace/augment with available feature list + @Tool(description = "List the modules installed on this server, this may be useful in inferring the available funcitonality. For instance, " + + "the presence of the `premium` module implies the availability of premium featues.") + @RequiresNoPermission + public String listModules(ToolContext context) + { + JSONArray modules = new JSONArray(); + ModuleLoader.getInstance().getModules().stream() + .map(module -> { + JSONObject obj = new JSONObject(); + obj.put("name", module.getName()); + if (StringUtils.isNotEmpty(module.getLabel())) + obj.put("label", module.getLabel()); + return obj; + }) + .forEach(modules::put); + return new JSONObject(Map.of("modules",modules)).toString(); + } + + @McpResource( uri = "resource://org/labkey/core/FileBasedModules.md", mimeType = "application/markdown", From e5136f03997851e13c9d96173e6093c019c71914 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Mon, 11 May 2026 16:41:48 -0700 Subject: [PATCH 3/8] pgvector store support local embedding --- api/src/org/labkey/api/mcp/McpService.java | 2 + .../org/labkey/api/mcp/NoopMcpService.java | 5 +++ .../org/labkey/devtools/TestController.java | 42 ++++++++++++------- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/api/src/org/labkey/api/mcp/McpService.java b/api/src/org/labkey/api/mcp/McpService.java index 6ad04872537..85d080e2b27 100644 --- a/api/src/org/labkey/api/mcp/McpService.java +++ b/api/src/org/labkey/api/mcp/McpService.java @@ -183,4 +183,6 @@ default List sendMessageEx(ChatClient chat, String message) * CONSIDER: Is it possible to implement VectorStoreRetriever wrapper for SearchService??? */ VectorStore getVectorStore(); + + void saveVectorStore(); } diff --git a/api/src/org/labkey/api/mcp/NoopMcpService.java b/api/src/org/labkey/api/mcp/NoopMcpService.java index f6f63534ce0..d2d5fa37d3e 100644 --- a/api/src/org/labkey/api/mcp/NoopMcpService.java +++ b/api/src/org/labkey/api/mcp/NoopMcpService.java @@ -84,4 +84,9 @@ public VectorStore getVectorStore() { return null; } + + @Override + public void saveVectorStore() + { + } } diff --git a/devtools/src/org/labkey/devtools/TestController.java b/devtools/src/org/labkey/devtools/TestController.java index ea0776edc9d..27e2b42e804 100644 --- a/devtools/src/org/labkey/devtools/TestController.java +++ b/devtools/src/org/labkey/devtools/TestController.java @@ -17,6 +17,7 @@ package org.labkey.devtools; import jakarta.servlet.http.HttpServletResponse; +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.json.JSONArray; @@ -34,7 +35,9 @@ import org.labkey.api.collections.LabKeyCollectors; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.mcp.AbstractAgentAction; @@ -1401,15 +1404,22 @@ public boolean handlePost(Object o, BindException errors) "title", (Object)wiki.title(), "source", wikiBase.clone().addParameter("name",wiki.name()).getURIString() ); - return new Document("documentation/"+wiki.name(), wiki.html().toString(), metadata); + return new Document(wiki.entityId(), wiki.html().toString(), metadata); }) - .gather(Gatherers.windowFixed(50)) - .forEach(vs); + .forEach(d -> { + try + { + vs.accept(List.of(d)); + } + catch (IllegalArgumentException x) + { + LogManager.getLogger(TestController.class).info(d.getMetadata().get("filename"),x); + } + }); - var db = FileUtil.getTempDirectoryFileLike().resolveChild("VectorStore.database"); try { - ((SimpleVectorStore)vs).save(db.toNioPathForRead().toFile()); + McpService.get().saveVectorStore(); return true; } catch (Exception x) @@ -1424,11 +1434,11 @@ public static class DocumentationMCP implements McpService.McpImpl { @Tool(description = "List of available documents from the LabKey user and administration manuals.") @RequiresNoPermission - JSONObject listDocuments(ToolContext toolContext) + String listDocuments(ToolContext toolContext) { Container documentsContainer = ContainerManager.getForPath("/Documentation"); if (null == documentsContainer) - return new JSONObject(Map.of("error","There is no /Documentation project on this server")); + return new JSONObject(Map.of("error","There is no /Documentation project on this server")).toString(); // CONSIDER include hierarchy or paths // TODO WikiService doesn't expose this, just do a query for now (even though this info is cached) @@ -1444,23 +1454,23 @@ JSONObject listDocuments(ToolContext toolContext) var ret = new JSONObject(); ret.put("Version", "26.3"); ret.put("Documents", array); - return ret; + return ret.toString(); } @Tool(description = "Return the entire document from the LabKey documentation using the `id` as returned by `searchDocumentation`.") @RequiresNoPermission - JSONObject retrieveDocument( + String retrieveDocument( ToolContext context, @ToolParam(description = "Id of the document to return") String id) { WikiService service = Objects.requireNonNull(WikiService.get()); Container documentsContainer = ContainerManager.getForPath("/Documentation"); if (null == documentsContainer) - return new JSONObject(Map.of("error","There is not /Documentation project on this server")); + return new JSONObject(Map.of("error","There is not /Documentation project on this server")).toString(); ActionURL wikiBase = new ActionURL("wiki","page",documentsContainer); - var path = Path.parse(id); - var name = path.getName(); + var sql = new SQLFragment("SELECT Name FROM ").append(CommSchema.getInstance().getTableInfoPages(), "p").append(" WHERE EntityId = ").appendValue(id); + var name = new SqlSelector(CommSchema.getInstance().getSchema(), sql).getObject(String.class); var wiki = service.getRenderedWiki(documentsContainer, name); if (null == wiki) throw new NotFoundException(); @@ -1472,13 +1482,13 @@ JSONObject retrieveDocument( ret.put("title", wiki.title()); ret.put("source", wikiBase.clone().addParameter("name",wiki.name()).getURIString()); ret.put("contents", wiki.html().toString()); - return ret; + return ret.toString(); } @Tool(description = "Search the LabKey documentation for documents semantically similar to a natural language query. " + "Returns matching documents with their content, metadata (title, source URL, content type), and similarity scores.") @RequiresNoPermission - JSONObject searchDocumentation( + String searchDocumentation( ToolContext context, @ToolParam(description = "Natural language search query describing what you're looking for") String query, @ToolParam(required = false, description = "Maximum number of results to return, defaults to 5") String topK) @@ -1516,11 +1526,13 @@ JSONObject searchDocumentation( }) .collect(LabKeyCollectors.toJSONArray()); - return new JSONObject(Map.of( + var ret = new JSONObject(Map.of( "query", query, "resultCount", results.size(), "results", docs )); +// LogManager.getLogger(TestController.class).info("Search: " + query + "\nResult: " +ret); + return ret.toString(); } } } From 2ab8e5f6ebb386e4bb328d1a68bb1bc219ec3a99 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Thu, 21 May 2026 11:10:00 -0700 Subject: [PATCH 4/8] error message --- devtools/src/org/labkey/devtools/TestController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devtools/src/org/labkey/devtools/TestController.java b/devtools/src/org/labkey/devtools/TestController.java index 27e2b42e804..5863e2100a1 100644 --- a/devtools/src/org/labkey/devtools/TestController.java +++ b/devtools/src/org/labkey/devtools/TestController.java @@ -1386,7 +1386,7 @@ public boolean handlePost(Object o, BindException errors) throw new NotFoundException(); VectorStore vs = McpService.get().getVectorStore(); if (null == vs) - throw new NotFoundException("/Documentation project was not found"); + throw new NotFoundException("VectorStore not enabled."); ActionURL wikiBase = new ActionURL("wiki","page",documentsContainer); From 51b8aa4b173341983ddfcb00fda1c958800629af Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Thu, 21 May 2026 12:42:11 -0700 Subject: [PATCH 5/8] load full_index.json if present --- .../org/labkey/devtools/TestController.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/devtools/src/org/labkey/devtools/TestController.java b/devtools/src/org/labkey/devtools/TestController.java index 5863e2100a1..77b52c77bcc 100644 --- a/devtools/src/org/labkey/devtools/TestController.java +++ b/devtools/src/org/labkey/devtools/TestController.java @@ -17,6 +17,7 @@ package org.labkey.devtools; import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -32,6 +33,7 @@ import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; import org.labkey.api.announcements.CommSchema; +import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.LabKeyCollectors; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; @@ -99,7 +101,6 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Gatherers; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.labkey.api.util.DOM.Attribute.name; @@ -1440,16 +1441,29 @@ String listDocuments(ToolContext toolContext) if (null == documentsContainer) return new JSONObject(Map.of("error","There is no /Documentation project on this server")).toString(); + try + { + // markdown index with summaries + return IOUtils.resourceToString("org/labkey/devtools/FULL_INDEX.json", null, DevtoolsModule.class.getClassLoader()); + } + catch (Exception io) + { + //pass + } + // CONSIDER include hierarchy or paths // TODO WikiService doesn't expose this, just do a query for now (even though this info is cached) TableInfo currentWikiVersions = CommSchema.getInstance().getSchema().getTable("CurrentWikiVersions"); SimpleFilter filter = SimpleFilter.createContainerFilter(documentsContainer); - Collection> rows = new TableSelector(currentWikiVersions, Set.of("Name","Title","RowId","Parent"), filter, null).getMapCollection(); + Collection> rows = new TableSelector(currentWikiVersions, Set.of("Name","Title","RowId","Parent","EntityId"), filter, null).getMapCollection(); JSONArray array = new JSONArray(); for (var row : rows) { - array.put(new JSONObject(row)); + CaseInsensitiveHashMap copy = new CaseInsensitiveHashMap<>(row); + copy.put("id", String.valueOf(copy.get("EntityId"))); + copy.remove("EntityId"); + array.put(new JSONObject(copy)); } var ret = new JSONObject(); ret.put("Version", "26.3"); @@ -1478,7 +1492,7 @@ String retrieveDocument( var ret = new JSONObject(); ret.put("Content-Type", "text/html"); ret.put("filename", wiki.name() + ".html"); - ret.put("id", "documentation/" + wiki.name()); + ret.put("id", wiki.entityId()); ret.put("title", wiki.title()); ret.put("source", wikiBase.clone().addParameter("name",wiki.name()).getURIString()); ret.put("contents", wiki.html().toString()); From 7617edb78f900fb89f5be796cc10618249f6f445 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Thu, 21 May 2026 14:30:24 -0700 Subject: [PATCH 6/8] paginiation --- .../org/labkey/devtools/TestController.java | 72 +++++++++++++------ 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/devtools/src/org/labkey/devtools/TestController.java b/devtools/src/org/labkey/devtools/TestController.java index 77b52c77bcc..b5e808a2a38 100644 --- a/devtools/src/org/labkey/devtools/TestController.java +++ b/devtools/src/org/labkey/devtools/TestController.java @@ -1433,41 +1433,67 @@ public boolean handlePost(Object o, BindException errors) public static class DocumentationMCP implements McpService.McpImpl { + static JSONObject full_index = null; + + static + { + try + { + full_index = new JSONObject(IOUtils.resourceToString("org/labkey/devtools/FULL_INDEX.json", null, DevtoolsModule.class.getClassLoader())); + } + catch(Exception x) + { + } + } + + @Tool(description = "List of available documents from the LabKey user and administration manuals.") @RequiresNoPermission - String listDocuments(ToolContext toolContext) + String listDocuments(ToolContext toolContext, + @ToolParam(description = "Index to start listing for paginatation (staring at 0)") Integer start, + @ToolParam(description = "Count of listings to return for pagination") Integer count) { Container documentsContainer = ContainerManager.getForPath("/Documentation"); if (null == documentsContainer) return new JSONObject(Map.of("error","There is no /Documentation project on this server")).toString(); - try + if (null == full_index) { - // markdown index with summaries - return IOUtils.resourceToString("org/labkey/devtools/FULL_INDEX.json", null, DevtoolsModule.class.getClassLoader()); - } - catch (Exception io) - { - //pass + // CONSIDER include hierarchy or paths + // TODO WikiService doesn't expose this, just do a query for now (even though this info is cached) + TableInfo currentWikiVersions = CommSchema.getInstance().getSchema().getTable("CurrentWikiVersions"); + SimpleFilter filter = SimpleFilter.createContainerFilter(documentsContainer); + Collection> rows = new TableSelector(currentWikiVersions, Set.of("Name","Title","RowId","Parent","EntityId"), filter, null).getMapCollection(); + + JSONArray array = new JSONArray(); + for (var row : rows) + { + CaseInsensitiveHashMap copy = new CaseInsensitiveHashMap<>(row); + copy.put("id", String.valueOf(copy.get("EntityId"))); + copy.remove("EntityId"); + array.put(new JSONObject(copy)); + } + var j = new JSONObject(); + j.put("pages", array); + full_index = j; } - // CONSIDER include hierarchy or paths - // TODO WikiService doesn't expose this, just do a query for now (even though this info is cached) - TableInfo currentWikiVersions = CommSchema.getInstance().getSchema().getTable("CurrentWikiVersions"); - SimpleFilter filter = SimpleFilter.createContainerFilter(documentsContainer); - Collection> rows = new TableSelector(currentWikiVersions, Set.of("Name","Title","RowId","Parent","EntityId"), filter, null).getMapCollection(); + int index = start instanceof Integer i && i >= 0 ? i : 0; + int num = count instanceof Integer i && i >= 0 ? i : Integer.MAX_VALUE; + + JSONArray pages = full_index.getJSONArray("pages"); + int total = pages.length(); + int end = (int) Math.min((long) index + num, total); + + JSONArray subset = new JSONArray(); + for (int i = index; i < end; i++) + subset.put(pages.get(i)); - JSONArray array = new JSONArray(); - for (var row : rows) - { - CaseInsensitiveHashMap copy = new CaseInsensitiveHashMap<>(row); - copy.put("id", String.valueOf(copy.get("EntityId"))); - copy.remove("EntityId"); - array.put(new JSONObject(copy)); - } var ret = new JSONObject(); - ret.put("Version", "26.3"); - ret.put("Documents", array); + ret.put("total", total); + ret.put("start", index); + ret.put("count", subset.length()); + ret.put("pages", subset); return ret.toString(); } From 979d7983b1bf6c1f9580b4c17a7d07c629d3bd06 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 27 May 2026 14:17:17 -0700 Subject: [PATCH 7/8] WikiService.getWikiMarkdown() --- api/src/org/labkey/api/ApiModule.java | 2 ++ .../org/labkey/api/wiki/WikiRendererType.java | Bin 3537 -> 18579 bytes api/src/org/labkey/api/wiki/WikiService.java | 9 +++++++ .../org/labkey/devtools/TestController.java | 13 ++++------- wiki/src/org/labkey/wiki/WikiManager.java | 22 ++++++++++++++++++ 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/api/src/org/labkey/api/ApiModule.java b/api/src/org/labkey/api/ApiModule.java index d613909b2b6..5eecd9f5647 100644 --- a/api/src/org/labkey/api/ApiModule.java +++ b/api/src/org/labkey/api/ApiModule.java @@ -186,6 +186,7 @@ import org.labkey.api.view.ViewServlet; import org.labkey.api.view.WebPartFactory; import org.labkey.api.webdav.WebdavResolverImpl; +import org.labkey.api.wiki.WikiRendererType; import org.labkey.api.writer.ContainerUser; import org.labkey.filters.ContentSecurityPolicyFilter; @@ -527,6 +528,7 @@ public void registerServlets(ServletContext servletCtx) UserManager.TestCase.class, ViewCategoryManager.TestCase.class, WebdavResolverImpl.TestCase.class, + WikiRendererType.TestCase.class, WorkbookContainerType.TestCase.class, WriteableLookAndFeelProperties.TestCase.class ); diff --git a/api/src/org/labkey/api/wiki/WikiRendererType.java b/api/src/org/labkey/api/wiki/WikiRendererType.java index 7260049370c90c4ffd3336223a63220f5a55f90f..bf7c2e916a762ed205286d1bf5e0c7c6885a7193 100644 GIT binary patch literal 18579 zcmd5^>2ll15zcQ{`5ng4Rs_-#V0pK4$TS`5aICBjmqf3#B`IEz7>ck!0Ks8Nv82kc zJU}W>m?z2CGXoMANkF!@!rC=KV5YmLuj%e-$ZS7;#M*4z4{k!|VjMGl*kaG0J^RD@ z^Jg1>Wc$|OM}C9HVc>^W?D!rH7gP3~A@?F~v&6G`$l@_)y}%mcOFda*Kk+ca6!zSB z#&jB>sYhDtg@AG6Cv0NfFwc)!5^(@=A~ted&iM6^2QhOzHuNWf>sa29vnwYa3q%Tx zA%Og>fcS&h!gvcaf*a@iG|j84E5i~*8-93Ux^hX8xxcr)dw81ErqKsB#g=Ic|^!O1N zeo4TKO-m;44oaEvSG(t%n$egxY5FcyzLMt%|)f1!WzsPw?|Z5|oBE}w8OopExwGa?2>AuSHV&*9P9`IV8ALhYtrf>mQlNbN$ZShz)pOm{BiPPp z#6#!{i*EuBV;Hdz#!K1<&*8=5$=Hh75S9lk@YsOE>WS~tvfuO%_Q`~NVN=BJNSe_* zXu&4yTa(aY2RHD2!YFpEm~XOPa=|u!zecY0cXS(u5F`V555^%(_8!=!Y>Q7 z+!c59R-V2BA874rQ68o%b?$U^%@j^wcpwuV$Rr&Q{~TL3_XT|8V-(2UD#OA?bSp*= z;0bzA%t;V$`yM0{_x%GaytMr*Pgi4<`Ym6MX)-W2w1&`9xF$Am{oy6;oMA+JCbl0) zbjZiP3$2Z?|Bxa(VXEKo*deD)H_ta(At1+o0s|8poa4EUhrJ=Rya=1ZBw{)oJhT>j z8hq9zkH?{(T#VNk#$85h*vyBaFP;SfTw03^qwpc??#Kmq1@&8!_?L{NBm9XN($z(f zP?)W(pA9GMCA3pmxCx>2k#v9d;9w>V

Co-!T& zqJWD@Dz_4c*vNv=kFgnW8}bugLQ$BN37%#jDI;|;fLXm+&8o;-PitpTVxE~i%%`&plS$7X>Q#*XE}CTPQ=&7k7r zjF?Uo8&Qnm(R&1Hx<-&8n(7$@M6B^8(^ghi7)e(`NCuGzB=jHVNtL?@vge?zi?Isf z_YXi=6JHg&=kw5!eEiV)2MiR94->br_oV_T@*#8rs0E@J%Z+qx9iLk4r*Rc-gRp>- zqfmwnE*u^l=L&h>5qu|ph(sp!{%ovNYL38*#Lo1Dq+dr(h@cIlijxlzTuq$S_!dd- z5IdyEiTpg{+LRPoZU`>jz}3kubDexXQVrx@joH?2*Jm4smhDS$t=c*v_dT&gZnD+Y z7@5WDD#ePXN!NI&OWz=T0;N$%LMc^zcf23XS%~rVxu8MbIoENcAv@2`PW98j z-JYGDpFc(5fQzV&8|m+JBJcFOjdT6G=Pf~C^)jP@+OdUKZ>2wIuc>lc*f`5@6rTe` ztrJufca$bAsdqr{D|$Aad} z=9twn1W>1cyE}i1fQr^XXWgLp7`96rVKy4IqN%mA5l8)|Vha=-Hv(4XfR1M`@YN}Y zD!BdhX~jB7x2KDc7lxuf54XKzxXsrnb!Rr}-s3sm zZVpBQO;J4o0Su%P@V2OH^|2od=&?5kU`o_%+loDdY!X>&!N;XT>3bTI7BQ+)ECwt< zxL_3oT?y21}XacE zK@yFLya!R>I!m!F5myEKmau|mh&>-QRVe&Rs$YyDwrl+v6_V0*sXE0v1HaqshZ;aL zQe;!Xq2vNA3)MO0X=Xm8kV<4mQNZ~n41>L5nn-Up5qK|`d|@X`k)_|2ieW)7e*0VY z1nQ3(R|s7W`ABOSPPB^xA{^BM^GX3{R&)9=^^C)%%0b97h|av*BM00r@8dM5fgeZg?S4Z){{P@rztCmXa3bWtpUjM{ ziK>&hW-|l79HG;Q>}QTLq(i1En)S?I*(lF-sJkrh>IFNW-q&QVnUYoYT}f(emp7Jp z+|-(pSvPYWs??coy|=+FjQZAfZU+zWBnwf`kzd8_yQ+Or#p&=|nasC#Zr9)aYyShC zz<=1^JN)rv<~|BqKF5R9J%M}cQa=L)ao-@>iwSBr%Ssd1gWmCvJ4f#iztyVJ3f?zZ z)5LBFbWz?|kR_R3!|X0y=`_PDYGOuP?-1Tdjw|i3sF+U&d@39+9@N$tUpRlA&cLBK z+d?+9&PS-f&=De?M~Y6J>|r>XEKYe@IL;;p2;V?E!3V=~Oq%hgqWVwvbHsQ0uk&;hLc|BYjg^3ky{3hQoxJa zGSk1bC?Yadb(<)p-(|66mOukgp)B)d4oamGL)Sth7qu4=%hA~dE3qRcJCRC?`X?GV z<3hFs5#z~Qz)~mq**Zso_i9yC9YD^e!K=AGYqzib(6!-F#F^HI$e-{JvGOK`{pGXouL9r*8y)jisBAvCxOm*ivFw0j;oKHv~ErI?=m?lWF6K zJR@Gf%p8^4N_wo!jy#TBNOtxtrc-;8n}kzsS(+|xG6hUCRQ)TzHmVsenpGQD1{o6A z`YXfvw>oEMeKD~TV{^e!!U>-?vxI_*=Xr%3|21{g)aBBO8eT~(DNi0e-2n=D>{%Pf zN;;tdmUcR)re?vK6;*SQz>&W5(t}#rcjWR<++A{3JL5d9UU`)18BQ&=Z{vWKPfE#w zKB}jvI55o60xz0Mq|;T9;p^})eDT42N8ig-pDWRp${=7fiY_p zWRn>?(r}8-CJ9;!(E&@H<2?L?!*`T!k(YqAB|96VMYBvZstziaA^6R8d7oMf7w|8y zUZ?xwm4$P?nwXZR&S#01^i#|4D8Q(RR71u=LzU_f4me9{?4l?Qt-)vqM9_sMc#cDQ zA8${HUVHF*rXe)xS6^1Mbv2jccG^<7K$b;Y5bd0IC%gN*+kG4YygoiUKzC_k-_U_0 zQX?m@yjTwUH7uf}@uF=YtKKuoiLFSZMW^9#*_Nt|Xvt!~)S9MJcq`jkBe_*jijVfr z`CbY-EXOc91%m^i#$xQQ=r3nd0ES`63%Sn2qdxf!XuRwl=Jbw8=2I62u*|Mo{@iVW)(ZXJSG4C5NK0RJ)Yd)P_hj6@_iSBf#CZl z2v$&e;&jkii(5Z*X*6*BFb79d@%GUeOgU4&LJUKAb{-9R1o2VIf)QwzUD&@qE-rv9J?U5sb>q>?Gp#yVKhwIhPS7!}E{=k1>t>-O2v~+|(_+XH zwk<72yB_6UjSj@wb7bG+A{YyF$f-F8J6T~{!JRhZwhBO+6Ktu)2QTe>1DF^%X zw;Q%`kQJS9+SKv>QFe*dg*u5IN$T=QqRKtQgtOa;L;P=ddmR)1($&`VYFl2JG{ls{ z7b>S%z_hp!r7Kwz@(UBZ#qQ&H5%0|Bw6#o|;_Iw#-jiL3rTbfR?r&?E`T^-j`k_|F z5|rK+#y@mJJ;Y%{xbH{2kGQgi%)|@bIHaYMOHC`U{Rx~~9Qt-5vW@cdKK*49{D*qC z`0ErXJK@bPb}mvAmrB;{>%oa_mj`dAaTgG!>Mzy;=PyydYOhoX7mZ* zN@Tia2v|v`vP#TO4N@%kIDNsVPJJ^>Ih2(MffOwS3NFriZMKO&@S^%Qvpj=gYE$0O z55*k1B)-tMS}$GF3h7%V7aFY->QG*!%t&N4O)ezcu;TjgTtt@CZMuSk$Si->P)g5) zzU6VKrH#?Iq4Xo4t0L^*!n*;K!>AQe8YAQ)vI4c;(-{-Cw z6h&wgF{OFBMhhXPXGA0Tj$1@7J;D-%Sm>B{i8P4TrLvf @Override public ModelAndView getConfirmView(Object o, BindException errors) { - var db = FileUtil.getTempDirectoryFileLike().resolveChild("VectorStore.database"); HtmlStringBuilder message = HtmlStringBuilder.of(); message.append("This will add the contents of /Documention wikis to the vector store.").append(HtmlString.BR); message.append("This may take a few minutes."); - if (db.exists()) - message.unsafeAppend("

").append("I see a vector store file already exists. Just FYI."); return new HtmlView(message); } @@ -1394,23 +1391,23 @@ public boolean handlePost(Object o, BindException errors) WikiService service = Objects.requireNonNull(WikiService.get()); List all = service.getNames(documentsContainer); all.stream() - .map(name -> service.getRenderedWiki(documentsContainer, name)) + .map(name -> service.getWikiMarkdown(documentsContainer, name)) .filter(Objects::nonNull) .map(wiki -> { - count.incrementAndGet(); var metadata = Map.of( - "Content-Type", "text/html", - "filename", wiki.name() + ".html", // CONSIDER add path information + "Content-Type", "text/markdown", + "filename", wiki.name() + ".md", "title", (Object)wiki.title(), "source", wikiBase.clone().addParameter("name",wiki.name()).getURIString() ); - return new Document(wiki.entityId(), wiki.html().toString(), metadata); + return new Document(wiki.entityId(), wiki.markdown(), metadata); }) .forEach(d -> { try { vs.accept(List.of(d)); + count.incrementAndGet(); } catch (IllegalArgumentException x) { diff --git a/wiki/src/org/labkey/wiki/WikiManager.java b/wiki/src/org/labkey/wiki/WikiManager.java index c487e0e1259..316b0bd49e8 100644 --- a/wiki/src/org/labkey/wiki/WikiManager.java +++ b/wiki/src/org/labkey/wiki/WikiManager.java @@ -869,6 +869,28 @@ public RenderedWiki getRenderedWiki(Container c, String name) } } + @Override + public WikiMarkdown getWikiMarkdown(Container c, String name) + { + if (null == c || null == name) + return null; + + try + { + Wiki wiki = WikiSelectManager.getWiki(c, name); + if (null == wiki) + return null; + WikiVersion version = wiki.getLatestVersion(); + String body = version.getBody(); + String markdown = version.getRendererTypeEnum().bestAttemptConvertToMarkdown(null == body ? "" : body); + return new WikiMarkdown(name, version.getTitle(), markdown, wiki.getEntityId()); + } + catch (Exception x) + { + throw new RuntimeException(x); + } + } + @Override public void insertWiki(User user, Container c, String name, String body, WikiRendererType renderType, String title) { From 676f69a9b61aab7aac7379093c6ad2cb37f2e805 Mon Sep 17 00:00:00 2001 From: labkey-matthewb Date: Wed, 27 May 2026 15:28:33 -0700 Subject: [PATCH 8/8] TokenTextSplitter --- api/src/org/labkey/api/mcp/McpService.java | 10 ++++++++++ api/src/org/labkey/api/mcp/NoopMcpService.java | 6 ++++++ devtools/src/org/labkey/devtools/TestController.java | 6 +++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/api/src/org/labkey/api/mcp/McpService.java b/api/src/org/labkey/api/mcp/McpService.java index 85d080e2b27..011d3800712 100644 --- a/api/src/org/labkey/api/mcp/McpService.java +++ b/api/src/org/labkey/api/mcp/McpService.java @@ -19,6 +19,7 @@ import org.springframework.ai.support.ToolCallbacks; import org.springframework.ai.tool.ToolCallback; import org.springframework.ai.tool.ToolCallbackProvider; +import org.springframework.ai.document.Document; import org.springframework.ai.vectorstore.VectorStore; import java.util.Arrays; @@ -184,5 +185,14 @@ default List sendMessageEx(ChatClient chat, String message) */ VectorStore getVectorStore(); + /** + * Adds documents to the vector store, automatically splitting any document whose token + * count exceeds the embedding model's input limit. Prefer this over + * {@code getVectorStore().add(...)} for indexing — it prevents the + * {@code IllegalArgumentException} that {@code TokenCountBatchingStrategy} throws on + * oversized inputs. + */ + void addDocuments(List documents); + void saveVectorStore(); } diff --git a/api/src/org/labkey/api/mcp/NoopMcpService.java b/api/src/org/labkey/api/mcp/NoopMcpService.java index d2d5fa37d3e..57583655059 100644 --- a/api/src/org/labkey/api/mcp/NoopMcpService.java +++ b/api/src/org/labkey/api/mcp/NoopMcpService.java @@ -8,6 +8,7 @@ import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.chat.model.ToolContext; import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.document.Document; import org.springframework.ai.vectorstore.VectorStore; import java.util.List; @@ -85,6 +86,11 @@ public VectorStore getVectorStore() return null; } + @Override + public void addDocuments(List documents) + { + } + @Override public void saveVectorStore() { diff --git a/devtools/src/org/labkey/devtools/TestController.java b/devtools/src/org/labkey/devtools/TestController.java index 25b638cdb19..548e3e5a2bc 100644 --- a/devtools/src/org/labkey/devtools/TestController.java +++ b/devtools/src/org/labkey/devtools/TestController.java @@ -1382,8 +1382,8 @@ public boolean handlePost(Object o, BindException errors) Container documentsContainer = ContainerManager.getForPath("/Documentation"); if (null == documentsContainer) throw new NotFoundException(); - VectorStore vs = McpService.get().getVectorStore(); - if (null == vs) + McpService mcp = McpService.get(); + if (null == mcp.getVectorStore()) throw new NotFoundException("VectorStore not enabled."); ActionURL wikiBase = new ActionURL("wiki","page",documentsContainer); @@ -1406,7 +1406,7 @@ public boolean handlePost(Object o, BindException errors) .forEach(d -> { try { - vs.accept(List.of(d)); + mcp.addDocuments(List.of(d)); count.incrementAndGet(); } catch (IllegalArgumentException x)