diff --git a/README.md b/README.md index a208cd6e..9240fe2a 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,7 @@ graph TD - `AUTO_CONTEXT_COMPRESS_ENABLED`: Whether to enable AI-powered intelligent context compression for long conversations (default: false) - `AUTO_CONTEXT_COMPRESS_TOKEN_LIMIT`: Token threshold to trigger context compression. Required when compression is enabled (default: 100000) - `AUTO_CONTEXT_COMPRESS_MAX_TOKEN_LIMIT`: Maximum allowed token limit, ensures the token limit doesn't exceed model capabilities (default: 200000) +- `UNDERSTAND_QUICKLY_TOKEN`: Optional GitHub PAT (`Repository dispatches: write` on `looptech-ai/understand-quickly` only). When set, OpenDeepWiki stamps `metadata.{tool, tool_version, generated_at, commit}` into the generated `graphify-out/graph.json` and fires a `repository_dispatch` so the [understand-quickly](https://github.com/looptech-ai/understand-quickly) registry resyncs the entry. Opt-in; default behavior is unchanged. See [`looptech-ai/uq-publish-action@v0.1.0`](https://github.com/looptech-ai/uq-publish-action) for the recommended CI step. **Intelligent Context Compression Features:** Uses **Prompt Encoding Compression** - an ultra-dense, structured format that achieves 90%+ compression while preserving ALL critical information. diff --git a/src/OpenDeepWiki/Program.cs b/src/OpenDeepWiki/Program.cs index 48a890fc..bf230d60 100644 --- a/src/OpenDeepWiki/Program.cs +++ b/src/OpenDeepWiki/Program.cs @@ -231,6 +231,22 @@ }); builder.Services.AddScoped(); + // Opt-in publish to the understand-quickly registry of code-knowledge graphs. + // https://github.com/looptech-ai/understand-quickly + builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection(UnderstandQuicklyOptions.SectionName)) + .PostConfigure(options => + { + var token = Environment.GetEnvironmentVariable("UNDERSTAND_QUICKLY_TOKEN"); + if (!string.IsNullOrWhiteSpace(token)) + { + options.Token = token; + options.Enabled = true; + } + }); + builder.Services + .AddHttpClient(); + // 配置 Wiki Generator builder.Services.AddOptions() .Bind(builder.Configuration.GetSection(WikiGeneratorOptions.SectionName)) diff --git a/src/OpenDeepWiki/Services/Graphify/GraphifyArtifactWorker.cs b/src/OpenDeepWiki/Services/Graphify/GraphifyArtifactWorker.cs index 355b65d8..3876e2cd 100644 --- a/src/OpenDeepWiki/Services/Graphify/GraphifyArtifactWorker.cs +++ b/src/OpenDeepWiki/Services/Graphify/GraphifyArtifactWorker.cs @@ -91,6 +91,7 @@ private async Task ProcessPendingArtifactsAsync(CancellationToken cancellationTo var context = scope.ServiceProvider.GetRequiredService(); var repositoryAnalyzer = scope.ServiceProvider.GetRequiredService(); var runner = scope.ServiceProvider.GetRequiredService(); + var publisher = scope.ServiceProvider.GetService(); var processingLogService = scope.ServiceProvider.GetService(); while (!cancellationToken.IsCancellationRequested) @@ -117,6 +118,7 @@ await ProcessArtifactAsync( context, repositoryAnalyzer, runner, + publisher, processingLogService, cancellationToken); } @@ -127,6 +129,7 @@ private async Task ProcessArtifactAsync( IContext context, IRepositoryAnalyzer repositoryAnalyzer, IGraphifyCliRunner runner, + IUnderstandQuicklyPublisher? publisher, IProcessingLogService? processingLogService, CancellationToken cancellationToken) { @@ -187,6 +190,27 @@ await processingLogService.LogAsync( $"Graphify generation complete: {branch.BranchName}, duration: {stopwatch.ElapsedMilliseconds}ms", cancellationToken: cancellationToken); } + + // Opt-in publish to understand-quickly. The publisher is a no-op when + // disabled; failures inside it are logged but never fail the artifact + // (the local graph.json is already written). + if (publisher != null && !string.IsNullOrWhiteSpace(repository.OrgName) && + !string.IsNullOrWhiteSpace(repository.RepoName)) + { + try + { + await publisher.PublishAsync( + result, + $"{repository.OrgName}/{repository.RepoName}", + cancellationToken); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "UnderstandQuickly publish failed for {Org}/{Repo} — graph.json is still local.", + repository.OrgName, repository.RepoName); + } + } } catch (Exception ex) { diff --git a/src/OpenDeepWiki/Services/Graphify/UnderstandQuicklyPublisher.cs b/src/OpenDeepWiki/Services/Graphify/UnderstandQuicklyPublisher.cs new file mode 100644 index 00000000..8835c1b3 --- /dev/null +++ b/src/OpenDeepWiki/Services/Graphify/UnderstandQuicklyPublisher.cs @@ -0,0 +1,226 @@ +using System.Net.Http.Headers; +using System.Reflection; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.Options; + +namespace OpenDeepWiki.Services.Graphify; + +/// +/// Opt-in publish to the understand-quickly registry of code-knowledge graphs. +/// https://github.com/looptech-ai/understand-quickly +/// +public class UnderstandQuicklyOptions +{ + public const string SectionName = "UnderstandQuickly"; + + public bool Enabled { get; set; } = false; + public string? Token { get; set; } + public string RegistryRepo { get; set; } = "looptech-ai/understand-quickly"; + public string Schema { get; set; } = "gitnexus@1"; +} + +public sealed record UnderstandQuicklyPublishResult( + bool MetadataStamped, + bool Dispatched, + string? Error = null); + +public interface IUnderstandQuicklyPublisher +{ + Task PublishAsync( + GraphifyRunResult result, + string repoSlug, + CancellationToken cancellationToken = default); +} + +public class UnderstandQuicklyPublisher : IUnderstandQuicklyPublisher +{ + private const string ToolName = "opendeepwiki"; + private const string DispatchEventType = "uq-publish"; + + private readonly UnderstandQuicklyOptions _options; + private readonly ILogger _logger; + private readonly HttpClient _httpClient; + + public UnderstandQuicklyPublisher( + IOptions options, + ILogger logger, + HttpClient httpClient) + { + _options = options.Value; + _logger = logger; + _httpClient = httpClient; + } + + public async Task PublishAsync( + GraphifyRunResult result, + string repoSlug, + CancellationToken cancellationToken = default) + { + if (!_options.Enabled) + { + return new UnderstandQuicklyPublishResult(false, false); + } + if (string.IsNullOrWhiteSpace(result.GraphJsonPath) || !File.Exists(result.GraphJsonPath)) + { + return new UnderstandQuicklyPublishResult(false, false, + $"graph.json not found at {result.GraphJsonPath}"); + } + + var toolVersion = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown"; + + JsonObject root; + try + { + await using (var read = File.OpenRead(result.GraphJsonPath)) + { + var node = await JsonNode.ParseAsync(read, cancellationToken: cancellationToken) + ?? new JsonObject(); + root = node.AsObject(); + } + + var metadata = root["metadata"]?.AsObject() ?? new JsonObject(); + metadata["tool"] = ToolName; + metadata["tool_version"] = toolVersion; + metadata["generated_at"] = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"); + if (!string.IsNullOrWhiteSpace(result.CommitId)) + { + metadata["commit"] = result.CommitId; + } + root["metadata"] = metadata; + + await File.WriteAllTextAsync( + result.GraphJsonPath, + root.ToJsonString(new JsonSerializerOptions { WriteIndented = true }), + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to stamp metadata into {Path}", result.GraphJsonPath); + return new UnderstandQuicklyPublishResult(false, false, ex.Message); + } + + if (string.IsNullOrWhiteSpace(_options.Token)) + { + _logger.LogInformation( + "UnderstandQuickly: stamped {Path}; token unset, skipping registry dispatch.", + result.GraphJsonPath); + return new UnderstandQuicklyPublishResult(true, false); + } + if (string.IsNullOrWhiteSpace(repoSlug) || !repoSlug.Contains('/')) + { + return new UnderstandQuicklyPublishResult(true, false, $"invalid repo slug: '{repoSlug}'"); + } + + // The understand-quickly registry fetches graphs from raw.githubusercontent.com, + // so it expects a repo-relative path (e.g. "graphify-out/graph.json"), not the + // absolute on-disk path under . Fall back to the file name when + // OutputRoot can't be resolved. + var graphPath = ToRepoRelativePath(result.OutputRoot, result.GraphJsonPath); + + var payload = new JsonObject + { + ["event_type"] = DispatchEventType, + ["client_payload"] = new JsonObject + { + ["repo"] = repoSlug, + ["schema"] = _options.Schema, + ["graph_path"] = graphPath, + ["tool"] = ToolName, + ["tool_version"] = toolVersion, + ["commit"] = result.CommitId, + }, + }; + + try + { + using var request = new HttpRequestMessage(HttpMethod.Post, + $"https://api.github.com/repos/{_options.RegistryRepo}/dispatches") + { + Content = new StringContent(payload.ToJsonString(), Encoding.UTF8, "application/json"), + }; + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.Token); + request.Headers.UserAgent.ParseAdd($"{ToolName}/{toolVersion}"); + request.Headers.Add("X-GitHub-Api-Version", "2022-11-28"); + + using var response = await _httpClient.SendAsync(request, cancellationToken); + if (response.IsSuccessStatusCode) + { + _logger.LogInformation( + "UnderstandQuickly: dispatched to {Registry} for {Slug} (HTTP {Code}).", + _options.RegistryRepo, repoSlug, (int)response.StatusCode); + return new UnderstandQuicklyPublishResult(true, true); + } + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + _logger.LogInformation( + "UnderstandQuickly: {Slug} not in registry — register once with `npx @understand-quickly/cli add`.", + repoSlug); + return new UnderstandQuicklyPublishResult(true, false, "repo not registered"); + } + // Surface enough detail for ops triage on 401/403/422/etc. + string responseBody; + try + { + responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + } + catch + { + responseBody = string.Empty; + } + if (responseBody.Length > 500) + { + responseBody = responseBody[..500]; + } + _logger.LogWarning( + "UnderstandQuickly: dispatch to {Registry} for {Slug} returned HTTP {Code}. Body: {Body}", + _options.RegistryRepo, repoSlug, (int)response.StatusCode, responseBody); + return new UnderstandQuicklyPublishResult(true, false, $"HTTP {(int)response.StatusCode}"); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "UnderstandQuickly: dispatch failed; metadata still stamped."); + return new UnderstandQuicklyPublishResult(true, false, ex.Message); + } + } + + /// + /// Convert an absolute graph.json path under into + /// a forward-slash, repo-relative path. Avoids leaking server filesystem paths + /// into the registry payload (and into GitHub Action logs). + /// + public static string ToRepoRelativePath(string? outputRoot, string graphJsonPath) + { + if (string.IsNullOrWhiteSpace(graphJsonPath)) + { + return string.Empty; + } + if (!string.IsNullOrWhiteSpace(outputRoot)) + { + try + { + var rel = Path.GetRelativePath(outputRoot, graphJsonPath); + if (!rel.StartsWith("..", StringComparison.Ordinal) && + !Path.IsPathRooted(rel)) + { + return rel.Replace(Path.DirectorySeparatorChar, '/'); + } + } + catch (ArgumentException) + { + // Fall through to the file-name fallback. + } + } + return Path.GetFileName(graphJsonPath); + } +} diff --git a/tests/OpenDeepWiki.Tests/Services/Graphify/UnderstandQuicklyPublisherTests.cs b/tests/OpenDeepWiki.Tests/Services/Graphify/UnderstandQuicklyPublisherTests.cs new file mode 100644 index 00000000..1e355192 --- /dev/null +++ b/tests/OpenDeepWiki.Tests/Services/Graphify/UnderstandQuicklyPublisherTests.cs @@ -0,0 +1,182 @@ +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using Moq.Protected; +using OpenDeepWiki.Services.Graphify; +using Xunit; + +namespace OpenDeepWiki.Tests.Services.Graphify; + +public class UnderstandQuicklyPublisherTests +{ + private static GraphifyRunResult MakeResult(string graphPath, string commit = "deadbeef") => + new( + OutputRoot: Path.GetDirectoryName(graphPath)!, + EntryFilePath: Path.Combine(Path.GetDirectoryName(graphPath)!, "graph.html"), + GraphJsonPath: graphPath, + ReportPath: Path.Combine(Path.GetDirectoryName(graphPath)!, "GRAPH_REPORT.md"), + CommitId: commit, + LogOutput: string.Empty); + + private static string MakeTempDir() + { + var tmp = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tmp); + return tmp; + } + + private static void SafeDelete(string dir) + { + try + { + if (Directory.Exists(dir)) + { + Directory.Delete(dir, recursive: true); + } + } + catch + { + // best-effort cleanup + } + } + + [Fact] + public async Task PublishAsync_Disabled_NoOpAndNoStamp() + { + var tmp = MakeTempDir(); + try + { + var graph = Path.Combine(tmp, "graph.json"); + await File.WriteAllTextAsync(graph, "{\"nodes\":[],\"links\":[]}"); + + var options = Options.Create(new UnderstandQuicklyOptions { Enabled = false }); + var publisher = new UnderstandQuicklyPublisher( + options, NullLogger.Instance, new HttpClient()); + + var result = await publisher.PublishAsync(MakeResult(graph), "owner/repo"); + + Assert.False(result.MetadataStamped); + Assert.False(result.Dispatched); + + // File untouched. + var doc = JsonNode.Parse(await File.ReadAllTextAsync(graph)); + Assert.Null(doc!["metadata"]); + } + finally + { + SafeDelete(tmp); + } + } + + [Fact] + public async Task PublishAsync_EnabledNoToken_StampsButDoesNotDispatch() + { + var tmp = MakeTempDir(); + try + { + var graph = Path.Combine(tmp, "graph.json"); + await File.WriteAllTextAsync(graph, "{\"nodes\":[],\"links\":[]}"); + + var options = Options.Create(new UnderstandQuicklyOptions { Enabled = true }); + var handler = new Mock(); + handler.Protected() + .Setup>("SendAsync", + ItExpr.IsAny(), ItExpr.IsAny()) + .ThrowsAsync(new InvalidOperationException("dispatch should not be called")); + var publisher = new UnderstandQuicklyPublisher( + options, NullLogger.Instance, + new HttpClient(handler.Object)); + + var result = await publisher.PublishAsync(MakeResult(graph, "abc123"), "owner/repo"); + + Assert.True(result.MetadataStamped); + Assert.False(result.Dispatched); + Assert.Null(result.Error); + + var doc = JsonNode.Parse(await File.ReadAllTextAsync(graph))!.AsObject(); + var md = doc["metadata"]!.AsObject(); + Assert.Equal("opendeepwiki", md["tool"]!.GetValue()); + Assert.Equal("abc123", md["commit"]!.GetValue()); + Assert.NotNull(md["tool_version"]); + Assert.EndsWith("Z", md["generated_at"]!.GetValue()); + } + finally + { + SafeDelete(tmp); + } + } + + [Fact] + public async Task PublishAsync_EnabledWithToken_DispatchesAndReportsSuccess() + { + var tmp = MakeTempDir(); + try + { + var graph = Path.Combine(tmp, "graph.json"); + await File.WriteAllTextAsync(graph, "{\"nodes\":[],\"links\":[]}"); + + HttpRequestMessage? captured = null; + var handler = new Mock(); + handler.Protected() + .Setup>("SendAsync", + ItExpr.IsAny(), ItExpr.IsAny()) + .Callback((req, _) => captured = req) + .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.NoContent)); + + var options = Options.Create(new UnderstandQuicklyOptions + { + Enabled = true, + Token = "ghp_fake", + }); + var publisher = new UnderstandQuicklyPublisher( + options, NullLogger.Instance, + new HttpClient(handler.Object)); + + var result = await publisher.PublishAsync(MakeResult(graph), "looptech-ai/demo"); + + Assert.True(result.MetadataStamped); + Assert.True(result.Dispatched); + Assert.NotNull(captured); + Assert.Equal("https://api.github.com/repos/looptech-ai/understand-quickly/dispatches", + captured!.RequestUri!.ToString()); + Assert.Equal("Bearer", captured.Headers.Authorization!.Scheme); + var body = await captured.Content!.ReadAsStringAsync(); + var payload = JsonDocument.Parse(body).RootElement; + Assert.Equal("uq-publish", payload.GetProperty("event_type").GetString()); + var clientPayload = payload.GetProperty("client_payload"); + Assert.Equal("looptech-ai/demo", clientPayload.GetProperty("repo").GetString()); + // graph_path must be repo-relative — never an absolute server path. + var graphPath = clientPayload.GetProperty("graph_path").GetString()!; + Assert.False(Path.IsPathRooted(graphPath), + $"graph_path '{graphPath}' must be repo-relative"); + Assert.Equal("graph.json", graphPath); + } + finally + { + SafeDelete(tmp); + } + } + + [Fact] + public void ToRepoRelativePath_StripsOutputRoot() + { + // Use platform-real paths so this test passes on Windows and *nix. + var root = Path.Combine(Path.GetTempPath(), "uq-rel-test"); + var nested = Path.Combine(root, "graphify-out", "graph.json"); + var direct = Path.Combine(root, "graph.json"); + + Assert.Equal("graphify-out/graph.json", + UnderstandQuicklyPublisher.ToRepoRelativePath(root, nested)); + Assert.Equal("graph.json", + UnderstandQuicklyPublisher.ToRepoRelativePath(root, direct)); + // Null/empty OutputRoot -> file name fallback (no leaked server path). + Assert.Equal("graph.json", + UnderstandQuicklyPublisher.ToRepoRelativePath(null, nested)); + Assert.Equal("graph.json", + UnderstandQuicklyPublisher.ToRepoRelativePath(string.Empty, nested)); + } +}