Skip to content

Budget the base prompt by section (and fix the custom-rules framing) #498

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
FuJacob merged 1 commit into from
Jun 1, 2026
Merged

Budget the base prompt by section (and fix the custom-rules framing) #498

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Cotabby.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
3B3E08D1204E85F3776D8853 /* Sparkle in Frameworks */ = {isa = PBXBuildFile; productRef = BAADA69C6172DD7F4A642E93 /* Sparkle */; };
3B5F96F9CC6D4D81B470DB2C /* EmojiSynonymCatalogTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF474064973F4752F79BB041 /* EmojiSynonymCatalogTests.swift */; };
3C23336EE6F6559857DE92EE /* SuggestionDebugLogger.swift in Sources */ = {isa = PBXBuildFile; fileRef = 003594B09C83EF2DF35577D5 /* SuggestionDebugLogger.swift */; };
3C561CD717064F9250200667 /* PromptSectionBudget.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFCFCCCB69C29A86E726B10A /* PromptSectionBudget.swift */; };
3CBBC3BFAC0DC8952EE24EF7 /* BundledRuntimeLocator.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA33F5FFAC5B99384E15CE3E /* BundledRuntimeLocator.swift */; };
3CF1A4E39F24917DF0470A7D /* PromptPolicyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4696A84D17890B154533A08F /* PromptPolicyTests.swift */; };
4134ADBE464D00BB748BD9AE /* GeneralPaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07480CE96ED0EBD94817C6B1 /* GeneralPaneView.swift */; };
Expand Down Expand Up @@ -136,6 +137,7 @@
7D6BB9AF72F7076A4E5EE96F /* DownloadableModelCatalogView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BB5C2AE9A7E55495D26AD074 /* DownloadableModelCatalogView.swift */; };
7E9413CE7C999C4612348248 /* SuggestionSessionReconcilerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9C8F07AC52C7A482F5FE34C5 /* SuggestionSessionReconcilerTests.swift */; };
7E99F5676A1D1DF7EA7D7702 /* SuggestionCoordinator+Prediction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1ED1EA9282E0AC7592E60889 /* SuggestionCoordinator+Prediction.swift */; };
7EB20783E0D36715D1230A5C /* PromptSectionBudgetTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E260C4D08C786CDBD527B329 /* PromptSectionBudgetTests.swift */; };
7FC103944F4EF39DB965F469 /* InMemoryLogging in Frameworks */ = {isa = PBXBuildFile; productRef = 88921938DC814625ED57D605 /* InMemoryLogging */; };
814E348C663B697537594F0C /* EmojiRecentsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 671689F289D45A124639C9C6 /* EmojiRecentsTests.swift */; };
82D4ADEAF05337ABDE4C586C /* RuntimeBootstrapModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 60629DFE309C1A4BD8A7FB3B /* RuntimeBootstrapModel.swift */; };
Expand Down Expand Up @@ -418,6 +420,7 @@
AD9573F3504CAE6891DF9B7D /* AppUpdateManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppUpdateManager.swift; sourceTree = "<group>"; };
ADBE3E6CC585C1683787C877 /* SuggestionEngineModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionEngineModels.swift; sourceTree = "<group>"; };
AF1E065C7FFB697FCEB2FA5C /* CotabbyTestFixtures.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CotabbyTestFixtures.swift; sourceTree = "<group>"; };
AFCFCCCB69C29A86E726B10A /* PromptSectionBudget.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PromptSectionBudget.swift; sourceTree = "<group>"; };
B2BFD19A159680A495EE02FD /* ScreenshotContextGeneratorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScreenshotContextGeneratorTests.swift; sourceTree = "<group>"; };
B2F95847D76893C8A5B504B4 /* SuggestionOverlayStabilityGate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionOverlayStabilityGate.swift; sourceTree = "<group>"; };
B424E2AC97C99D335B0D5751 /* SuggestionTextNormalizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestionTextNormalizer.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -472,6 +475,7 @@
E19A5B462891263BDFB56607 /* TrailingDuplicationFilterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TrailingDuplicationFilterTests.swift; sourceTree = "<group>"; };
E1D2782B6C7BE3F56BCB22DE /* LlamaVisualContextSummarizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LlamaVisualContextSummarizer.swift; sourceTree = "<group>"; };
E217A66717D78E1E49350EC8 /* DownloadOutcomeClassifierTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DownloadOutcomeClassifierTests.swift; sourceTree = "<group>"; };
E260C4D08C786CDBD527B329 /* PromptSectionBudgetTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PromptSectionBudgetTests.swift; sourceTree = "<group>"; };
E27B962C66727776D00069DE /* EmojiPopularity.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmojiPopularity.swift; sourceTree = "<group>"; };
E3C84377F352140759B448C9 /* CaretGeometrySelector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CaretGeometrySelector.swift; sourceTree = "<group>"; };
E43E587E421AF544A8300CE4 /* CustomRulesCatalog.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CustomRulesCatalog.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -763,6 +767,7 @@
12DD19BCE610808F1E38702D /* PermissionOverlayTrackerTests.swift */,
0D80CC2CCAAFE3F23FB8C37A /* PromptContextSanitizerTests.swift */,
4696A84D17890B154533A08F /* PromptPolicyTests.swift */,
E260C4D08C786CDBD527B329 /* PromptSectionBudgetTests.swift */,
B2BFD19A159680A495EE02FD /* ScreenshotContextGeneratorTests.swift */,
2D7360A6D4261989A66658ED /* SentenceBoundaryClassifierTests.swift */,
2BC293F6125E2B14DCF05AD9 /* SettingsAttentionEvaluatorTests.swift */,
Expand Down Expand Up @@ -913,6 +918,7 @@
FA878B447441BB4F3E327CC8 /* OnboardingTemplateRecommender.swift */,
E6423D6CC8CC371D2DA899DE /* PermissionOverlayTracker.swift */,
FA4B45B91D4DEAC979C3113E /* PromptContextSanitizer.swift */,
AFCFCCCB69C29A86E726B10A /* PromptSectionBudget.swift */,
6DC693E00430F46E41CB56E6 /* RequestID.swift */,
D4B56C250DDEF3E81F9DCBD7 /* SentenceBoundaryClassifier.swift */,
2A02336442BB735EE2E8D064 /* SettingsAttentionEvaluator.swift */,
Expand Down Expand Up @@ -1163,6 +1169,7 @@
90DC9508F27F712EB61EEB06 /* PermissionReminderView.swift in Sources */,
39571AB31481959CD5C223AE /* PermissionsPaneView.swift in Sources */,
98E2E14A069384C1088CDB44 /* PromptContextSanitizer.swift in Sources */,
3C561CD717064F9250200667 /* PromptSectionBudget.swift in Sources */,
A5A6CE0EF01CA6A9AFA7A400 /* RequestID.swift in Sources */,
82D4ADEAF05337ABDE4C586C /* RuntimeBootstrapModel.swift in Sources */,
2C6159231472A849F15BD0AE /* ScreenFrameReader.swift in Sources */,
Expand Down Expand Up @@ -1271,6 +1278,7 @@
4F38CE1C2602CF4F41323032 /* PermissionOverlayTrackerTests.swift in Sources */,
934885ACC2DEA20B27F10948 /* PromptContextSanitizerTests.swift in Sources */,
3CF1A4E39F24917DF0470A7D /* PromptPolicyTests.swift in Sources */,
7EB20783E0D36715D1230A5C /* PromptSectionBudgetTests.swift in Sources */,
1B3FFCB9A979F49BF86EAAD4 /* ScreenshotContextGeneratorTests.swift in Sources */,
1D1C6FF0B8F50AC14A1000F4 /* SentenceBoundaryClassifierTests.swift in Sources */,
C618C5595DA9C57C806A3E03 /* SettingsAttentionEvaluatorTests.swift in Sources */,
Expand Down
135 changes: 73 additions & 62 deletions Cotabby/Support/BaseCompletionPromptRenderer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,19 @@ import Foundation
///
/// Design: a *base* model has no instruction-following channel and will happily continue a bare
/// "Task:" line as if it were the document, so an instruction-blob prompt would leak scaffolding into
/// the ghost text. This renderer instead treats the model as a pure text continuer:
/// the ghost text. This renderer treats the model as a pure text continuer: persona, style, language,
/// and supporting context are folded into a short conditioning preface (a base model conditions on
/// description, it does not obey commands), and the caret prefix is the LAST thing in the prompt with
/// trailing whitespace trimmed so generation begins at a clean word boundary.
///
/// - No task preamble and no standalone `Label:` lines.
/// - Custom instructions work by *conditioning*, not obedience: persona, voice, and language are
/// folded into a short framing sentence that makes the desired continuation the most likely one.
/// - Supporting context (notes/screen/clipboard) is included as compact prose ahead of the prefix.
/// - The single invariant that locates the caret is that `prefixText` is the LAST thing in the
/// prompt, with trailing whitespace trimmed so generation begins at a clean word boundary.
/// Sections are character-budgeted via `PromptSectionBudget` so a large glossary, clipboard, or
/// screen capture can never crowd out the caret text: the prefix gets top priority and a guaranteed
/// minimum, and context fills the remaining budget by priority.
enum BaseCompletionPromptRenderer {
/// Total character budget for the preface plus caret prefix. The prefix arrives already windowed
/// by `SuggestionRequestFactory`, so this mainly caps how much optional context rides along.
static let defaultContextBudget = 2400

static func prompt(
prefixText: String,
applicationName: String,
Expand All @@ -22,80 +26,87 @@ enum BaseCompletionPromptRenderer {
extendedContext: String? = nil,
languageInstruction: String? = nil,
clipboardContext: String? = nil,
visualContextSummary: String? = nil
visualContextSummary: String? = nil,
contextBudget: Int = defaultContextBudget
) -> String {
var preface: [String] = []
let trimmedPrefix = Self.trimmingTrailingWhitespace(prefixText)

// Persona/voice/language framing, phrased as a description of the document rather than a
// command, because a base model conditions on description but ignores instructions. Emitted
// only when the user supplied something, so a bare field stays pure continuation (the
// strongest base-model setup). `applicationName` is intentionally not stated as a label here;
// app/window metadata biases a base model toward code/numbers over prose.
if let framing = authorFraming(
userName: userName,
customRules: customRules,
languageInstruction: languageInstruction
) {
preface.append(framing)
var sections: [PromptSection] = []
if let persona = Self.personaLine(userName) {
sections.append(Self.contextSection("persona", persona, priority: 60, maxChars: 200))
}

// Free-form reference notes (glossary/terminology) ahead of the prefix so the user's terms
// become likelier continuations through in-context conditioning.
if let extendedContext, !extendedContext.isEmpty {
preface.append("Notes the writer keeps in mind: \(extendedContext)")
if let style = Self.styleLine(customRules) {
sections.append(Self.contextSection("style", style, priority: 55, maxChars: 300))
}
if let language = Self.nonEmpty(languageInstruction) {
sections.append(Self.contextSection("language", language, priority: 50, maxChars: 300))
}
if let notes = Self.nonEmpty(extendedContext) {
sections.append(Self.contextSection("notes", "Notes the writer keeps in mind: \(notes)", priority: 40, maxChars: 600))
}
if let visualContextSummary, !visualContextSummary.isEmpty {
preface.append("Nearby on screen: \(visualContextSummary)")
if let clip = Self.nonEmpty(clipboardContext) {
sections.append(Self.contextSection("clipboard", "On the clipboard: \(clip)", priority: 35, maxChars: 400))
}
if let clipboardContext, !clipboardContext.isEmpty {
preface.append("On the clipboard: \(clipboardContext)")
if let screen = Self.nonEmpty(visualContextSummary) {
sections.append(Self.contextSection("screen", "Nearby on screen: \(screen)", priority: 30, maxChars: 500))
}
// The caret prefix: top priority so it is never starved, kept by its END (the text nearest
// the caret), and rendered last with no label so the model continues from where the user
// stopped. `applicationName` is intentionally not stated; app/window metadata biases a base
// model toward code/numbers over prose.
sections.append(
PromptSection(
name: "prefix",
content: trimmedPrefix,
priority: 100,
minChars: 1,
maxChars: max(1, trimmedPrefix.count),
truncation: .preserveEnd
)
)

// Trailing whitespace is trimmed so the model continues from a clean word boundary instead of
// being asked to emit a leading-space token (which base models do poorly). A prefix ending
// mid-word keeps its final partial word, so mid-word continuation still works. Output spacing
// is reconciled downstream by `SuggestionTextNormalizer`.
let trimmedPrefix = Self.trimmingTrailingWhitespace(prefixText)
let kept = PromptSectionBudget.allocate(sections, totalChars: contextBudget)
let prefix = kept.first { $0.name == "prefix" }?.content ?? trimmedPrefix
Comment on lines +57 to +69
Copy link
Copy Markdown
Contributor

@greptile-apps greptile-apps Bot Jun 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Prefix trimming may strip a leading space after truncation

When the total budget is smaller than trimmedPrefix.count, the allocator calls truncate(trimmedPrefix, toChars: cap, mode: .preserveEnd) and then trimmingCharacters(in: .whitespacesAndNewlines) on the result. A suffix of the prefix can start with a space or newline (e.g. "hello world".suffix(7) = " world" → trimmed to "world"), so the model receives a prompt that ends mid-word-boundary, defeating the invariant described in the doc comment. The original code always passed the full trimmedPrefix. This only fires when contextBudget is smaller than the prefix length (unlikely with the 2400-char default and pre-windowed input), but as a code path it silently breaks the caret-continuation contract.

Fix in Codex Fix in Claude Code

let preface = kept.filter { $0.name != "prefix" }.map(\.content)

guard !preface.isEmpty else {
// No context to condition on: hand the model the bare text and let it continue.
return trimmedPrefix
return prefix
}

// A blank line separates the conditioning preface from the live text without a label the
// model could copy. The prefix remains the final bytes of the prompt.
return preface.joined(separator: "\n") + "\n\n" + trimmedPrefix
return preface.joined(separator: "\n") + "\n\n" + prefix
}

/// Builds the conditioning sentence from persona/style/language, or nil when none were supplied.
private static func authorFraming(
userName: String?,
customRules: [String],
languageInstruction: String?
) -> String? {
let name = (userName ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
private static func contextSection(
_ name: String,
_ content: String,
priority: Int,
maxChars: Int
) -> PromptSection {
PromptSection(name: name, content: content, priority: priority, minChars: 0, maxChars: maxChars, truncation: .preserveStart)
}

/// "Written by <name>." or nil. Conditions the voice via authorship framing.
private static func personaLine(_ userName: String?) -> String? {
guard let name = Self.nonEmpty(userName) else { return nil }
return "Written by \(name)."
}

/// "Writing style: <rules>." or nil. Rendered as its own line rather than jammed into an
/// "in a <rules> style" clause, so multi-word and sentence-shaped rules read correctly and
/// condition cleanly (the old clause produced broken prose like "in a Use British spelling style").
private static func styleLine(_ customRules: [String]) -> String? {
let rules = customRules
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
.filter { !$0.isEmpty }
let language = (languageInstruction ?? "").trimmingCharacters(in: .whitespacesAndNewlines)

guard !name.isEmpty || !rules.isEmpty || !language.isEmpty else {
return nil
}
guard !rules.isEmpty else { return nil }
return "Writing style: \(rules.joined(separator: ", "))."
}

var sentence = "The following is text"
if !name.isEmpty {
sentence += " written by \(name)"
}
if !rules.isEmpty {
sentence += " in a \(rules.joined(separator: ", ")) style"
}
sentence += "."
if !language.isEmpty {
// `languageInstruction` is already a soft directive sentence; append it verbatim.
sentence += " \(language)"
}
return sentence
private static func nonEmpty(_ text: String?) -> String? {
let trimmed = (text ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? nil : trimmed
}

/// Drops trailing spaces, tabs, and newlines so the base-model prompt ends at a word boundary.
Expand Down
Loading