From 91255d0f19722a5a2d4e76739ffe6e3fb8dc06af Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:26:32 +0000 Subject: [PATCH 01/30] Run broadcast agent catch callbacks when the error broadcast throws A failed BroadcastAgent job broadcasts "stream_failed" and then runs the response's catch callbacks. If that broadcast threw, the callbacks never ran. They now run in a finally block, so they run either way. Ported from laravel/ai#1223 at a2cb93a2. The trait alias takes upstream's invokeCatchCallbacks name. Hypervel's existing callback test is folded into upstream's testFailedRunsTheCatchCallbacksWithTheException, keeping the manual-failure case and the broadcast-first check, and the throwing-broadcast test is ported. No docs change upstream. Validation: the changed test file passes; formatting and full static analysis pass. --- src/ai/src/Jobs/BroadcastAgent.php | 24 ++++--- tests/Ai/Feature/BroadcastAgentTest.php | 93 ++++++++++++++++--------- 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/src/ai/src/Jobs/BroadcastAgent.php b/src/ai/src/Jobs/BroadcastAgent.php index 18aaaa933b..1e8e5cf8a5 100644 --- a/src/ai/src/Jobs/BroadcastAgent.php +++ b/src/ai/src/Jobs/BroadcastAgent.php @@ -23,7 +23,7 @@ class BroadcastAgent implements ShouldQueue { use InvokesQueuedResponseCallbacks { - failed as protected invokeFailureCallbacks; + failed as protected invokeCatchCallbacks; } use Queueable; @@ -69,16 +69,18 @@ public function handle(): void */ public function failed(?Throwable $exception): void { - (new Error( - id: ulid(), - type: 'stream_failed', - message: 'The stream failed.', - recoverable: false, - timestamp: time(), - ))->withInvocationId($this->invocationId) - ->broadcastNow($this->channels); - - $this->invokeFailureCallbacks($exception); + try { + (new Error( + id: ulid(), + type: 'stream_failed', + message: 'The stream failed.', + recoverable: false, + timestamp: time(), + ))->withInvocationId($this->invocationId) + ->broadcastNow($this->channels); + } finally { + $this->invokeCatchCallbacks($exception); + } } /** diff --git a/tests/Ai/Feature/BroadcastAgentTest.php b/tests/Ai/Feature/BroadcastAgentTest.php index efb3c5c52e..15792f7e7b 100644 --- a/tests/Ai/Feature/BroadcastAgentTest.php +++ b/tests/Ai/Feature/BroadcastAgentTest.php @@ -18,6 +18,7 @@ use Hypervel\Tests\Ai\Fixtures\Agents\AssistantAgent; use Hypervel\Tests\Ai\Fixtures\Agents\ConversationalAgent; use Hypervel\Tests\Ai\TestCase; +use InvalidArgumentException; use JMac\Testing\Double; use JMac\Testing\Integrations\PHPUnit\VerifiesDoubles; use PHPUnit\Framework\Attributes\DataProvider; @@ -144,6 +145,67 @@ public function testFailedBroadcastsOnEveryChannelWhenGivenAnArray(): void && $event->broadcastOn() === $channels); } + #[DataProvider('failureExceptions')] + public function testFailedRunsTheCatchCallbacksWithTheException(?Throwable $exception): void + { + Event::fake(); + + $job = new BroadcastAgent(new AssistantAgent, 'Say hello', new Channel('test-channel')); + $invocationId = $job->invocationId; + $failures = []; + $job->catch(function (?Throwable $exception) use ($invocationId, &$failures): void { + Event::assertDispatched(AnonymousEvent::class, fn (AnonymousEvent $event): bool => $event->broadcastAs() === 'stream_failed' + && $event->broadcastWith()['invocation_id'] === $invocationId); + + $failures[] = $exception; + }); + + $job->failed($exception); + + $this->assertSame([$exception], $failures); + } + + /** + * Provide exception and manual failures. + */ + public static function failureExceptions(): array + { + return [ + 'exception' => [new RuntimeException('Something went wrong')], + 'manual failure' => [null], + ]; + } + + public function testFailedRunsTheCatchCallbacksWhenTheErrorBroadcastThrows(): void + { + $broadcastException = new InvalidArgumentException('Broadcast connection [x] is not defined.'); + Broadcast::shouldReceive('on')->andThrow($broadcastException); + + $received = null; + + $job = new BroadcastAgent( + agent: new AssistantAgent, + prompt: 'Say hello', + channels: new Channel('test-channel'), + ); + + $job->catch(function (Throwable $e) use (&$received): void { + $received = $e; + }); + + $exception = new RuntimeException('boom'); + + try { + $job->failed($exception); + + $this->fail('The broadcast exception was not thrown.'); + } catch (InvalidArgumentException $caught) { + $this->assertSame($broadcastException, $caught); + } + + $this->assertSame($exception, $received); + } + #[TestWith(['overloaded_error'])] #[TestWith(['not_found'])] #[TestWith(['unknown_error'])] @@ -235,35 +297,4 @@ public function testStreamedResponsePassedToThenIsFullyResolved(): void $this->assertNotEmpty($received->events); $this->assertSame('Hello world', $received->text); } - - #[DataProvider('failureExceptions')] - public function testFailedInvokesCatchCallbacksAfterBroadcasting(?Throwable $exception): void - { - Event::fake(); - - $job = new BroadcastAgent(new AssistantAgent, 'Say hello', new Channel('test-channel')); - $invocationId = $job->invocationId; - $failures = []; - $job->catch(function (?Throwable $exception) use ($invocationId, &$failures): void { - Event::assertDispatched(AnonymousEvent::class, fn (AnonymousEvent $event): bool => $event->broadcastAs() === 'stream_failed' - && $event->broadcastWith()['invocation_id'] === $invocationId); - - $failures[] = $exception; - }); - - $job->failed($exception); - - $this->assertSame([$exception], $failures); - } - - /** - * Provide exception and manual failures. - */ - public static function failureExceptions(): array - { - return [ - 'exception' => [new RuntimeException('Something went wrong')], - 'manual failure' => [null], - ]; - } } From a3a794185705cf552617dcd961abf169f73147e6 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:26:40 +0000 Subject: [PATCH 02/30] Let provider options override OpenAI image moderation OpenAI gpt-image generations and edits, and Azure OpenAI generations, always sent moderation "low". They now send the moderation provider option when given, defaulting to "low". Ported from laravel/ai#1226 at a2cb93a2. Azure image edits, a Hypervel addition, send no moderation, so provider options already apply there unchanged. Ported the OpenAI generation override test and the edit default and override tests; the Azure default test is renamed and an override test added. No docs change upstream. Validation: the changed test files pass; formatting and full static analysis pass. --- .../AzureOpenAi/AzureOpenAiGateway.php | 2 +- src/ai/src/Gateway/OpenAi/OpenAiGateway.php | 4 ++-- .../AzureOpenAi/ImageGenerationTest.php | 19 ++++++++++++++- .../Providers/OpenAi/ImageEditTest.php | 23 +++++++++++++++++++ .../Providers/OpenAi/ImageGenerationTest.php | 18 +++++++++++++++ 5 files changed, 62 insertions(+), 4 deletions(-) diff --git a/src/ai/src/Gateway/AzureOpenAi/AzureOpenAiGateway.php b/src/ai/src/Gateway/AzureOpenAi/AzureOpenAiGateway.php index 129fdb858b..923b6f37b2 100644 --- a/src/ai/src/Gateway/AzureOpenAi/AzureOpenAiGateway.php +++ b/src/ai/src/Gateway/AzureOpenAi/AzureOpenAiGateway.php @@ -115,7 +115,7 @@ public function generateImage( ...$providerOptions, 'model' => $model, 'prompt' => $prompt, - 'moderation' => 'low', + 'moderation' => $providerOptions['moderation'] ?? 'low', ...$provider->defaultImageOptions($size, $quality), ]), ); diff --git a/src/ai/src/Gateway/OpenAi/OpenAiGateway.php b/src/ai/src/Gateway/OpenAi/OpenAiGateway.php index f90ad9dd77..0ff075c68b 100644 --- a/src/ai/src/Gateway/OpenAi/OpenAiGateway.php +++ b/src/ai/src/Gateway/OpenAi/OpenAiGateway.php @@ -124,7 +124,7 @@ protected function sendImageGenerationRequest( 'prompt' => $prompt, ...$provider->defaultImageOptions($size, $quality), ...(str_starts_with($model, 'gpt-image') - ? ['moderation' => 'low'] + ? ['moderation' => $providerOptions['moderation'] ?? 'low'] : ['response_format' => 'b64_json']), ]); } @@ -165,7 +165,7 @@ protected function sendImageEditRequest( 'prompt' => $prompt, ...$provider->defaultImageOptions($size, $quality), ...($isGptImage - ? ['moderation' => 'low'] + ? ['moderation' => $providerOptions['moderation'] ?? 'low'] : ['response_format' => 'b64_json']), ])); } diff --git a/tests/Ai/Feature/Providers/AzureOpenAi/ImageGenerationTest.php b/tests/Ai/Feature/Providers/AzureOpenAi/ImageGenerationTest.php index b2c8a6c5c7..25862f1aa6 100644 --- a/tests/Ai/Feature/Providers/AzureOpenAi/ImageGenerationTest.php +++ b/tests/Ai/Feature/Providers/AzureOpenAi/ImageGenerationTest.php @@ -123,7 +123,7 @@ public function testImageRequestDoesNotIncludeSizeWhenNotSpecified(): void }); } - public function testImageRequestAlwaysIncludesModerationLow(): void + public function testImageRequestIncludesModerationLowByDefault(): void { Http::fake([ '*' => $this->fakeAzureImageResponse(), @@ -138,6 +138,23 @@ public function testImageRequestAlwaysIncludesModerationLow(): void }); } + public function testImageRequestModerationCanBeSetThroughProviderOptions(): void + { + Http::fake([ + '*' => $this->fakeAzureImageResponse(), + ]); + + Image::of('A red apple') + ->withProviderOptions(['moderation' => 'auto']) + ->generate(provider: 'azure', model: 'gpt-image-1'); + + Http::assertSent(function (Request $request): bool { + $body = json_decode($request->body(), true); + + return ($body['moderation'] ?? null) === 'auto'; + }); + } + #[DataProvider('editAttachments')] public function testImageGenerationSendsAttachmentsToTheEditEndpoint(Closure $attachment): void { diff --git a/tests/Ai/Feature/Providers/OpenAi/ImageEditTest.php b/tests/Ai/Feature/Providers/OpenAi/ImageEditTest.php index b3d2e9c1d0..3269ef0d55 100644 --- a/tests/Ai/Feature/Providers/OpenAi/ImageEditTest.php +++ b/tests/Ai/Feature/Providers/OpenAi/ImageEditTest.php @@ -151,4 +151,27 @@ public function testNonGptImageModelsSendASingleImageField(): void Http::assertSent(fn (Request $request): bool => $request->hasFile('image', 'source-bytes', 'image.png')); } + + public function testGptImageEditsSendModerationLowByDefault(): void + { + Http::fake(['*' => $this->fakeOpenAiImageEditResponse()]); + + Image::of('Make it brighter') + ->attachments([new Base64Image(base64_encode('source-bytes'), 'image/png')]) + ->generate(provider: 'openai', model: 'gpt-image-1'); + + Http::assertSent(fn (Request $request): bool => collect($request->data())->pluck('contents', 'name')->get('moderation') === 'low'); + } + + public function testGptImageEditsModerationCanBeSetThroughProviderOptions(): void + { + Http::fake(['*' => $this->fakeOpenAiImageEditResponse()]); + + Image::of('Make it brighter') + ->attachments([new Base64Image(base64_encode('source-bytes'), 'image/png')]) + ->withProviderOptions(['moderation' => 'auto']) + ->generate(provider: 'openai', model: 'gpt-image-1'); + + Http::assertSent(fn (Request $request): bool => collect($request->data())->pluck('contents', 'name')->get('moderation') === 'auto'); + } } diff --git a/tests/Ai/Feature/Providers/OpenAi/ImageGenerationTest.php b/tests/Ai/Feature/Providers/OpenAi/ImageGenerationTest.php index 901162533e..ec280db920 100644 --- a/tests/Ai/Feature/Providers/OpenAi/ImageGenerationTest.php +++ b/tests/Ai/Feature/Providers/OpenAi/ImageGenerationTest.php @@ -83,6 +83,24 @@ public function testImageRequestIncludesModerationLowForGptImageModels(): void }); } + public function testImageRequestModerationCanBeSetThroughProviderOptionsForGptImageModels(): void + { + Http::fake([ + '*' => fakeOpenAiImageResponse(), + ]); + + Image::of('A red apple') + ->withProviderOptions(['moderation' => 'auto']) + ->generate(provider: 'openai', model: 'gpt-image-1'); + + Http::assertSent(function (Request $request): bool { + $body = json_decode($request->body(), true); + + return $body['model'] === 'gpt-image-1' + && $body['moderation'] === 'auto'; + }); + } + public function testImageRequestIncludesQualityWhenExplicitlySpecified(): void { Http::fake([ From 45ad4fb6c639a1dd91d79b4f09bec49bf0f350c2 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 03:27:01 +0000 Subject: [PATCH 03/30] Port sub-agent failures, stored Gemini continuations, title agent and Anthropic containers These upstream changes share the tool loop, the AI documentation page and the agent events tests, so they land together. Ported from laravel/ai at a2cb93a2. laravel/ai#1228: a sub-agent that throws is handled like any other tool failure. AgentTool no longer catches; the loop dispatches ToolFailed, reports the exception and gives the parent a failed tool result, hiding the message outside debug mode. Cancellation is never wrapped as a tool failure. Hypervel's concurrent streamed tool path recovers the same way through a shared helper. Ported the AgentToolTest and AgentEventsTest cases, adapting the after-output case to a streamed sub-agent because Hypervel only snapshots text output; reporting, debug-mode and concurrent coverage moved to the tool failure and concurrency tests. The sub-agent docs now point to ordinary tool failures. laravel/ai#1189: with the store provider option, Gemini returns the interaction id as the continuation token and the next step sends previous_interaction_id with only the new function results. Upstream defect fixed in the shared loop, which also affects OpenAI and xAI: a response to messages edited by step middleware is no longer continued, since the provider stored the edited history that later steps must not inherit. Ported the Gemini tool loop and streaming tests, with first-step edit tests for the generate and stream paths. laravel/ai#1229: conversation titles come from ConversationTitleAgent, resolved from the container so apps can bind their own, and faked when the conversation's agent is faked so the title doesn't use the agent's fake responses. Agent fakes therefore no longer force serial titles. Hypervel passes a concrete provider instance instead of its name, so an on-demand provider keeps its credentials; providers that only implement the contract are registered by name and still go by name. Upstream defect fixed: resolving the title agent sits inside the title fallback, so a failing binding no longer loses the completed turn. Ported the ConversationTitleTest cases and SupportTitleAgent fixture, with tests for on-demand and contract-only providers and the failing binding. Upstream has no docs; the conversation titles section now covers the agent, titling app-created conversations and binding a replacement, and the testing docs cover title fakes. laravel/ai#1213: Anthropic returns its code execution container id as the continuation token and the next step reuses the container unless provider options set one. Upstream defect fixed: the loop dropped the token when a step changed its history, model or instructions, so a second-step withInstructions() opened a new container and lost its files. Gateways that send the full history every step now implement ReplaysFullHistory, and the loop keeps their token across step changes. The loop's protected method signatures and the token fields are unchanged. Ported the Anthropic tool loop and streaming tests, which also run with changed instructions and edited messages. Validation: the changed test files and the AI suite pass; formatting and full static analysis pass. --- src/ai/src/Agents/ConversationTitleAgent.php | 26 +++ .../Contracts/Gateway/ReplaysFullHistory.php | 12 ++ .../Gateway/Anthropic/AnthropicGateway.php | 7 +- .../Anthropic/Concerns/BuildsTextRequests.php | 19 ++ .../Concerns/HandlesTextStreaming.php | 3 + .../Concerns/ParsesTextResponses.php | 3 + .../Gemini/Concerns/BuildsTextRequests.php | 30 ++- .../Gemini/Concerns/HandlesTextStreaming.php | 2 + .../Gemini/Concerns/ParsesTextResponses.php | 10 + src/ai/src/Gateway/Gemini/GeminiGateway.php | 4 +- src/ai/src/Gateway/ParentInvocation.php | 2 +- src/ai/src/Gateway/StepContext.php | 2 +- src/ai/src/Gateway/TextGenerationLoop.php | 59 ++++-- .../src/Middleware/RememberConversation.php | 33 ++-- src/ai/src/Tools/AgentTool.php | 41 +--- src/docs/ai-sdk.md | 42 +++- tests/Ai/ConcurrentToolsTest.php | 35 ++++ tests/Ai/Feature/AgentEventsTest.php | 113 ++++++++++- tests/Ai/Feature/AgentMiddlewareTest.php | 1 + tests/Ai/Feature/ConversationTitleTest.php | 185 ++++++++++++++++++ .../Providers/Anthropic/StreamingTest.php | 51 +++++ .../Providers/Anthropic/ToolCallLoopTest.php | 163 +++++++++++++++ .../Providers/Gemini/StreamingTest.php | 78 ++++++++ .../Providers/Gemini/ToolCallLoopTest.php | 127 ++++++++++++ tests/Ai/Feature/RememberConversationTest.php | 52 +++-- tests/Ai/Feature/ToolFailureRecoveryTest.php | 34 ++++ tests/Ai/Fixtures/Agents/StoredToolAgent.php | 56 ++++++ .../Ai/Fixtures/Agents/SupportTitleAgent.php | 20 ++ tests/Ai/Unit/Tools/AgentToolTest.php | 95 +-------- 29 files changed, 1101 insertions(+), 204 deletions(-) create mode 100644 src/ai/src/Agents/ConversationTitleAgent.php create mode 100644 src/ai/src/Contracts/Gateway/ReplaysFullHistory.php create mode 100644 tests/Ai/Feature/ConversationTitleTest.php create mode 100644 tests/Ai/Fixtures/Agents/StoredToolAgent.php create mode 100644 tests/Ai/Fixtures/Agents/SupportTitleAgent.php diff --git a/src/ai/src/Agents/ConversationTitleAgent.php b/src/ai/src/Agents/ConversationTitleAgent.php new file mode 100644 index 0000000000..dcd1c21cfa --- /dev/null +++ b/src/ai/src/Agents/ConversationTitleAgent.php @@ -0,0 +1,26 @@ +withContainer($body, $stepContext); + $response = $this->withErrorHandling( $provider->name(), fn () => $this->client($provider, $timeout)->post('messages', $body), @@ -116,6 +119,8 @@ public function generateStreamStep( $options, ); + $body = $this->withContainer($body, $stepContext); + $body['stream'] = true; $response = $this->withErrorHandling( diff --git a/src/ai/src/Gateway/Anthropic/Concerns/BuildsTextRequests.php b/src/ai/src/Gateway/Anthropic/Concerns/BuildsTextRequests.php index f137ca85c0..c16146e72f 100644 --- a/src/ai/src/Gateway/Anthropic/Concerns/BuildsTextRequests.php +++ b/src/ai/src/Gateway/Anthropic/Concerns/BuildsTextRequests.php @@ -7,6 +7,7 @@ use Hypervel\Ai\Attributes\CacheInstructions; use Hypervel\Ai\Attributes\CacheToolDefinitions; use Hypervel\Ai\Gateway\Anthropic\AnthropicSchemaSanitizer; +use Hypervel\Ai\Gateway\StepContext; use Hypervel\Ai\Gateway\TextGenerationOptions; use Hypervel\Ai\ObjectSchema; use Hypervel\Ai\Providers\Provider; @@ -81,6 +82,24 @@ protected function buildTextRequestBody( return $this->applyPromptCacheBreakpoints(array_merge($body, $providerOptions), $options); } + /** + * Run the step in the code execution container the previous step used. + */ + protected function withContainer(array $body, StepContext $stepContext): array + { + if (blank($stepContext->continuationToken)) { + return $body; + } + + if (is_array($body['container'] ?? null)) { + $body['container']['id'] ??= $stepContext->continuationToken; + } else { + $body['container'] ??= $stepContext->continuationToken; + } + + return $body; + } + /** * Stamp the requested cache breakpoints onto the final request body. */ diff --git a/src/ai/src/Gateway/Anthropic/Concerns/HandlesTextStreaming.php b/src/ai/src/Gateway/Anthropic/Concerns/HandlesTextStreaming.php index 35c11045d1..d6222f2fa3 100644 --- a/src/ai/src/Gateway/Anthropic/Concerns/HandlesTextStreaming.php +++ b/src/ai/src/Gateway/Anthropic/Concerns/HandlesTextStreaming.php @@ -91,6 +91,7 @@ protected function processTextStream( $messageUsage = []; $usage = null; $stopReason = ''; + $container = null; $citedUrls = []; $emitTextStart = function () use (&$textStartEmitted, $messageId, $invocationId): ?StreamEvent { @@ -372,6 +373,7 @@ protected function processTextStream( if ($type === 'message_delta') { $stopReason = $data['delta']['stop_reason'] ?? ''; + $container = $data['delta']['container']['id'] ?? $container; // Usage on message_delta is cumulative for the whole message... $usage = $this->extractUsage(['usage' => array_merge($messageUsage, $data['usage'] ?? [])]); @@ -407,6 +409,7 @@ protected function processTextStream( usage: $usage ?? new TextUsage(0, 0), finishReason: $this->extractFinishReason(['stop_reason' => $stopReason]), structured: false, + container: $container, ); } diff --git a/src/ai/src/Gateway/Anthropic/Concerns/ParsesTextResponses.php b/src/ai/src/Gateway/Anthropic/Concerns/ParsesTextResponses.php index d28dce7d9f..d35ae80a3c 100644 --- a/src/ai/src/Gateway/Anthropic/Concerns/ParsesTextResponses.php +++ b/src/ai/src/Gateway/Anthropic/Concerns/ParsesTextResponses.php @@ -53,6 +53,7 @@ protected function parseTextResponse( $this->extractUsage($data), $this->extractFinishReason($data), $structured, + $data['container']['id'] ?? null, ); } @@ -66,6 +67,7 @@ protected function buildStepResponse( TextUsage $usage, FinishReason $finishReason, bool $structured, + ?string $container = null, ): StepResponse { $text = $this->extractText($content); $toolCalls = $this->extractToolCalls($content); @@ -101,6 +103,7 @@ protected function buildStepResponse( usage: $usage, meta: new Meta($provider->name(), $model, $citations), structured: $structuredData, + continuationToken: $container, // The synthetic output has no tool result, so replaying its call would leave an unmatched tool-use ID. replayBlocks: array_values(array_filter($content, fn (array $block): bool => ($block['type'] ?? '') !== 'tool_use' || ($block['name'] ?? '') !== 'output_structured_data')), reasoning: $this->extractReasoning($content), diff --git a/src/ai/src/Gateway/Gemini/Concerns/BuildsTextRequests.php b/src/ai/src/Gateway/Gemini/Concerns/BuildsTextRequests.php index e21832d171..1a37c17ef2 100644 --- a/src/ai/src/Gateway/Gemini/Concerns/BuildsTextRequests.php +++ b/src/ai/src/Gateway/Gemini/Concerns/BuildsTextRequests.php @@ -6,6 +6,7 @@ use Hypervel\Ai\Gateway\StepContext; use Hypervel\Ai\Gateway\TextGenerationOptions; +use Hypervel\Ai\Messages\ToolResultMessage; use Hypervel\Ai\ObjectSchema; use Hypervel\Ai\Providers\Provider; use Hypervel\Ai\Responses\Data\ToolResult; @@ -39,7 +40,34 @@ protected function buildStepBody( ?TextGenerationOptions $options, StepContext $stepContext, ): array { - return $this->buildTextRequestBody($provider, $model, $instructions, $messages, $tools, $schema, $options); + if (! $previousInteractionId = $stepContext->continuationToken) { + return $this->buildTextRequestBody($provider, $model, $instructions, $messages, $tools, $schema, $options); + } + + return [ + ...$this->assembleRequestBody($model, $this->extractToolResultsInput($messages), $instructions, $tools, $schema, $options, $provider), + 'previous_interaction_id' => $previousInteractionId, + ]; + } + + /** + * Extract function result steps from the trailing tool result message for a continuation request. + */ + protected function extractToolResultsInput(array $messages): array + { + $lastMessage = end($messages); + + return $lastMessage instanceof ToolResultMessage + ? $this->buildFunctionResultSteps($lastMessage->toolResults->all()) + : []; + } + + /** + * Determine if Gemini will retain the interaction described by the given request body. + */ + protected function storesInteraction(array $body): bool + { + return $body['store'] === true; } /** diff --git a/src/ai/src/Gateway/Gemini/Concerns/HandlesTextStreaming.php b/src/ai/src/Gateway/Gemini/Concerns/HandlesTextStreaming.php index 4387dbe81a..33882fa847 100644 --- a/src/ai/src/Gateway/Gemini/Concerns/HandlesTextStreaming.php +++ b/src/ai/src/Gateway/Gemini/Concerns/HandlesTextStreaming.php @@ -36,6 +36,7 @@ protected function processTextStream( Provider $provider, string $model, StreamInterface $streamBody, + bool $stored, ): Generator { $messageId = $this->generateEventId(); $reasoningId = ''; @@ -250,6 +251,7 @@ protected function processTextStream( finishReason: $this->extractFinishReason($final, $functionCallSteps), usage: $this->extractUsage($final), meta: new Meta($provider->name(), $model), + continuationToken: $this->continuationToken($final, $stored), replayBlocks: $this->replayableSteps($steps), reasoning: $this->extractReasoning($steps), providerToolCalls: $this->extractProviderToolCalls($steps), diff --git a/src/ai/src/Gateway/Gemini/Concerns/ParsesTextResponses.php b/src/ai/src/Gateway/Gemini/Concerns/ParsesTextResponses.php index 9e6d97a59e..175b507175 100644 --- a/src/ai/src/Gateway/Gemini/Concerns/ParsesTextResponses.php +++ b/src/ai/src/Gateway/Gemini/Concerns/ParsesTextResponses.php @@ -64,6 +64,7 @@ protected function parseTextResponse( Provider $provider, string $model, bool $structured, + bool $stored, ): StepResponse { $steps = $data['steps'] ?? []; @@ -77,12 +78,21 @@ protected function parseTextResponse( usage: $this->extractUsage($data), meta: new Meta($provider->name(), $model, $this->extractCitations($steps)), structured: $structured ? $this->decodeStructuredOutput($text) : null, + continuationToken: $this->continuationToken($data, $stored), replayBlocks: $this->replayableSteps($steps), reasoning: $this->extractReasoning($steps), providerToolCalls: $this->extractProviderToolCalls($steps), ); } + /** + * Get the interaction id the next step may continue from, which only a stored interaction offers. + */ + protected function continuationToken(array $data, bool $stored): ?string + { + return $stored ? ($data['id'] ?? null) : null; + } + /** * Get the steps that must be replayed verbatim on the next request. */ diff --git a/src/ai/src/Gateway/Gemini/GeminiGateway.php b/src/ai/src/Gateway/Gemini/GeminiGateway.php index 57bce9b17f..d97c0e80bf 100644 --- a/src/ai/src/Gateway/Gemini/GeminiGateway.php +++ b/src/ai/src/Gateway/Gemini/GeminiGateway.php @@ -81,7 +81,7 @@ public function generateTextStep( $this->validateTextResponse($data); - return $this->parseTextResponse($data, $provider, $model, filled($schema))->withRawResponse($response); + return $this->parseTextResponse($data, $provider, $model, filled($schema), $this->storesInteraction($body))->withRawResponse($response); } /** @@ -109,7 +109,7 @@ public function generateStreamStep( ->post('interactions?alt=sse', array_merge($body, ['stream' => true])), ); - return yield from $this->processTextStream($invocationId, $provider, $model, $response->getBody()); + return yield from $this->processTextStream($invocationId, $provider, $model, $response->getBody(), $this->storesInteraction($body)); } /** diff --git a/src/ai/src/Gateway/ParentInvocation.php b/src/ai/src/Gateway/ParentInvocation.php index e7ae975d39..459bd15cb5 100644 --- a/src/ai/src/Gateway/ParentInvocation.php +++ b/src/ai/src/Gateway/ParentInvocation.php @@ -24,7 +24,7 @@ public static function current(): array /** * Run the given callback with the given invocation and tool invocation as the delegating parent. */ - public static function within(?string $invocationId, string $toolInvocationId, Closure $callback): mixed + public static function within(?string $invocationId, ?string $toolInvocationId, Closure $callback): mixed { $previous = CoroutineContext::get(static::PARENT_INVOCATION_CONTEXT_KEY); diff --git a/src/ai/src/Gateway/StepContext.php b/src/ai/src/Gateway/StepContext.php index b7574b4a73..97c319f77a 100644 --- a/src/ai/src/Gateway/StepContext.php +++ b/src/ai/src/Gateway/StepContext.php @@ -9,7 +9,7 @@ class StepContext /** * Create the context for a generation step. * - * @param null|string $continuationToken provider handle for stateful continuation; null for stateless providers that replay full history + * @param null|string $continuationToken provider handle the previous step returned for the next one, such as a response or container id */ public function __construct( public readonly int $stepNumber = 0, diff --git a/src/ai/src/Gateway/TextGenerationLoop.php b/src/ai/src/Gateway/TextGenerationLoop.php index 96bb4a79eb..f63411ab74 100644 --- a/src/ai/src/Gateway/TextGenerationLoop.php +++ b/src/ai/src/Gateway/TextGenerationLoop.php @@ -15,6 +15,7 @@ use Hypervel\Ai\Attributes\RepairToolCalls; use Hypervel\Ai\Concerns\JoinsReasoning; use Hypervel\Ai\Contracts\Approvable; +use Hypervel\Ai\Contracts\Gateway\ReplaysFullHistory; use Hypervel\Ai\Contracts\Gateway\StepTextGateway; use Hypervel\Ai\Contracts\HasMiddleware; use Hypervel\Ai\Contracts\Providers\SupportsCodeExecution; @@ -241,7 +242,8 @@ public function generate( break; } - $continuationToken = $lastResult->continuationToken; + // Unlike upstream, a response to edited messages is not continued, since the provider stored the edited history the next step must not inherit. A gateway that replays the full history stores none. + $continuationToken = $this->gateway instanceof ReplaysFullHistory || $prepared->messages === $pending->messages ? $lastResult->continuationToken : null; $previous = $prepared; } @@ -468,7 +470,8 @@ public function stream( break; } - $continuationToken = $result->continuationToken; + // Unlike upstream, a response to edited messages is not continued, since the provider stored the edited history the next step must not inherit. A gateway that replays the full history stores none. + $continuationToken = $this->gateway instanceof ReplaysFullHistory || $prepared->messages === $pending->messages ? $result->continuationToken : null; $previous = $prepared; } @@ -588,12 +591,14 @@ protected function stepFailed(?RunContext $context, ?StepResult $attempt, Throwa */ protected function stepContextFor(PendingStep $step, ?PendingStep $previous, array $history, ?string $continuationToken): StepContext { - $unchanged = $step->messages === $history && $step->model === $previous?->model && $step->instructions === $previous->instructions; + // Unlike upstream, the token of a gateway that sends the full history every step, such as Anthropic's execution container, outlives changes to the step. + $reusable = $this->gateway instanceof ReplaysFullHistory + || ($step->messages === $history && $step->model === $previous?->model && $step->instructions === $previous->instructions); return new StepContext( stepNumber: $step->number, isFinalStep: $step->isFinalStep, - continuationToken: $unchanged ? $continuationToken : null, + continuationToken: $reusable ? $continuationToken : null, ); } @@ -764,11 +769,7 @@ protected function streamedStepToolResults(StepResponse $result, bool $isFinalSt ); } - $events = $this->executeAgentToolStreaming($tool, $toolCall->arguments, $toolCall->id, $context); - - yield from $this->preliminaryToolResults($events, $toolCall, $invocationId); - - return $this->recordedToolResult($this->toolResult($toolCall, $events->getReturn()), $context); + return yield from $this->streamedAgentToolResult($tool, $toolCall, $invocationId, $context); }, $context); return [$results, $pendingApprovals]; @@ -786,14 +787,28 @@ protected function streamedStepToolResults(StepResponse $result, bool $isFinalSt continue; } - $events = $this->executeAgentToolStreaming($tool, $toolCall->arguments, $toolCall->id, $context); + $toolResults[] = yield from $this->streamedAgentToolResult($tool, $toolCall, $invocationId, $context); + } + + return [$toolResults, $pendingApprovals]; + } + + /** + * Stream a sub-agent's output as preliminary results, then record its final or failed result. + * + * @return Generator + */ + private function streamedAgentToolResult(AgentTool $tool, ToolCall $toolCall, string $invocationId, ?RunContext $context): Generator + { + $events = $this->executeAgentToolStreaming($tool, $toolCall->arguments, $toolCall->id, $context); + try { yield from $this->preliminaryToolResults($events, $toolCall, $invocationId); - $toolResults[] = $this->recordedToolResult($this->toolResult($toolCall, $events->getReturn()), $context); + return $this->recordedToolResult($this->toolResult($toolCall, $events->getReturn()), $context); + } catch (ToolExecutionException $exception) { + return $this->recordedToolResult($this->failedToolResult($toolCall, $exception), $context); } - - return [$toolResults, $pendingApprovals]; } /** @@ -857,6 +872,7 @@ protected function executeAgentToolStreaming(AgentTool $tool, array $arguments, $startedAt = hrtime(true); + // Only the sub-agent itself may fail the tool call, so a listener that throws is never reported as a tool failure... try { $events = $tool->stream(new Request($arguments, $toolCallId, $toolInvocationId)); @@ -873,7 +889,8 @@ protected function executeAgentToolStreaming(AgentTool $tool, array $arguments, } catch (Throwable $exception) { // Keep the original exception, including cancellation, as previous if failure reporting throws. try { - throw $exception; + // Unlike upstream, cancellation is not wrapped, so the caller never recovers it as the tool's own failure. + throw Coroutine::causedByCancellation($exception) ? $exception : new ToolExecutionException($exception); } finally { $context?->toolFailed($tool, $arguments, $exception, $toolInvocationId, $this->elapsedMilliseconds($startedAt)); } @@ -1029,12 +1046,20 @@ protected function executedToolResult(ToolCall $toolCall, Tool $tool, ?RunContex try { return $this->toolResult($toolCall, $this->executeTool($tool, $toolCall->arguments, $toolCall->id, $context)); } catch (ToolExecutionException $exception) { - report($exception->exception); - - return $this->toolResult($toolCall, $this->failedToolResultText($exception->exception), failed: true); + return $this->failedToolResult($toolCall, $exception); } } + /** + * Report the given tool failure and turn it into a result the model can react to. + */ + protected function failedToolResult(ToolCall $toolCall, ToolExecutionException $exception): ToolResult + { + report($exception->exception); + + return $this->toolResult($toolCall, $this->failedToolResultText($exception->exception), failed: true); + } + /** * Get the result the model receives for a tool that threw. */ diff --git a/src/ai/src/Middleware/RememberConversation.php b/src/ai/src/Middleware/RememberConversation.php index 0b1cb91e34..d2a8d1a648 100644 --- a/src/ai/src/Middleware/RememberConversation.php +++ b/src/ai/src/Middleware/RememberConversation.php @@ -6,8 +6,8 @@ use Closure; use Generator; +use Hypervel\Ai\Agents\ConversationTitleAgent; use Hypervel\Ai\Ai; -use Hypervel\Ai\AnonymousAgent; use Hypervel\Ai\Concerns\RemembersConversations as RemembersConversationsTrait; use Hypervel\Ai\Contracts\Agent; use Hypervel\Ai\Contracts\ConversationStore; @@ -15,9 +15,11 @@ use Hypervel\Ai\Contracts\RemembersConversations; use Hypervel\Ai\Contracts\StoresConversationTurns; use Hypervel\Ai\Exceptions\StreamAbandonedException; +use Hypervel\Ai\Gateway\ParentInvocation; use Hypervel\Ai\Messages\UserMessage; use Hypervel\Ai\Models\Conversation; use Hypervel\Ai\Prompts\AgentPrompt; +use Hypervel\Ai\Providers\Provider; use Hypervel\Ai\Responses\AgentResponse; use Hypervel\Ai\Responses\StreamableAgentResponse; use Hypervel\Ai\Storage\StoredTurn; @@ -348,13 +350,12 @@ protected function startConcurrentTitle(AgentPrompt $prompt): ?PendingConversati /** @var Agent&RemembersConversations $agent */ $agent = $prompt->agent; - // Decisions are validated during generation, and agent fakes hand out responses in call order. + // Decisions are validated during generation. if (! config()->boolean('ai.conversations.concurrent_title', false) || ! config()->boolean('ai.conversations.generate_title', true) || $prompt->hasApprovalDecisions() || ! $this->startsConversation($prompt) - || ! $this->shouldRememberTurn($agent, $prompt) - || Ai::hasFakeGatewayFor($agent::class)) { + || ! $this->shouldRememberTurn($agent, $prompt)) { return null; } @@ -376,20 +377,18 @@ protected function generateTitle(AgentPrompt $prompt): string } try { - $response = $this->provider->prompt(new AgentPrompt( - new AnonymousAgent( - 'Generate a concise 3-5 word title for a conversation that starts with the following message. Use the same language as the message. Respond with only the title, no quotes or punctuation.', - [], - [], - ), + $agent = resolve(ConversationTitleAgent::class); + + // A faked conversation must not reach a real provider for its title... + if (Ai::hasFakeGatewayFor($prompt->agent) && ! $agent::isFaked()) { + $agent::fake(); + } + + // Unlike upstream, a provider instance is passed instead of its name, so an on-demand provider keeps its credentials. On-demand providers are always concrete; providers that only implement the contract are registered by name. + $response = ParentInvocation::within($prompt->invocationId, null, fn (): AgentResponse => $agent->prompt( Str::limit($prompt->prompt, 500), - [], - $this->provider, - $this->provider->cheapestTextModel(), - parentInvocationId: $prompt->invocationId, + provider: $this->provider instanceof Provider ? $this->provider : $this->provider->name(), )); - - return Str::limit($response->text, 100); } catch (Throwable $exception) { if (Coroutine::causedByCancellation($exception)) { throw $exception; @@ -397,5 +396,7 @@ protected function generateTitle(AgentPrompt $prompt): string return Str::limit($prompt->prompt, 100, preserveWords: true); } + + return Str::limit($response->text, 100); } } diff --git a/src/ai/src/Tools/AgentTool.php b/src/ai/src/Tools/AgentTool.php index c3a2babcd2..62c762bd14 100644 --- a/src/ai/src/Tools/AgentTool.php +++ b/src/ai/src/Tools/AgentTool.php @@ -15,9 +15,7 @@ use Hypervel\Ai\Streaming\Events\StreamEvent; use Hypervel\Contracts\Container\Transient; use Hypervel\Contracts\JsonSchema\JsonSchema; -use Hypervel\Coroutine\Coroutine; use Stringable; -use Throwable; class AgentTool implements Approvable, Tool, Transient { @@ -58,16 +56,7 @@ public function description(): Stringable|string */ public function handle(Request $request): string { - try { - return $this->agent->prompt((string) $request['task'])->text; - } catch (Throwable $throwable) { - if (Coroutine::causedByCancellation($throwable)) { - throw $throwable; - } - - // Degraded to a result so the parent run survives; the failure surfaces as the sub-agent's own AgentFailed... - return $this->failedResult($throwable); - } + return $this->agent->prompt((string) $request['task'])->text; } /** @@ -81,33 +70,11 @@ public function stream(Request $request): Generator return $this->handle($request); } - try { - $stream = $this->agent->stream((string) $request['task']); - - yield from $stream; - - return (string) $stream->text; - } catch (Throwable $throwable) { - if (Coroutine::causedByCancellation($throwable)) { - throw $throwable; - } - - return $this->failedResult($throwable); - } - } + $stream = $this->agent->stream((string) $request['task']); - /** - * Report the sub-agent's failure and get the result the parent agent receives. - */ - protected function failedResult(Throwable $throwable): string - { - // Unlike upstream, the failure is reported, since AgentFailed only reaches listeners and the parent run never sees the exception. - report($throwable); + yield from $stream; - // Unlike upstream, the exception message is only included in debug mode, matching ordinary tool failures... - return config()->boolean('app.debug') - ? 'Agent failed: ' . $throwable->getMessage() - : 'Agent failed.'; + return (string) $stream->text; } /** diff --git a/src/docs/ai-sdk.md b/src/docs/ai-sdk.md index da9f9a0081..3ecea9f1e7 100644 --- a/src/docs/ai-sdk.md +++ b/src/docs/ai-sdk.md @@ -677,7 +677,35 @@ The store uses your default database connection unless `ai.conversations.connect The SDK generates a title when it creates a conversation. To use the opening message as the title instead, set `conversations.generate_title` to `false` in your `config/ai.php` configuration file. -Title generation runs as its own agent run using the turn's provider and its cheapest model, so [agent events](#events) such as `AgentPrompted` and `AgentFailed` also report it. Its prompt's `parentInvocationId` identifies the turn that triggered it. If the title request fails, the SDK uses the start of the opening message without failing the turn. +Titles are generated by the `Hypervel\Ai\Agents\ConversationTitleAgent` agent, which runs on the turn's provider using its cheapest model, so [agent events](#events) such as `AgentPrompted` and `AgentFailed` also report it. Its prompt's `parentInvocationId` identifies the turn that triggered it. If the title request fails, the SDK uses the start of the opening message without failing the turn. + +If your application creates a conversation itself, you may use the same agent to title it: + +```php +use Hypervel\Ai\Agents\ConversationTitleAgent; + +$conversation->update([ + 'title' => app(ConversationTitleAgent::class)->prompt($message)->text, +]); +``` + +To customize how titles are generated, extend the agent and bind your agent in the container. PHP doesn't inherit attributes, so give your agent a model attribute if it shouldn't use the provider's default model: + +```php +use Hypervel\Ai\Agents\ConversationTitleAgent; +use Hypervel\Ai\Attributes\UseCheapestModel; + +#[UseCheapestModel] +class SupportTitleAgent extends ConversationTitleAgent +{ + public function instructions(): string + { + return 'Title this support thread in 3-6 words, naming the product area first.'; + } +} + +$this->app->bind(ConversationTitleAgent::class, SupportTitleAgent::class); +``` If the opening turn is canceled or abandoned, such as when the client disconnects, the SDK saves the work already done without starting or waiting for a title. It keeps a title that has already finished or uses the start of the opening message, and the title isn't generated later. Enabling concurrent titles makes a generated title more likely in this case. A turn that fails with an error is titled as usual. @@ -694,7 +722,7 @@ When starting a new conversation, concurrent title generation runs alongside the Concurrent titles run in a separate coroutine with a copy of the caller's `Context` values. They use the caller's current default database connection, such as one selected with `DB::usingConnection()`. If your provider configuration or other hooks depend on additional application context, register [context capture](#capturing-application-context) so that work sees the same values. -Agent fakes retain their usual response order by generating titles serially. When using HTTP fakes with concurrent titles, match requests by their content instead of relying on the order of a response sequence. The title and main requests can use the same provider endpoint. +When using HTTP fakes with concurrent titles, match requests by their content instead of relying on the order of a response sequence. The title and main requests can use the same provider endpoint. #### Conversation Participants @@ -2054,7 +2082,7 @@ class RefundsAgent implements Agent, CanActAsTool, HasTools If a sub-agent does not implement `CanActAsTool`, Hypervel will use the agent's class basename as the tool name and a generic description that asks the parent agent to pass a clear, self-contained task description. Each sub-agent invocation runs in isolation and does not receive the parent agent's conversation history. -If a sub-agent fails, the parent agent's run continues. The exception is reported to your application's exception handler, and the parent agent receives "Agent failed." as the tool's result. Like other [tool failures](#validating-tool-arguments), the result also includes the exception message when your application's `debug` configuration option is enabled. +If a sub-agent throws an exception, it is handled like any other [tool failure](#validating-tool-arguments): the parent agent's run continues, the exception is reported to your application's exception handler, and the parent agent receives a failed tool result. > [!WARNING] > Sub-agents do not currently support tools that require [human approval](#human-tool-approval). If a sub-agent's tool requires approval, the sub-agent pauses. However, the parent agent receives a normal tool result and continues its run without notifying your application of the pending approval. Register tools that require approval on the top-level agent. @@ -3805,6 +3833,14 @@ $response->reasoning; // 'They asked about pricing.' > **Note:** When `Agent::fake()` is invoked on an agent that returns structured output and fake output was not explicitly provided, Hypervel will automatically generate fake data that matches your agent's defined output schema. +When a faked agent starts a [conversation](#conversation-titles), the `ConversationTitleAgent` is faked for the rest of the test, so the title doesn't use one of the agent's fake responses. You may fake the title agent yourself to choose the title: + +```php +use Hypervel\Ai\Agents\ConversationTitleAgent; + +ConversationTitleAgent::fake(['Billing Question']); +``` + After prompting the agent, you may make assertions about the prompts that were received: ```php diff --git a/tests/Ai/ConcurrentToolsTest.php b/tests/Ai/ConcurrentToolsTest.php index 9aff7098af..bb562c4cac 100644 --- a/tests/Ai/ConcurrentToolsTest.php +++ b/tests/Ai/ConcurrentToolsTest.php @@ -38,6 +38,8 @@ use Hypervel\Engine\Coroutine as EngineCoroutine; use Hypervel\Support\Facades\Context; use Hypervel\Support\Facades\DB; +use Hypervel\Support\Facades\Exceptions; +use Hypervel\Testbench\Attributes\WithConfig; use Hypervel\Tests\Ai\Fixtures\Tools\NamedTool; use InvalidArgumentException; use JMac\Testing\Double; @@ -263,6 +265,39 @@ public function testAbandoningLiveSubAgentOutputCancelsAndJoinsItsProducers(): v $this->assertFalse($resumed); } + #[WithConfig('app.debug', false)] + public function testAFailingSubAgentReturnsAFailedResultWhileTheOtherToolsFinish(): void + { + Exceptions::fake(); + $agent = Double::for(Agent::class, CanActAsTool::class)->strict(); + $agent->allows('name')->returns('child'); + $agent->expects('stream')->times(2)->resolves(fn (string $task): StreamableAgentResponse => new StreamableAgentResponse($task, function () use ($task): Generator { + yield new TextDelta($task, $task, $task, time()); + + if ($task === 'first') { + throw new RuntimeException('The sub-agent failed.'); + } + })); + $provider = Double::for(TextProvider::class)->strict(); + $provider->allows('name')->returns('provider'); + $results = []; + $gateway = $this->gateway(true, [ + new ToolCall('first', 'child', ['task' => 'first']), + new ToolCall('second', 'child', ['task' => 'second']), + ], $results); + + iterator_to_array((new TextGenerationLoop($gateway))->stream( + 'invocation', + $provider, + 'model', + '', + tools: [new AgentTool($agent)], + options: new TextGenerationOptions(agent: new ConcurrentToolsTestAgent('', [], [])), + ), false); + + $this->assertSame(['The tool call failed.', 'second'], $results); + } + #[DataProvider('cancellationModes')] public function testCancelingTheConsumerCancelsAndJoinsTheTools(bool $throwException): void { diff --git a/tests/Ai/Feature/AgentEventsTest.php b/tests/Ai/Feature/AgentEventsTest.php index 8535d581c3..3428b49862 100644 --- a/tests/Ai/Feature/AgentEventsTest.php +++ b/tests/Ai/Feature/AgentEventsTest.php @@ -6,8 +6,11 @@ use Closure; use Exception; -use Hypervel\Ai\AnonymousAgent; +use Generator; +use Hypervel\Ai\Agents\ConversationTitleAgent; use Hypervel\Ai\Approvals\Decisions; +use Hypervel\Ai\Contracts\Agent; +use Hypervel\Ai\Contracts\CanActAsTool; use Hypervel\Ai\Contracts\Conversational; use Hypervel\Ai\Contracts\ConversationStore; use Hypervel\Ai\Events\AgentFailed; @@ -37,7 +40,10 @@ use Hypervel\Ai\Responses\Data\FinishReason; use Hypervel\Ai\Responses\Data\TextUsage; use Hypervel\Ai\Responses\Data\ToolCall; +use Hypervel\Ai\Responses\StreamableAgentResponse; use Hypervel\Ai\Streaming\Events\Error; +use Hypervel\Ai\Streaming\Events\TextDelta; +use Hypervel\Ai\Streaming\Events\ToolResult as ToolResultEvent; use Hypervel\Ai\Tools\AgentTool; use Hypervel\Coroutine\Coroutine; use Hypervel\Foundation\Testing\RefreshDatabase; @@ -72,6 +78,7 @@ use JMac\Testing\Double; use JMac\Testing\Integrations\PHPUnit\VerifiesDoubles; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use RuntimeException; use Swoole\Coroutine\CanceledException; use Throwable; @@ -378,7 +385,7 @@ public function testAFailedConversationTitleRunFallsBackToThePromptWithoutFailin $failed = Event::dispatched(AgentFailed::class)->sole()[0]; - $this->assertInstanceOf(AnonymousAgent::class, $failed->prompt->agent); + $this->assertInstanceOf(ConversationTitleAgent::class, $failed->prompt->agent); $this->assertSame($response->invocationId, $failed->prompt->parentInvocationId); $this->assertInstanceOf(RequestException::class, $failed->exception); } @@ -558,6 +565,99 @@ public function testAThrowingToolDispatchesToolFailedAndTheRunContinues(): void $this->assertSame($invoking->invocationId, $failed->invocationId); } + #[WithConfig('app.debug', true)] + public function testASubAgentThatThrowsDispatchesToolFailedAndTheRunContinues(): void + { + Event::fake(); + + DelegatingAgent::fake([ + new ToolCall('call_123', 'research_agent', ['task' => 'Research Hypervel']), + 'The research could not be completed.', + ]); + + ResearchAgent::fake(fn () => throw new RuntimeException('Research failed.')); + + $response = (new DelegatingAgent)->prompt('Delegate research about Hypervel'); + + $invoking = Event::dispatched(InvokingTool::class)->first()[0]; + + Event::assertDispatched(ToolFailed::class, fn (ToolFailed $event): bool => $event->toolInvocationId === $invoking->toolInvocationId + && $event->exception->getMessage() === 'Research failed.'); + + Event::assertNotDispatched(ToolInvoked::class); + + Event::assertDispatched(AgentFailed::class, fn (AgentFailed $event): bool => $event->prompt->agent instanceof ResearchAgent); + + $result = $response->steps->first()->toolResults[0]; + + $this->assertTrue($result->failed); + $this->assertSame('The tool call failed: Research failed.', $result->result); + } + + #[WithConfig('app.debug', true)] + public function testASubAgentThatThrowsWhileStreamedAsAToolDispatchesToolFailedAndTheRunContinues(): void + { + Event::fake(); + + DelegatingAgent::fake([ + new ToolCall('call_123', 'research_agent', ['task' => 'Research Hypervel']), + 'The research could not be completed.', + ]); + + ResearchAgent::fake(fn () => throw new RuntimeException('Research failed.')); + + $events = iterator_to_array((new DelegatingAgent)->stream('Delegate research about Hypervel'), false); + + $invoking = Event::dispatched(InvokingTool::class)->first()[0]; + + Event::assertDispatched(ToolFailed::class, fn (ToolFailed $event): bool => $event->toolInvocationId === $invoking->toolInvocationId + && $event->exception->getMessage() === 'Research failed.'); + + Event::assertNotDispatched(ToolInvoked::class); + + Event::assertDispatched(AgentFailed::class, fn (AgentFailed $event): bool => $event->prompt->agent instanceof ResearchAgent); + + $result = collect($events)->first(fn ($event): bool => $event instanceof ToolResultEvent && ! $event->preliminary); + + $this->assertFalse($result->successful); + $this->assertSame('The tool call failed: Research failed.', $result->error); + } + + #[WithConfig('app.debug', true)] + public function testASubAgentThatThrowsAfterStreamingOutputEndsItsToolCallWithAFailedResult(): void + { + Event::fake(); + + DelegatingAgent::fake([ + new ToolCall('call_123', 'middle_manager', ['task' => 'Research Hypervel']), + 'The research could not be completed.', + ]); + + // Unlike upstream, only text output produces a preliminary result, so the sub-agent streams a snapshot's worth of text before it throws... + $subAgent = Double::for(Agent::class, CanActAsTool::class)->strict(); + $subAgent->allows('name')->returns('middle_manager'); + $subAgent->allows('description')->returns('Delegate research tasks.'); + $subAgent->expects('stream')->returns(new StreamableAgentResponse('sub-invocation', function (): Generator { + yield new TextDelta('event-1', 'message-1', str_repeat('Hypervel is a PHP framework. ', 10), time()); + + throw new RuntimeException('Research failed.'); + })); + + $results = collect(iterator_to_array((new DelegatingAgent)->withTools([new AgentTool($subAgent)])->stream('Delegate research about Hypervel'), false)) + ->filter(fn ($event): bool => $event instanceof ToolResultEvent) + ->values(); + + Event::assertDispatched(ToolFailed::class, fn (ToolFailed $event): bool => $event->tool instanceof AgentTool + && $event->tool->name() === 'middle_manager' + && $event->exception->getMessage() === 'Research failed.'); + + $this->assertTrue($results->first()->preliminary); + $this->assertTrue($results->first()->successful); + $this->assertFalse($results->last()->preliminary); + $this->assertFalse($results->last()->successful); + $this->assertSame('The tool call failed: Research failed.', $results->last()->error); + } + #[WithConfig('app.debug', false)] public function testAToolThatFailsWhileResumingAnApprovalReportsTheFailureOnceAndContinuesTheRun(): void { @@ -1169,17 +1269,22 @@ public function testAbandonmentReportsBothTheParentAndNestedAgent(): void Event::assertNotDispatched(AgentFailed::class); } - public function testStreamingSubAgentCancellationReportsToolFailure(): void + #[TestWith([false])] + #[TestWith([true])] + public function testStreamingSubAgentCancellationReportsToolFailure(bool $cleanupThrows): void { Event::fake(); DelegatingAgent::fake([new ToolCall('research', 'research_agent', ['task' => 'Research'])]); $exception = new CanceledException('Research canceled.'); + if ($cleanupThrows) { + $exception = new RuntimeException('Research cleanup failed.', previous: $exception); + } ResearchAgent::fake([fn (): never => throw $exception]); try { iterator_to_array((new DelegatingAgent)->stream('Research')); $this->fail('Cancellation must reach the caller.'); - } catch (CanceledException $caught) { + } catch (Throwable $caught) { $this->assertSame($exception, $caught); } diff --git a/tests/Ai/Feature/AgentMiddlewareTest.php b/tests/Ai/Feature/AgentMiddlewareTest.php index 87c5819c75..13719128db 100644 --- a/tests/Ai/Feature/AgentMiddlewareTest.php +++ b/tests/Ai/Feature/AgentMiddlewareTest.php @@ -466,6 +466,7 @@ public function testAgentMiddlewareThatReplacesTheHistoryOrTheModelDropsTheProvi $this->assertSame([null, 'resp_1'], $run(fn (PendingStep $step, Closure $next): StepResult => $next($step))); $this->assertSame([null, null], $run(fn (PendingStep $step, Closure $next): StepResult => $next($step->withMessages(array_slice($step->messages, -1))))); + $this->assertSame([null, null], $run(fn (PendingStep $step, Closure $next): StepResult => $next($step->isFirstStep() ? $step->withMessages([new UserMessage('Summarized')]) : $step))); $this->assertSame([null, null], $run(fn (PendingStep $step, Closure $next): StepResult => $next($step->isFirstStep() ? $step : $step->withModel('cheaper-model')))); $this->assertSame([null, null], $run(fn (PendingStep $step, Closure $next): StepResult => $next($step->isFirstStep() ? $step->withModel('cheaper-model') : $step))); $this->assertSame([null, 'resp_1'], $run(fn (PendingStep $step, Closure $next): StepResult => $next($step->withModel('cheaper-model')))); diff --git a/tests/Ai/Feature/ConversationTitleTest.php b/tests/Ai/Feature/ConversationTitleTest.php new file mode 100644 index 0000000000..62d537de2f --- /dev/null +++ b/tests/Ai/Feature/ConversationTitleTest.php @@ -0,0 +1,185 @@ +make('config')->set('ai.providers.groq.key', 'test-key'); + } + + /** + * Reject provider requests the test did not fake. + */ + protected function setUp(): void + { + parent::setUp(); + + Http::preventStrayRequests(); + } + + public function testAConversationIsTitledByTheTitleAgentOnTheCheapestModelOfItsProvider(): void + { + Event::fake(); + + Http::fakeSequence() + ->pushResponse(fakeGroqResponse('Hello there.')) + ->pushResponse(fakeGroqResponse('Greeting Conversation')); + + $response = (new RememberingAssistantAgent)->forUser((object) ['id' => 1])->prompt('Hello', provider: 'groq'); + + $this->assertSame('Greeting Conversation', Conversation::find($response->conversationId)->title); + + Event::assertDispatched(AgentPrompted::class, fn (AgentPrompted $event): bool => $event->prompt->agent instanceof ConversationTitleAgent + && $event->prompt->prompt === 'Hello' + && $event->prompt->model === 'openai/gpt-oss-20b' + && $event->prompt->parentInvocationId === $response->invocationId); + } + + public function testATitleAgentBoundInTheContainerTitlesTheConversation(): void + { + Event::fake(); + + $this->app->bind(ConversationTitleAgent::class, SupportTitleAgent::class); + + Http::fakeSequence() + ->pushResponse(fakeGroqResponse('Hello there.')) + ->pushResponse(fakeGroqResponse('Billing Refund Request')); + + $this->assertSame('Billing Refund Request', $this->titleOfConversationStartedWith('I want a refund')); + + Event::assertDispatched(AgentPrompted::class, fn (AgentPrompted $event): bool => $event->prompt->agent instanceof SupportTitleAgent + && $event->prompt->model === 'llama-3.1-8b-instant'); + } + + public function testAFailedTitleRunFallsBackToTheMessage(): void + { + Http::fakeSequence() + ->pushResponse(fakeGroqResponse('Hello there.')) + ->pushStatus(500); + + $message = str_repeat('word ', 30); + + $this->assertSame(Str::limit($message, 100, preserveWords: true), $this->titleOfConversationStartedWith($message)); + } + + public function testATitleAgentThatCannotBeResolvedFallsBackToTheMessage(): void + { + $this->app->bind(ConversationTitleAgent::class, fn (): never => throw new RuntimeException('The title agent could not be built.')); + + Http::fakeSequence()->pushResponse(fakeGroqResponse('Hello there.')); + + $response = (new RememberingAssistantAgent)->forUser((object) ['id' => 1])->prompt('Hello', provider: 'groq'); + + $this->assertSame('Hello there.', $response->text); + $this->assertSame('Hello', Conversation::find($response->conversationId)->title); + } + + public function testDisabledTitleGenerationShortensTheMessageWithoutATitleRun(): void + { + config(['ai.conversations.generate_title' => false]); + + Http::fakeSequence()->pushResponse(fakeGroqResponse('Hello there.')); + + $this->assertSame('word word word word word word word word word...', $this->titleOfConversationStartedWith(str_repeat('word ', 20))); + + Http::assertSentCount(1); + } + + #[TestWith([false])] + #[TestWith([true])] + public function testAFakedConversationFakesItsTitleRunInsteadOfSpendingItsOwnResponses(bool $concurrent): void + { + config(['ai.conversations.concurrent_title' => $concurrent]); + + RememberingAssistantAgent::fake(['First answer', 'Second answer']); + + $user = (object) ['id' => 1]; + + $first = (new RememberingAssistantAgent)->forUser($user)->prompt('Hello'); + $second = (new RememberingAssistantAgent)->continue($first->conversationId, as: $user)->prompt('Again'); + + $this->assertSame('Second answer', $second->text); + $this->assertSame('Fake response for prompt: Hello', Conversation::find($first->conversationId)->title); + + ConversationTitleAgent::assertPrompted('Hello'); + } + + public function testAFakedTitleAgentTitlesAFakedConversation(): void + { + RememberingAssistantAgent::fake(['First answer']); + ConversationTitleAgent::fake(['Greeting Conversation']); + + $response = (new RememberingAssistantAgent)->forUser((object) ['id' => 1])->prompt('Hello'); + + $this->assertSame('First answer', $response->text); + $this->assertSame('Greeting Conversation', Conversation::find($response->conversationId)->title); + } + + public function testABoundTitleAgentIsFakedUnderItsOwnClass(): void + { + $this->app->bind(ConversationTitleAgent::class, SupportTitleAgent::class); + + RememberingAssistantAgent::fake(['First answer']); + SupportTitleAgent::fake(['Billing Refund Request']); + + $this->assertSame('Billing Refund Request', $this->titleOfConversationStartedWith('I want a refund')); + } + + public function testATitleRunsOnTheProviderInstanceOfTheTurn(): void + { + // Built in another coroutine, the on-demand provider is not registered under its name in this one... + [$provider] = parallel([fn (): Provider => Ai::build(['driver' => 'groq', 'key' => 'tenant-key'])]); + + Http::fakeSequence() + ->pushResponse(fakeGroqResponse('Hello there.')) + ->pushResponse(fakeGroqResponse('Greeting Conversation')); + + $response = (new RememberingAssistantAgent)->forUser((object) ['id' => 1])->prompt('Hello', provider: $provider); + + $this->assertSame('Greeting Conversation', Conversation::find($response->conversationId)->title); + $this->assertSame( + [['Bearer tenant-key'], ['Bearer tenant-key']], + Http::recorded()->map(fn (array $pair): array => $pair[0]->header('Authorization'))->all(), + ); + } + + /** + * Start a conversation with the given message and return its title. + */ + private function titleOfConversationStartedWith(string $message): string + { + $response = (new RememberingAssistantAgent)->forUser((object) ['id' => 1])->prompt($message, provider: 'groq'); + + return Conversation::find($response->conversationId)->title; + } +} diff --git a/tests/Ai/Feature/Providers/Anthropic/StreamingTest.php b/tests/Ai/Feature/Providers/Anthropic/StreamingTest.php index 7a82293603..8973ad5eb4 100644 --- a/tests/Ai/Feature/Providers/Anthropic/StreamingTest.php +++ b/tests/Ai/Feature/Providers/Anthropic/StreamingTest.php @@ -4,7 +4,11 @@ namespace Hypervel\Tests\Ai\Feature\Providers\Anthropic; +use Closure; use Hypervel\Ai\Exceptions\StreamErrorException; +use Hypervel\Ai\Gateway\StepResult; +use Hypervel\Ai\Messages\UserMessage; +use Hypervel\Ai\PendingStep; use Hypervel\Ai\Responses\Data\FinishReason; use Hypervel\Ai\Streaming\Events\Citation as CitationEvent; use Hypervel\Ai\Streaming\Events\Error; @@ -24,6 +28,7 @@ use Hypervel\Tests\Ai\Fixtures\AnthropicHelpers; use Hypervel\Tests\Ai\TestCase; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use stdClass; class StreamingTest extends TestCase @@ -182,6 +187,52 @@ public function testStreamingHandlesToolCalls(): void $this->assertSame('The number is 72019', TextDelta::combine($events)); } + #[TestWith([false])] + #[TestWith([true])] + public function testStreamingReusesTheCodeExecutionContainerOnTheFollowUpRequest(bool $editedFirstStep): void + { + Http::fake([ + 'api.anthropic.com/*' => Http::sequence([ + Http::response( + body: $this->ssePayload([ + $this->messageStart(), + $this->contentBlockStart(0, ['type' => 'tool_use', 'id' => 'toolu_1', 'name' => 'FixedNumberGenerator', 'input' => '']), + $this->contentBlockDelta(0, ['type' => 'input_json_delta', 'partial_json' => '{}']), + $this->contentBlockStop(0), + [ + 'type' => 'message_delta', + 'delta' => ['stop_reason' => 'tool_use', 'container' => ['id' => 'container_123', 'expires_at' => '2026-11-08T11:00:00Z']], + 'usage' => ['output_tokens' => 5], + ], + ]), + status: 200, + headers: ['Content-Type' => 'text/event-stream'], + ), + Http::response( + body: $this->ssePayload([ + $this->messageStart(), + $this->contentBlockStart(0, ['type' => 'text', 'text' => '']), + $this->contentBlockDelta(0, ['type' => 'text_delta', 'text' => 'The number is 72019']), + $this->contentBlockStop(0), + $this->messageDelta('end_turn', 10), + ]), + status: 200, + headers: ['Content-Type' => 'text/event-stream'], + ), + ]), + ]); + + $this->collectStreamEvents(agent: (new ProviderOptionsWithToolsAgent)->withMiddleware([ + fn (PendingStep $step, Closure $next): StepResult => $next($editedFirstStep && $step->isFirstStep() ? $step->withMessages([new UserMessage('Summarized')]) : $step), + ])); + + $recorded = Http::recorded(); + + $this->assertCount(2, $recorded); + $this->assertArrayNotHasKey('container', $recorded[0][0]->data()); + $this->assertSame('container_123', $recorded[1][0]->data()['container']); + } + public function testStreamingRunsToolCallsWhenTheMessageEndsWithEndTurn(): void { Http::fake([ diff --git a/tests/Ai/Feature/Providers/Anthropic/ToolCallLoopTest.php b/tests/Ai/Feature/Providers/Anthropic/ToolCallLoopTest.php index 201bf6220d..fa5bae897c 100644 --- a/tests/Ai/Feature/Providers/Anthropic/ToolCallLoopTest.php +++ b/tests/Ai/Feature/Providers/Anthropic/ToolCallLoopTest.php @@ -4,12 +4,23 @@ namespace Hypervel\Tests\Ai\Feature\Providers\Anthropic; +use Closure; +use Hypervel\Ai\Contracts\Agent; +use Hypervel\Ai\Contracts\HasProviderOptions; +use Hypervel\Ai\Contracts\HasTools; +use Hypervel\Ai\Enums\Lab; +use Hypervel\Ai\Gateway\StepResult; +use Hypervel\Ai\Messages\UserMessage; +use Hypervel\Ai\PendingStep; +use Hypervel\Ai\Promptable; use Hypervel\JsonSchema\JsonSchemaTypeFactory; use Hypervel\Support\Facades\Http; use Hypervel\Tests\Ai\Fixtures\Agents\ToolUsingAgent; use Hypervel\Tests\Ai\Fixtures\AnthropicHelpers; use Hypervel\Tests\Ai\Fixtures\Tools\FixedNumberGenerator; use Hypervel\Tests\Ai\TestCase; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use stdClass; use function Hypervel\Ai\agent; @@ -120,6 +131,158 @@ public function testToolCallsRunWhenTheMessageEndsWithEndTurn(): void $this->assertSame('The number is 72019', $response->text); } + #[TestWith([null])] + #[TestWith(['instructions'])] + #[TestWith(['messages'])] + public function testTheCodeExecutionContainerIsReusedByTheFollowUpRequest(?string $edit): void + { + Http::fake([ + 'api.anthropic.com/*' => Http::sequence([ + Http::response([ + 'id' => 'msg_tool_123', + 'type' => 'message', + 'role' => 'assistant', + 'model' => 'claude-sonnet-4-6', + 'content' => [ + ['type' => 'tool_use', 'id' => 'toolu_123', 'name' => 'FixedNumberGenerator', 'input' => new stdClass], + ], + 'container' => ['id' => 'container_123', 'expires_at' => '2026-11-08T11:00:00Z'], + 'stop_reason' => 'tool_use', + 'usage' => ['input_tokens' => 10, 'output_tokens' => 5], + ]), + $this->fakeTextResponse('The number is 72019'), + ]), + ]); + + (new ToolUsingAgent(fixed: true))->withMiddleware([ + fn (PendingStep $step, Closure $next): StepResult => $next(match ($edit) { + 'instructions' => $step->isFirstStep() ? $step : $step->withInstructions('Wrap up.'), + 'messages' => $step->isFirstStep() ? $step->withMessages([new UserMessage('Summarized')]) : $step, + default => $step, + }), + ])->prompt( + 'Generate a random number', + provider: 'anthropic', + ); + + $recorded = Http::recorded(); + + $this->assertCount(2, $recorded); + $this->assertArrayNotHasKey('container', $recorded[0][0]->data()); + $this->assertSame('container_123', $recorded[1][0]->data()['container']); + + if ($edit === 'instructions') { + $this->assertSame('Wrap up.', $recorded[1][0]->data()['system']); + } + } + + public function testTheCodeExecutionContainerIsReusedWhenAPausedTurnResumes(): void + { + Http::fake([ + 'api.anthropic.com/*' => Http::sequence([ + Http::response([ + 'id' => 'msg_pause', + 'type' => 'message', + 'role' => 'assistant', + 'model' => 'claude-sonnet-4-6', + 'content' => [ + ['type' => 'server_tool_use', 'id' => 'srvtoolu_1', 'name' => 'bash_code_execution', 'input' => (object) ['command' => 'ls']], + ], + 'container' => ['id' => 'container_123', 'expires_at' => '2026-11-08T11:00:00Z'], + 'stop_reason' => 'pause_turn', + 'usage' => ['input_tokens' => 10, 'output_tokens' => 5], + ]), + $this->fakeTextResponse('Done'), + ]), + ]); + + (new ToolUsingAgent(fixed: true))->prompt('Ask', provider: 'anthropic'); + + $recorded = Http::recorded(); + + $this->assertCount(2, $recorded); + $this->assertSame('container_123', $recorded[1][0]->data()['container']); + } + + #[DataProvider('providedContainers')] + public function testAContainerSetWithProviderOptionsCarriesOverToTheFollowUpRequest(array|string $container, array|string $followUp): void + { + Http::fake([ + 'api.anthropic.com/*' => Http::sequence([ + Http::response([ + 'id' => 'msg_tool_123', + 'type' => 'message', + 'role' => 'assistant', + 'model' => 'claude-sonnet-4-6', + 'content' => [ + ['type' => 'tool_use', 'id' => 'toolu_123', 'name' => 'FixedNumberGenerator', 'input' => new stdClass], + ], + 'container' => ['id' => 'container_123', 'expires_at' => '2026-11-08T11:00:00Z'], + 'stop_reason' => 'tool_use', + 'usage' => ['input_tokens' => 10, 'output_tokens' => 5], + ]), + $this->fakeTextResponse('The number is 72019'), + ]), + ]); + + $agent = new class($container) implements Agent, HasProviderOptions, HasTools { + use Promptable; + + /** + * Create an agent that sends the given container. + */ + public function __construct(protected array|string $container) + { + } + + /** + * Get the agent's instructions. + */ + public function instructions(): string + { + return 'You are a helpful assistant.'; + } + + /** + * Get the agent's tools. + */ + public function tools(): iterable + { + return [new FixedNumberGenerator]; + } + + /** + * Get the provider options. + */ + public function providerOptions(Lab|string $provider): array + { + return ['container' => $this->container]; + } + }; + + $agent->prompt('Generate a random number', provider: 'anthropic'); + + $recorded = Http::recorded(); + + $this->assertCount(2, $recorded); + $this->assertSame($container, $recorded[0][0]->data()['container']); + $this->assertSame($followUp, $recorded[1][0]->data()['container']); + } + + /** + * Provide containers set through provider options and the container each follow-up sends. + */ + public static function providedContainers(): array + { + return [ + 'skills' => [ + ['skills' => [['type' => 'anthropic', 'skill_id' => 'xlsx', 'version' => 'latest']]], + ['skills' => [['type' => 'anthropic', 'skill_id' => 'xlsx', 'version' => 'latest']], 'id' => 'container_123'], + ], + 'container id' => ['container_user', 'container_user'], + ]; + } + public function testToolCallsDoNotRunWhenTheMessageEndsWithMaxTokens(): void { Http::fake([ diff --git a/tests/Ai/Feature/Providers/Gemini/StreamingTest.php b/tests/Ai/Feature/Providers/Gemini/StreamingTest.php index c8dcb3d39d..442c8841bd 100644 --- a/tests/Ai/Feature/Providers/Gemini/StreamingTest.php +++ b/tests/Ai/Feature/Providers/Gemini/StreamingTest.php @@ -4,8 +4,12 @@ namespace Hypervel\Tests\Ai\Feature\Providers\Gemini; +use Closure; use GuzzleHttp\Promise\PromiseInterface; use Hypervel\Ai\Exceptions\StreamErrorException; +use Hypervel\Ai\Gateway\StepResult; +use Hypervel\Ai\Messages\UserMessage; +use Hypervel\Ai\PendingStep; use Hypervel\Ai\Responses\Data\FinishReason; use Hypervel\Ai\Streaming\Events\Citation as CitationEvent; use Hypervel\Ai\Streaming\Events\Error; @@ -23,6 +27,7 @@ use Hypervel\Ai\Tools\Request; use Hypervel\Support\Facades\Http; use Hypervel\Tests\Ai\Fixtures\Agents\ProviderOptionsWithToolsAgent; +use Hypervel\Tests\Ai\Fixtures\Agents\StoredToolAgent; use Hypervel\Tests\Ai\Fixtures\GeminiHelpers; use Hypervel\Tests\Ai\Fixtures\Tools\FixedNumberGenerator; use Hypervel\Tests\Ai\TestCase; @@ -555,6 +560,79 @@ public static function finishReasons(): array 'incomplete maps to Length' => ['incomplete', FinishReason::Length], ]; } + + public function testAStoredStreamIsContinuedWithPreviousInteractionId(): void + { + Http::fake([ + 'generativelanguage.googleapis.com/*' => Http::sequence() + ->push(...$this->geminiStream([ + $this->stepStart(0, ['type' => 'function_call', 'id' => 'call_1', 'name' => 'FixedNumberGenerator']), + $this->stepStop(0), + $this->interactionCompleted(), + ])) + ->push(...$this->geminiStream([ + $this->stepDelta(0, 'text', 'The number is 72019'), + $this->interactionCompleted(), + ])), + ]); + + $this->collectStreamEvents(agent: new StoredToolAgent); + + $followUp = Http::recorded()[1][0]->data(); + + $this->assertSame('int_123', $followUp['previous_interaction_id']); + $this->assertCount(1, $followUp['input']); + $this->assertSame('function_result', $followUp['input'][0]['type']); + $this->assertSame('call_1', $followUp['input'][0]['call_id']); + } + + public function testAStoredStreamWithoutAnInteractionIdReplaysTheFullHistory(): void + { + Http::fake([ + 'generativelanguage.googleapis.com/*' => Http::sequence() + ->push(...$this->geminiStream([ + $this->stepStart(0, ['type' => 'function_call', 'id' => 'call_1', 'name' => 'FixedNumberGenerator']), + $this->stepStop(0), + ['event_type' => 'interaction.completed', 'status' => 'completed'], + ])) + ->push(...$this->geminiStream([ + $this->stepDelta(0, 'text', 'Done'), + $this->interactionCompleted(), + ])), + ]); + + $this->collectStreamEvents(agent: new StoredToolAgent); + + $followUp = Http::recorded()[1][0]->data(); + + $this->assertArrayNotHasKey('previous_interaction_id', $followUp); + $this->assertSame(['user_input', 'function_call', 'function_result'], array_column($followUp['input'], 'type')); + } + + public function testAStoredStreamWhoseFirstStepEditedItsMessagesReplaysTheFullHistory(): void + { + Http::fake([ + 'generativelanguage.googleapis.com/*' => Http::sequence() + ->push(...$this->geminiStream([ + $this->stepStart(0, ['type' => 'function_call', 'id' => 'call_1', 'name' => 'FixedNumberGenerator']), + $this->stepStop(0), + $this->interactionCompleted(), + ])) + ->push(...$this->geminiStream([ + $this->stepDelta(0, 'text', 'Done'), + $this->interactionCompleted(), + ])), + ]); + + $this->collectStreamEvents(agent: (new StoredToolAgent)->withMiddleware([ + fn (PendingStep $step, Closure $next): StepResult => $next($step->isFirstStep() ? $step->withMessages([new UserMessage('Summarized')]) : $step), + ])); + + $followUp = Http::recorded()[1][0]->data(); + + $this->assertArrayNotHasKey('previous_interaction_id', $followUp); + $this->assertSame(['user_input', 'function_call', 'function_result'], array_column($followUp['input'], 'type')); + } } class GeminiStreamingCountingTool extends FixedNumberGenerator diff --git a/tests/Ai/Feature/Providers/Gemini/ToolCallLoopTest.php b/tests/Ai/Feature/Providers/Gemini/ToolCallLoopTest.php index b518883a0e..fde41511f0 100644 --- a/tests/Ai/Feature/Providers/Gemini/ToolCallLoopTest.php +++ b/tests/Ai/Feature/Providers/Gemini/ToolCallLoopTest.php @@ -4,12 +4,21 @@ namespace Hypervel\Tests\Ai\Feature\Providers\Gemini; +use Hypervel\Ai\Contracts\Agent; +use Hypervel\Ai\Contracts\HasProviderOptions; +use Hypervel\Ai\Contracts\HasStructuredOutput; +use Hypervel\Ai\Contracts\HasTools; +use Hypervel\Ai\Enums\Lab; use Hypervel\Ai\Exceptions\NoSuchToolException; +use Hypervel\Ai\Promptable; +use Hypervel\Contracts\JsonSchema\JsonSchema; use Hypervel\Support\Facades\Http; use Hypervel\Tests\Ai\Fixtures\Agents\MultiStepToolAgent; use Hypervel\Tests\Ai\Fixtures\Agents\NamedToolAgent; +use Hypervel\Tests\Ai\Fixtures\Agents\StoredToolAgent; use Hypervel\Tests\Ai\Fixtures\Agents\ToolUsingAgent; use Hypervel\Tests\Ai\Fixtures\GeminiHelpers; +use Hypervel\Tests\Ai\Fixtures\Tools\FixedNumberGenerator; use Hypervel\Tests\Ai\TestCase; /** @@ -220,4 +229,122 @@ public function testTheUserInputEchoedByGeminiIsNotReplayedTwice(): void $this->assertCount(1, geminiFollowUpSteps(Http::recorded(), 'user_input')); } + + public function testAStoredInteractionIsContinuedWithPreviousInteractionId(): void + { + Http::fake([ + 'generativelanguage.googleapis.com/*' => Http::sequence([ + $this->fakeUniqueToolCallResponse(), + $this->fakeTextResponse('The number is 72019'), + ]), + ]); + + (new StoredToolAgent)->prompt('Generate a random number', provider: 'gemini'); + + $recorded = Http::recorded(); + + $followUp = $recorded[1][0]->data(); + + $this->assertArrayNotHasKey('previous_interaction_id', $recorded[0][0]->data()); + $this->assertSame('int_123', $followUp['previous_interaction_id']); + $this->assertCount(1, $followUp['input']); + $this->assertSame('function_result', $followUp['input'][0]['type']); + $this->assertSame('FixedNumberGenerator', $followUp['input'][0]['name']); + $this->assertNotEmpty($followUp['system_instruction']); + $this->assertNotEmpty($followUp['tools']); + } + + public function testEachStepOfAStoredToolLoopContinuesFromTheInteractionBeforeIt(): void + { + Http::fake([ + 'generativelanguage.googleapis.com/*' => Http::sequence([ + Http::response(array_merge($this->fakeInteraction([$this->functionCallStep('FixedNumberGenerator', [], 'call_1')]), ['id' => 'int_1'])), + Http::response(array_merge($this->fakeInteraction([$this->functionCallStep('FixedNumberGenerator', [], 'call_2')]), ['id' => 'int_2'])), + $this->fakeTextResponse('Done'), + ]), + ]); + + (new StoredToolAgent)->prompt('Generate numbers', provider: 'gemini'); + + $recorded = Http::recorded(); + + $this->assertCount(3, $recorded); + $this->assertSame('int_1', $recorded[1][0]->data()['previous_interaction_id']); + $this->assertSame('call_1', $recorded[1][0]->data()['input'][0]['call_id']); + $this->assertSame('int_2', $recorded[2][0]->data()['previous_interaction_id']); + $this->assertSame('call_2', $recorded[2][0]->data()['input'][0]['call_id']); + } + + public function testAStoredInteractionContinuesFromTheCallersOwnPreviousInteractionOnlyOnTheFirstStep(): void + { + Http::fake([ + 'generativelanguage.googleapis.com/*' => Http::sequence([ + $this->fakeUniqueToolCallResponse(), + $this->fakeTextResponse('Done'), + ]), + ]); + + (new StoredToolAgent(['previous_interaction_id' => 'int_earlier']))->prompt('Generate a random number', provider: 'gemini'); + + $recorded = Http::recorded(); + + $this->assertSame('int_earlier', $recorded[0][0]->data()['previous_interaction_id']); + $this->assertSame('int_123', $recorded[1][0]->data()['previous_interaction_id']); + } + + public function testAContinuationCarriesTheSameSchemaAndGenerationConfigAsTheFirstRequest(): void + { + Http::fake([ + 'generativelanguage.googleapis.com/*' => Http::sequence([ + $this->fakeUniqueToolCallResponse(), + $this->fakeStructuredResponse(['number' => 72019]), + ]), + ]); + + $agent = new class implements Agent, HasProviderOptions, HasStructuredOutput, HasTools { + use Promptable; + + /** + * Get the agent's instructions. + */ + public function instructions(): string + { + return 'You are a helpful assistant that generates numbers.'; + } + + /** + * Get the agent's tools. + */ + public function tools(): iterable + { + return [new FixedNumberGenerator]; + } + + /** + * Get the agent's structured output schema. + */ + public function schema(JsonSchema $schema): array + { + return ['number' => $schema->integer()->required()]; + } + + /** + * Get the provider options. + */ + public function providerOptions(Lab|string $provider): array + { + return ['store' => true, 'thinking_level' => 'high']; + } + }; + + $agent->prompt('Generate a random number', provider: 'gemini'); + + [$first, $followUp] = Http::recorded()->map(fn (array $pair): array => $pair[0]->data())->all(); + + $this->assertSame('int_123', $followUp['previous_interaction_id']); + $this->assertSame($first['response_format'], $followUp['response_format']); + $this->assertSame('application/json', $followUp['response_format']['mime_type']); + $this->assertSame($first['generation_config'], $followUp['generation_config']); + $this->assertSame('high', $followUp['generation_config']['thinking_level']); + } } diff --git a/tests/Ai/Feature/RememberConversationTest.php b/tests/Ai/Feature/RememberConversationTest.php index 685862fbdc..0a89ec3668 100644 --- a/tests/Ai/Feature/RememberConversationTest.php +++ b/tests/Ai/Feature/RememberConversationTest.php @@ -7,6 +7,7 @@ use Closure; use Generator; use GuzzleHttp\Promise\PromiseInterface; +use Hypervel\Ai\Agents\ConversationTitleAgent; use Hypervel\Ai\Ai; use Hypervel\Ai\Approvals\Decisions; use Hypervel\Ai\Approvals\PendingApproval; @@ -735,28 +736,6 @@ public function testUnconsumedStreamsDoNotStartTitlesAndAbandonmentJoinsThem(): } } - public function testAgentFakesKeepSerialTitleOrderingWhenConcurrencyIsEnabled(): void - { - config(['ai.conversations.generate_title' => true, 'ai.conversations.concurrent_title' => true]); - $prompt = $this->prompt(); - Ai::fakeAgent($prompt->agent::class); - $generated = false; - $middleware = Double::for(new RememberConversation(new DatabaseConversationStore, $prompt->provider()))->passthru(); - $middleware->expects('generateTitle')->resolves(function () use (&$generated): string { - $this->assertTrue($generated); - - return 'Fake conversation'; - }); - - $middleware->handle($prompt, function () use (&$generated): AgentResponse { - $generated = true; - - return $this->response(); - }); - - $this->assertDatabaseHas('agent_conversations', ['title' => 'Fake conversation']); - } - public function testCancelingTheStreamConsumerJoinsItsTitleRequest(): void { config(['ai.conversations.generate_title' => true, 'ai.conversations.concurrent_title' => true]); @@ -919,21 +898,35 @@ public static function lateTitleOutcomes(): array return ['failed turn' => [true], 'completed turn' => [false]]; } + public function testAContractOnlyProviderTitlesTheConversationUnderItsName(): void + { + config(['ai.conversations.generate_title' => true]); + $prompt = $this->prompt(); + $provider = $prompt->provider(); + Ai::extend('test', static fn (): TextProvider => $provider); + $provider->expects('cheapestTextModel')->returns('title-model'); + $provider->expects('prompt')->resolves(static fn (AgentPrompt $title): AgentResponse => new AgentResponse($title->invocationId, 'Greeting Conversation', new TextUsage, new Meta('test', 'title-model'))); + + $response = $this->response(); + (new RememberConversation(new DatabaseConversationStore, $provider))->handle($prompt, fn (): AgentResponse => $response); + + $this->assertDatabaseHas('agent_conversations', ['id' => $response->conversationId, 'title' => 'Greeting Conversation']); + } + #[TestWith([false])] #[TestWith([true])] public function testTitleCancellationSavesTheCompletedTurnBeforeRethrowing(bool $cleanupThrows): void { - config(['ai.conversations.generate_title' => true]); + config(['ai.conversations.generate_title' => true, 'ai.providers.openai.key' => 'test-key']); $prompt = $this->prompt(); $failure = new CanceledException('Title request canceled'); if ($cleanupThrows) { $failure = new RuntimeException('Title cleanup failed.', previous: $failure); } - $prompt->provider()->expects('prompt')->throws($failure); - $prompt->provider()->expects('cheapestTextModel')->returns('title-model'); + ConversationTitleAgent::fake(fn () => throw $failure); $response = $this->response(); - $this->assertThrowsSame($failure, fn () => (new RememberConversation(new DatabaseConversationStore, $prompt->provider())) + $this->assertThrowsSame($failure, fn () => (new RememberConversation(new DatabaseConversationStore, Ai::textProvider('openai'))) ->handle($prompt, fn (): AgentResponse => $response)); $this->assertDatabaseHas('agent_conversations', ['id' => $response->conversationId, 'title' => 'Hello']); @@ -943,18 +936,17 @@ public function testTitleCancellationSavesTheCompletedTurnBeforeRethrowing(bool public function testStorageFailurePreservesTitleCancellation(): void { - config(['ai.conversations.generate_title' => true]); + config(['ai.conversations.generate_title' => true, 'ai.providers.openai.key' => 'test-key']); $prompt = $this->prompt(); $cancellation = new CanceledException('Title request canceled.'); $storageFailure = new RuntimeException('Conversation storage failed.'); - $prompt->provider()->expects('prompt')->throws($cancellation); - $prompt->provider()->expects('cheapestTextModel')->returns('title-model'); + ConversationTitleAgent::fake(fn () => throw $cancellation); $store = Double::for(DatabaseConversationStore::class)->strict(); $store->expects('storeTurn')->throws($storageFailure); $caught = null; try { - (new RememberConversation($store, $prompt->provider())) + (new RememberConversation($store, Ai::textProvider('openai'))) ->handle($prompt, fn () => $this->response()); } catch (Throwable $exception) { $caught = $exception; diff --git a/tests/Ai/Feature/ToolFailureRecoveryTest.php b/tests/Ai/Feature/ToolFailureRecoveryTest.php index ff48c74e1d..779634ba22 100644 --- a/tests/Ai/Feature/ToolFailureRecoveryTest.php +++ b/tests/Ai/Feature/ToolFailureRecoveryTest.php @@ -14,8 +14,11 @@ use Hypervel\Support\Facades\Event; use Hypervel\Support\Facades\Exceptions; use Hypervel\Testbench\Attributes\WithConfig; +use Hypervel\Tests\Ai\Fixtures\Agents\DelegatingAgent; +use Hypervel\Tests\Ai\Fixtures\Agents\ResearchAgent; use Hypervel\Tests\Ai\Fixtures\Agents\ThrowingToolAgent; use Hypervel\Tests\Ai\TestCase; +use PHPUnit\Framework\Attributes\TestWith; use RuntimeException; class ToolFailureRecoveryTest extends TestCase @@ -81,6 +84,37 @@ public function testAThrowingToolStreamsAFailedResult(): void Exceptions::assertReported(fn (Exception $exception): bool => $exception->getMessage() === 'Forced to throw exception.'); } + #[TestWith([false], 'prompt')] + #[TestWith([true], 'stream')] + #[WithConfig('app.debug', false)] + public function testAThrowingSubAgentIsReportedOnceAndHiddenFromTheModelOutsideDebugMode(bool $streaming): void + { + Exceptions::fake(); + $exception = new RuntimeException('Research failed.'); + + DelegatingAgent::fake([ + new ToolCall('call_1', 'research_agent', ['task' => 'Research']), + 'The research could not be completed.', + ]); + ResearchAgent::fake(fn () => throw $exception); + + if ($streaming) { + $events = iterator_to_array((new DelegatingAgent)->stream('Research'), false); + $result = collect($events)->first(fn ($event): bool => $event instanceof ToolResult && ! $event->preliminary); + + $this->assertFalse($result->successful); + $this->assertSame('The tool call failed.', $result->error); + } else { + $response = (new DelegatingAgent)->prompt('Research'); + + $this->assertFalse($response->toolResults[0]->successful()); + $this->assertSame('The tool call failed.', $response->toolResults[0]->error()); + } + + Exceptions::assertReportedCount(1); + Exceptions::assertReported(fn (RuntimeException $reported): bool => $reported === $exception); + } + public function testAnExceptionFromAToolEventListenerStillFailsTheRun(): void { Exceptions::fake(); diff --git a/tests/Ai/Fixtures/Agents/StoredToolAgent.php b/tests/Ai/Fixtures/Agents/StoredToolAgent.php new file mode 100644 index 0000000000..b24c772c75 --- /dev/null +++ b/tests/Ai/Fixtures/Agents/StoredToolAgent.php @@ -0,0 +1,56 @@ + true, ...$this->options]; + } +} diff --git a/tests/Ai/Fixtures/Agents/SupportTitleAgent.php b/tests/Ai/Fixtures/Agents/SupportTitleAgent.php new file mode 100644 index 0000000000..044e2ba635 --- /dev/null +++ b/tests/Ai/Fixtures/Agents/SupportTitleAgent.php @@ -0,0 +1,20 @@ +assertSame('{"answer":42}', $generator->getReturn()); } - #[WithConfig('app.debug', true)] - public function testAFailingSubAgentSurfacesItsErrorAsTheToolResultOnTheStreamingPath(): void + public function testAFailingSubAgentThrowsItsErrorOnTheStreamingPath(): void { - Exceptions::fake(); $exception = new RuntimeException('provider exploded'); $agent = Double::for(Agent::class)->strict(); @@ -78,91 +71,11 @@ public function testAFailingSubAgentSurfacesItsErrorAsTheToolResultOnTheStreamin $generator = (new AgentTool($agent))->stream(new Request(['task' => 'Do the thing'])); - $this->assertSame([], iterator_to_array($generator)); - $this->assertSame('Agent failed: provider exploded', $generator->getReturn()); - - Exceptions::assertReportedCount(1); - Exceptions::assertReported(fn (RuntimeException $reported): bool => $reported === $exception); - } - - #[DataProvider('promptedFailureModes')] - #[WithConfig('app.debug', false)] - public function testAFailingPromptedSubAgentIsReportedOnceAndHidesItsErrorOutsideDebugMode(bool $structuredStream): void - { - Exceptions::fake(); - $exception = new RuntimeException('provider exploded'); - - $agent = $structuredStream ? Double::for(Agent::class, HasStructuredOutput::class)->strict() : Double::for(Agent::class)->strict(); - $agent->expects('stream')->never(); - $agent->expects('prompt')->throws($exception); - - $tool = new AgentTool($agent); - $request = new Request(['task' => 'Do the thing']); - - if ($structuredStream) { - $generator = $tool->stream($request); - - $this->assertSame([], iterator_to_array($generator)); - $this->assertSame('Agent failed.', $generator->getReturn()); - } else { - $this->assertSame('Agent failed.', $tool->handle($request)); - } - - Exceptions::assertReportedCount(1); - Exceptions::assertReported(fn (RuntimeException $reported): bool => $reported === $exception); - } - - /** - * Provide the prompt path and the structured sub-agent's streaming path, which prompts. - */ - public static function promptedFailureModes(): array - { - return [ - 'prompt' => [false], - 'structured stream' => [true], - ]; - } - - #[DataProvider('deliveryModes')] - public function testCancellationEscapesTheSubAgent(bool $streaming, bool $cleanupThrows): void - { - $exception = new CanceledException('Sub-agent canceled.'); - if ($cleanupThrows) { - $exception = new RuntimeException('Sub-agent cleanup failed.', previous: $exception); - } - $agent = Double::for(Agent::class)->strict(); - - if ($streaming) { - $agent->expects('stream')->returns(new StreamableAgentResponse('invocation', function () use ($exception): Generator { - yield new TextDelta('event', 'message', 'Partial', time()); - - throw $exception; - })); - } else { - $agent->expects('prompt')->throws($exception); - } - - $tool = new AgentTool($agent); - $request = new Request(['task' => 'Do the thing']); - try { - $streaming ? iterator_to_array($tool->stream($request)) : $tool->handle($request); - $this->fail('Cancellation must reach the parent agent.'); - } catch (Throwable $caught) { + iterator_to_array($generator); + $this->fail('The sub-agent exception was not thrown.'); + } catch (RuntimeException $caught) { $this->assertSame($exception, $caught); } } - - /** - * Provide synchronous and streaming cancellation with normal and throwing cleanup. - */ - public static function deliveryModes(): array - { - return [ - 'prompt' => [false, false], - 'stream' => [true, false], - 'prompt cleanup failure' => [false, true], - 'stream cleanup failure' => [true, true], - ]; - } } From 6d566e5ef6855cbd29a5d2b86a583fba21db965c Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:39:31 +0000 Subject: [PATCH 04/30] Keep rows with a null column in PostgreSQL multi-column full-text search On PostgreSQL, whereFullText() over several columns concatenated their vectors with ||, so a null in any column made the whole vector null and the row never matched, even when another column held the search term. Each vector of a multi-column search is now wrapped in coalesce(..., ''::tsvector), for both to_tsvector expressions and stored vector columns. The schema grammar's multi-column full-text index uses the same expression, which PostgreSQL requires before a search can use the index. Single-column searches and indexes are unchanged. Laravel has the same behavior, so the porting guide notes that existing multi-column full-text indexes must be recreated to match. Tests: the Laravel-ported SQL assertions now expect the coalesced expressions, and the PostgreSQL integration tests cover a row with a null column and an EXPLAIN check that the search uses the multi-column index. Validation: the changed test files pass on PostgreSQL 17; formatting and full static analysis pass. --- src/database/README.md | 1 + .../src/Query/Grammars/PostgresGrammar.php | 12 ++++++++-- .../src/Schema/Grammars/PostgresGrammar.php | 6 +++++ src/docs/porting-from-laravel.md | 2 ++ src/docs/queries.md | 2 ++ .../DatabasePostgresSchemaGrammarTest.php | 2 +- tests/Database/DatabaseQueryBuilderTest.php | 6 ++--- .../Database/Postgres/FulltextTest.php | 24 ++++++++++++++++++- 8 files changed, 48 insertions(+), 7 deletions(-) diff --git a/src/database/README.md b/src/database/README.md index 0f8721558d..b2a77cf0c2 100644 --- a/src/database/README.md +++ b/src/database/README.md @@ -12,6 +12,7 @@ Documentation: https://hypervel.org/docs/database - Default retry and lost-connection detection recognizes PDO failures, not arbitrary exception text. Non-PDO drivers provide their own policies through `LostConnectionDetector::detectUsing()` for lost connections or the existing detector bindings; see [detecting database failures](https://hypervel.org/docs/database#detecting-database-failures). - `DB::transaction()` throws if its closure manually ends the transaction it owns. Let `transaction()` manage completion, or use the manual transaction API separately. Balanced nested transactions remain supported; see [database transactions](https://hypervel.org/docs/database#database-transactions). - Transaction records use the resolved connection's `getNameWithReadWriteType()`, not its base `getName()`, so distinct base, read and write sessions own separate callbacks. Extensions inspecting transaction records must compare against the resolved owner's full name. +- PostgreSQL multi-column full text indexes and `whereFullText` queries coalesce each column's vector, so a null column doesn't hide a row. Recreate multi-column full text indexes created by Laravel's grammar so searches can use them; see [full text where clauses](https://hypervel.org/docs/queries#full-text-where-clauses). - SQLite requires 3.32.0 or later rather than Laravel's 3.26.0 minimum, so framework code assumes the newer default limit of 32,766 bindings per statement. - Custom schema metadata grammars and processors must use Hypervel's paired row formats. MySQL/MariaDB and PostgreSQL use ordered index and foreign-key rows; SQLite supplies native automatic-key metadata and encoded key lists. See [extending database connections](https://hypervel.org/docs/database#extending-database-connections). - MariaDB has a higher [minimum version](https://hypervel.org/docs/database#introduction) than Laravel. Supported servers provide native column and index renaming and UUID columns without legacy compatibility paths. diff --git a/src/database/src/Query/Grammars/PostgresGrammar.php b/src/database/src/Query/Grammars/PostgresGrammar.php index 717e4fa5c0..181e9d1d41 100755 --- a/src/database/src/Query/Grammars/PostgresGrammar.php +++ b/src/database/src/Query/Grammars/PostgresGrammar.php @@ -147,8 +147,16 @@ public function whereFullText(Builder $query, array $where): string $columns = (new Collection($where['columns'])) ->map(fn ($column) => $isVector ? $this->wrap($column) - : "to_tsvector('{$language}', {$this->wrap($column)})") - ->implode(' || '); + : "to_tsvector('{$language}', {$this->wrap($column)})"); + + // Unlike Laravel, each vector of a multi-column search is coalesced, since a null column would make the whole + // concatenation null and hide the row. The schema grammar's full-text index uses the same expression, which + // PostgreSQL requires before it can use the index. + if ($columns->count() > 1) { + $columns = $columns->map(fn (string $vector): string => "coalesce({$vector}, ''::tsvector)"); + } + + $columns = $columns->implode(' || '); $mode = 'plainto_tsquery'; diff --git a/src/database/src/Schema/Grammars/PostgresGrammar.php b/src/database/src/Schema/Grammars/PostgresGrammar.php index b9cfa63f3d..b72be3a857 100755 --- a/src/database/src/Schema/Grammars/PostgresGrammar.php +++ b/src/database/src/Schema/Grammars/PostgresGrammar.php @@ -449,6 +449,12 @@ public function compileFulltext(Blueprint $blueprint, Fluent $command): string return "to_tsvector({$this->quoteString($language)}, {$this->wrap($column)})"; }, $command->columns); + // Unlike Laravel, each vector of a multi-column index is coalesced, matching the query grammar's full-text + // expression so a null column doesn't hide the row and searches can still use the index. + if (count($columns) > 1) { + $columns = array_map(fn (string $vector): string => "coalesce({$vector}, ''::tsvector)", $columns); + } + return sprintf( 'create index %s%s on %s using gin ((%s))', $command->online ? 'concurrently ' : '', diff --git a/src/docs/porting-from-laravel.md b/src/docs/porting-from-laravel.md index 7704e08739..87067cbadd 100644 --- a/src/docs/porting-from-laravel.md +++ b/src/docs/porting-from-laravel.md @@ -848,6 +848,8 @@ For schema dumps and loads over TLS, use MySQL command-line tools version 5.7.11 SQLite JSON-path updates replace assigned objects and retain JSON null. Review any reliance on Laravel's object merging or null-key deletion when [updating JSON columns](/docs/{{version}}/queries#updating-json-columns). +PostgreSQL multi-column full text indexes and `whereFullText` queries coalesce each column, so a null column no longer hides a row. Indexes created by Laravel's grammar no longer match these queries; recreate existing multi-column full text indexes so searches can use them. See [full text where clauses](/docs/{{version}}/queries#full-text-where-clauses). + Review pagination on update and delete queries. SQLite and PostgreSQL honor offsets, including without a limit. MySQL and MariaDB reject write offsets and limits on joined updates, and PostgreSQL rejects limits or offsets on `updateFrom`, instead of silently ignoring them. For these unsupported forms, select the intended primary keys first and write those rows explicitly. For cursor or ID-based pagination across unions, select plain columns in corresponding positions, or use `fromSub` to paginate the combined output. Hypervel rejects projections whose correspondence cannot be established, including computed paging columns. For query-builder `chunk`, `lazy`, and `cursorPaginate` calls, add `orderBy` after the union; an order on the first query alone throws. Union members use positional mapping instead of the protected `getOriginalColumnNameForCursorPagination` override; root queries retain that override. See [paginating unions](/docs/{{version}}/queries#paginating-unions). diff --git a/src/docs/queries.md b/src/docs/queries.md index 57c6d249d1..fa0bc9d6d9 100644 --- a/src/docs/queries.md +++ b/src/docs/queries.md @@ -1433,6 +1433,8 @@ $users = DB::table('users') ->get(); ``` +When searching several PostgreSQL columns, a row with a null column can still match the others. Multi-column full text indexes use the same expression, so these searches can use the index. + ### Vector Similarity Clauses diff --git a/tests/Database/DatabasePostgresSchemaGrammarTest.php b/tests/Database/DatabasePostgresSchemaGrammarTest.php index 6045288259..a025a8a1d0 100755 --- a/tests/Database/DatabasePostgresSchemaGrammarTest.php +++ b/tests/Database/DatabasePostgresSchemaGrammarTest.php @@ -579,7 +579,7 @@ public function testAddingFulltextIndexMultipleColumns(): void $statements = $blueprint->toSql(); $this->assertCount(1, $statements); - $this->assertSame('create index "users_body_title_fulltext" on "users" using gin ((to_tsvector(\'english\', "body") || to_tsvector(\'english\', "title")))', $statements[0]); + $this->assertSame('create index "users_body_title_fulltext" on "users" using gin ((coalesce(to_tsvector(\'english\', "body"), \'\'::tsvector) || coalesce(to_tsvector(\'english\', "title"), \'\'::tsvector)))', $statements[0]); } public function testAddingFulltextIndexWithLanguage(): void diff --git a/tests/Database/DatabaseQueryBuilderTest.php b/tests/Database/DatabaseQueryBuilderTest.php index 55d9df2827..40ed8f886f 100755 --- a/tests/Database/DatabaseQueryBuilderTest.php +++ b/tests/Database/DatabaseQueryBuilderTest.php @@ -2048,12 +2048,12 @@ public function testWhereFulltextPostgres(): void $builder = $this->getPostgresBuilderWithProcessor(); $builder->select('*')->from('users')->whereFullText(['body', 'title'], 'Car Plane'); - $this->assertSame('select * from "users" where (to_tsvector(\'english\', "body") || to_tsvector(\'english\', "title")) @@ plainto_tsquery(\'english\', ?)', $builder->toSql()); + $this->assertSame('select * from "users" where (coalesce(to_tsvector(\'english\', "body"), \'\'::tsvector) || coalesce(to_tsvector(\'english\', "title"), \'\'::tsvector)) @@ plainto_tsquery(\'english\', ?)', $builder->toSql()); $this->assertEquals(['Car Plane'], $builder->getBindings()); $builder = $this->getPostgresBuilderWithProcessor(); $builder->select('*')->from('users')->whereFullText(['body', 'title'], 'Air | Plan:* -Car', ['mode' => 'raw']); - $this->assertSame('select * from "users" where (to_tsvector(\'english\', "body") || to_tsvector(\'english\', "title")) @@ to_tsquery(\'english\', ?)', $builder->toSql()); + $this->assertSame('select * from "users" where (coalesce(to_tsvector(\'english\', "body"), \'\'::tsvector) || coalesce(to_tsvector(\'english\', "title"), \'\'::tsvector)) @@ to_tsquery(\'english\', ?)', $builder->toSql()); $this->assertEquals(['Air | Plan:* -Car'], $builder->getBindings()); $builder = $this->getPostgresBuilderWithProcessor(); @@ -2073,7 +2073,7 @@ public function testWhereFulltextPostgres(): void $builder = $this->getPostgresBuilderWithProcessor(); $builder->select('*')->from('users')->whereFullText(['tsv_title', 'tsv_body'], 'Car Plane', ['vector' => true]); - $this->assertSame('select * from "users" where ("tsv_title" || "tsv_body") @@ plainto_tsquery(\'english\', ?)', $builder->toSql()); + $this->assertSame('select * from "users" where (coalesce("tsv_title", \'\'::tsvector) || coalesce("tsv_body", \'\'::tsvector)) @@ plainto_tsquery(\'english\', ?)', $builder->toSql()); $this->assertSame(['Car Plane'], $builder->getBindings()); } diff --git a/tests/Integration/Database/Postgres/FulltextTest.php b/tests/Integration/Database/Postgres/FulltextTest.php index b11d1b9548..b8082f6b24 100644 --- a/tests/Integration/Database/Postgres/FulltextTest.php +++ b/tests/Integration/Database/Postgres/FulltextTest.php @@ -20,7 +20,7 @@ protected function afterRefreshingDatabase(): void Schema::create('articles', function (Blueprint $table) { $table->id('id'); $table->string('title', 200); - $table->text('body'); + $table->text('body')->nullable(); $table->fulltext(['title', 'body']); }); @@ -79,4 +79,26 @@ public function testWhereFulltextWithRaw(): void $this->assertSame(['PostgreSQL Tutorial', 'Optimizing PostgreSQL'], $articles->pluck('title')->all()); } + + public function testWhereFulltextMatchesARowWithANullColumn(): void + { + DB::table('articles')->insert(['title' => 'Database Primer', 'body' => null]); + + $articles = DB::table('articles')->whereFullText(['title', 'body'], 'primer')->get(); + + $this->assertSame(['Database Primer'], $articles->pluck('title')->all()); + } + + public function testWhereFulltextUsesTheMultiColumnFulltextIndex(): void + { + $query = DB::table('articles')->whereFullText(['title', 'body'], 'database'); + + $plan = DB::transaction(function () use ($query): string { + DB::statement('set local enable_seqscan = off'); + + return collect(DB::select('explain ' . $query->toSql(), $query->getBindings()))->pluck('QUERY PLAN')->implode("\n"); + }); + + $this->assertStringContainsString('articles_title_body_fulltext', $plan); + } } From b099967a0cb4191539880bfe58b410033ba27251 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:39:52 +0000 Subject: [PATCH 05/30] Add native PostgreSQL full-text search to the Scout database engine Ports laravel/scout#1016 from laravel/scout at bb583fec, with its documentation from laravel/docs 4cfebfe805. On PostgreSQL, the database engine can now search stored tsvector columns, choose the tsquery mode (including raw tsquery) and language, rank with ts_rank or ts_rank_cd, and use pg_trgm trigram matching as a fallback when full-text search finds nothing or blended into the results. Options come from the SearchUsingFullText attribute and can be overridden per query through the builder's options. The searchable() and dropSearchable() Blueprint macros add or drop a generated, weighted tsvector column with its GIN index and optional trigram indexes. The hybrid vector search applies the same trigram fallback to its text query. Adaptations: the language is bound as a regconfig parameter; the pg_trgm check is cached on the engine, which the manager keeps for the worker lifetime, instead of in a static property; the attribute options are cached per model class and the default search path returns them without merging; Hypervel's qualified key ordering is kept, with the key as a tiebreaker after relevance. Upstream's tsvector type macro and PostgreSQL guard are not ported, because Hypervel's schema grammars compile tsvector columns natively and reject them on other drivers. Upstream defects fixed: simple pagination no longer falls back to trigram matching on an empty later page while the full-text search has matches, so every page belongs to the same result set; the check reads the first page, since pagination replaces any offset and limit on the query. Multi-column expression searches coalesce each column's vector, matching the framework's multi-column full-text index, so a null column doesn't hide a row. Tests: ported SearchableSchemaTest, the full-text options tests and the PostgreSQL search tests, and added tests for pagination past the last full-text match, including a query with an offset, and for a row with a null column. Validation: the Scout suite passes, and the PostgreSQL Scout tests pass on PostgreSQL 17; formatting and full static analysis pass. --- src/docs/scout.md | 91 ++++ src/docs/search.md | 4 +- .../src/Attributes/SearchUsingFullText.php | 11 +- .../Concerns/PerformsDatabaseVectorSearch.php | 40 +- src/scout/src/Engines/DatabaseEngine.php | 362 ++++++++++++---- src/scout/src/ScoutServiceProvider.php | 2 + src/scout/src/SearchableSchema.php | 106 +++++ .../Postgres/DatabasePgsqlSearchTest.php | 404 ++++++++++++++++++ .../DatabaseEngineFullTextOptionsTest.php | 73 ++++ tests/Scout/Unit/SearchableSchemaTest.php | 37 ++ 10 files changed, 1031 insertions(+), 99 deletions(-) create mode 100644 src/scout/src/SearchableSchema.php create mode 100644 tests/Integration/Scout/Database/Postgres/DatabasePgsqlSearchTest.php create mode 100644 tests/Scout/Feature/DatabaseEngineFullTextOptionsTest.php create mode 100644 tests/Scout/Unit/SearchableSchemaTest.php diff --git a/src/docs/scout.md b/src/docs/scout.md index 62e78db223..d67ed00580 100644 --- a/src/docs/scout.md +++ b/src/docs/scout.md @@ -436,6 +436,97 @@ When using PostgreSQL, only include text-compatible columns in the default, pref > [!WARNING] > Before specifying that a column should use full text query constraints, ensure that the column has been assigned a [full text index](/docs/{{version}}/migrations#available-index-types). + +#### PostgreSQL Full-Text Search + +When using PostgreSQL, Scout can search stored `tsvector` columns for indexed, weighted full-text search. To get started, use the `searchable` method in a migration to add a generated `search_vector` column and its GIN index: + +```php +use Hypervel\Database\Schema\Blueprint; +use Hypervel\Support\Facades\Schema; + +Schema::table('posts', function (Blueprint $table) { + $table->searchable(['title' => 'A', 'body' => 'D']); +}); +``` + +Column weights range from `A` (highest) to `D` (lowest). You may also pass a list of column names without weights, such as `['title', 'body']`, to assign each column the default weight of `D`. PostgreSQL automatically updates the generated column when the source columns change. + +Next, apply the `SearchUsingFullText` attribute to your model's `toSearchableArray` method, setting the `vector` option to `true`: + +```php +use Hypervel\Scout\Attributes\SearchUsingFullText; + +/** + * Get the indexable data array for the model. + * + * @return array + */ +#[SearchUsingFullText(['search_vector'], ['vector' => true])] +public function toSearchableArray(): array +{ + return ['search_vector' => null]; +} +``` + +Only the column name is needed; Scout searches the stored value directly. Return only the vector column to avoid adding `LIKE` clauses for other attributes, which can prevent PostgreSQL from using the GIN index. + +The `searchable` method also accepts a text search configuration, columns for trigram indexes, and a custom vector column name: + +```php +$table->searchable( + columns: ['title' => 'A', 'body' => 'D'], + language: 'english', + trigram: ['title'], + tsvector: 'search_vector', +); +``` + +To remove the generated column and its indexes when rolling back a migration, use the `dropSearchable` method, passing the same trigram columns and vector column name: + +```php +Schema::table('posts', function (Blueprint $table) { + $table->dropSearchable(trigram: ['title'], tsvector: 'search_vector'); +}); +``` + + +#### PostgreSQL Search Options + +You may customize PostgreSQL full-text search using the second argument to the `SearchUsingFullText` attribute: + +| Option | Description | Default | +| --- | --- | --- | +| `vector` | Whether the full-text columns are stored `tsvector` columns. | `false` | +| `language` | The text search configuration, such as `english` or `public.english_unaccent`. | `english` | +| `mode` | The query syntax: `plain`, `phrase`, `websearch`, or `raw`. | `plain` | +| `rank` | The ranking function: `ts_rank` or `ts_rank_cd`. | `ts_rank` | +| `trigram` | A column name or array of columns for typo-tolerant matching. | None | +| `trigram_mode` | Whether trigram matching is used as a `fallback` or blended with full-text matches using `blend`. | `fallback` | + +The `plain` mode searches for all query terms, while `phrase` matches terms in order. The `websearch` mode supports web search syntax, such as quoted phrases and excluded terms, and `raw` accepts PostgreSQL `tsquery` syntax. When searching a stored vector column, the `language` must match the text search configuration used to generate the column. + +To enable typo tolerance, specify trigram columns in both the migration and the attribute options: + +```php +#[SearchUsingFullText(['search_vector'], ['vector' => true, 'trigram' => ['title']])] +public function toSearchableArray(): array +{ + return ['search_vector' => null]; +} +``` + +By default, Scout retries searches using trigram matching only when the initial search returns no results. Set `trigram_mode` to `blend` to include trigram matches alongside full-text matches and incorporate trigram similarity into relevance ranking. + +> [!WARNING] +> Trigram matching requires the `pg_trgm` extension to be available on the connection's search path. When trigram columns are provided, the `searchable` migration method creates the extension, so the database user running migrations must have permission to create it. + +You may override the `mode`, `language`, `rank`, `trigram`, and `trigram_mode` options for an individual search using the `options` method: + +```php +$posts = Post::search('"hypervel scout"')->options(['mode' => 'websearch'])->get(); +``` + ### Collection Engine diff --git a/src/docs/search.md b/src/docs/search.md index 12015cd1a7..0920d92252 100644 --- a/src/docs/search.md +++ b/src/docs/search.md @@ -20,7 +20,7 @@ Almost every application needs search. Whether your users are searching a knowledge base for relevant articles, exploring a product catalog, or asking natural-language questions against a corpus of documents, Hypervel provides built-in tools to handle each of these scenarios — and you often don't need any external services to get there. -Most applications will find that the built-in database-powered options provided by Hypervel are more than sufficient — external search services are only necessary when you need features like typo tolerance, faceted filtering, or geo-search at massive scale. +Most applications will find that the built-in database-powered options provided by Hypervel are more than sufficient — external search services are only necessary when you need features like faceted filtering or geo-search at massive scale. #### Full-Text Search @@ -200,6 +200,8 @@ $articles = Article::search('Hypervel')->get(); The database engine is a great choice when your search needs are moderate and you want the convenience of Scout's automatic index syncing without deploying an external service. It handles the most common search use cases well, including filtering, pagination, and soft-deleted record handling. For complete details, consult the [Scout documentation](/docs/{{version}}/scout#database-engine). +On PostgreSQL, Scout also supports [weighted full-text search and typo tolerance](/docs/{{version}}/scout#postgresql-full-text-search) using stored `tsvector` columns and the `pg_trgm` extension. + ### Third-Party Engines diff --git a/src/scout/src/Attributes/SearchUsingFullText.php b/src/scout/src/Attributes/SearchUsingFullText.php index 23ddd4e6d4..e5462c8129 100644 --- a/src/scout/src/Attributes/SearchUsingFullText.php +++ b/src/scout/src/Attributes/SearchUsingFullText.php @@ -42,7 +42,16 @@ class SearchUsingFullText * Create a new attribute instance. * * @param array|string $columns - * @param array $options + * @param array{ + * mode?: 'boolean'|'natural'|'phrase'|'plain'|'raw'|'websearch', + * language?: string, + * vector?: bool, + * trigram?: array|string, + * trigram_mode?: 'blend'|'fallback', + * rank?: 'ts_rank'|'ts_rank_cd', + * expanded?: bool, + * ... + * } $options */ public function __construct(array|string $columns, array $options = []) { diff --git a/src/scout/src/Engines/Concerns/PerformsDatabaseVectorSearch.php b/src/scout/src/Engines/Concerns/PerformsDatabaseVectorSearch.php index 6abc885506..265b03a082 100644 --- a/src/scout/src/Engines/Concerns/PerformsDatabaseVectorSearch.php +++ b/src/scout/src/Engines/Concerns/PerformsDatabaseVectorSearch.php @@ -100,24 +100,26 @@ protected function hybridSearchModels(Builder $builder, int $limit): EloquentCol $baseQuery = $this->newSearchQuery($builder); - // Build the text query... - /** @var EloquentBuilder $textQuery */ - $textQuery = $this->constrainSearchQuery($builder, $this->addTextSearchConstraints( - clone $baseQuery, - $builder, - array_keys($builder->model->toSearchableArray()), - $this->getPrefixColumns($builder), - $this->getFullTextColumns($builder) - )->take(1000)); - - // Order the text query... - if (! $this->getFullTextColumns($builder)) { - $textQuery->orderBy($builder->model->getQualifiedKeyName(), 'desc'); - } elseif ($this->shouldOrderByRelevance($builder)) { - $this->orderByRelevance($builder, $textQuery); - - $textQuery->orderBy($builder->model->getQualifiedKeyName()); - } + $textResults = $this->searchWithTrigramFallback($builder, function (Builder $builder) use ($baseQuery): EloquentCollection { + /** @var EloquentBuilder $textQuery */ + $textQuery = $this->constrainSearchQuery($builder, $this->addTextSearchConstraints( + clone $baseQuery, + $builder, + array_keys($builder->model->toSearchableArray()), + $this->getPrefixColumns($builder), + $this->getFullTextColumns($builder) + )->take(1000)); + + if (! $this->getFullTextColumns($builder)) { + $textQuery->orderBy($builder->model->getQualifiedKeyName(), 'desc'); + } elseif ($this->shouldOrderByRelevance($builder)) { + $this->orderByRelevance($builder, $textQuery); + + $textQuery->orderBy($builder->model->getQualifiedKeyName()); + } + + return $textQuery->get(); + }, fn (EloquentCollection $models): bool => $models->isEmpty()); // Build the semantic query... /** @var EloquentBuilder $semanticQuery */ @@ -129,7 +131,7 @@ protected function hybridSearchModels(Builder $builder, int $limit): EloquentCol return $this->fuseSearchResults( $builder, - $textQuery->get(), + $textResults, $semanticQuery->get() )->take($limit)->values(); } diff --git a/src/scout/src/Engines/DatabaseEngine.php b/src/scout/src/Engines/DatabaseEngine.php index e67c8d430b..ca385994de 100644 --- a/src/scout/src/Engines/DatabaseEngine.php +++ b/src/scout/src/Engines/DatabaseEngine.php @@ -4,6 +4,7 @@ namespace Hypervel\Scout\Engines; +use Closure; use Hypervel\Container\Container; use Hypervel\Contracts\Pagination\LengthAwarePaginator as LengthAwarePaginatorContract; use Hypervel\Contracts\Pagination\Paginator as PaginatorContract; @@ -20,9 +21,13 @@ use Hypervel\Scout\Contracts\SupportsSemanticSearch; use Hypervel\Scout\Engines\Concerns\PaginatesDatabaseVectorSearch; use Hypervel\Scout\Engines\Concerns\PerformsDatabaseVectorSearch; +use Hypervel\Scout\SearchableSchema; +use Hypervel\Support\Arr; use Hypervel\Support\Collection; use Hypervel\Support\LazyCollection; +use InvalidArgumentException; use ReflectionMethod; +use RuntimeException; use function blank; @@ -36,6 +41,15 @@ class DatabaseEngine extends Engine implements PaginatesEloquentModelsUsingDatab use PaginatesDatabaseVectorSearch; use PerformsDatabaseVectorSearch; + /** + * The connections known to have the pg_trgm extension, keyed by connection name, database, and search path. + * + * Unlike upstream, this is held on the engine, which the manager keeps for the worker lifetime, instead of in a static property. + * + * @var array + */ + protected array $trigramExtensionInstalled = []; + /** * Cached searchable attribute columns by model class and attribute class. * @@ -44,7 +58,7 @@ class DatabaseEngine extends Engine implements PaginatesEloquentModelsUsingDatab protected array $attributeColumns = []; /** - * Cached full-text options by model class. + * Cached full-text attribute options by model class. * * @var array, array> */ @@ -99,19 +113,12 @@ public function paginateUsingDatabase( return $this->paginateSemanticSearch($builder, $perPage, $pageName, $page); } - return $this->buildSearchQuery($builder) - ->when(count($builder->orders) > 0, function (EloquentBuilder $query) use ($builder): void { - foreach ($builder->orders as $order) { - $query->orderBy($order['column'], $order['direction']); - } - }) - ->when(count($this->getFullTextColumns($builder)) === 0, function (EloquentBuilder $query) use ($builder): void { - $query->orderBy($builder->model->getQualifiedKeyName(), 'desc'); - }) - ->when($this->shouldOrderByRelevance($builder), function (EloquentBuilder $query) use ($builder): void { - $this->orderByRelevance($builder, $query); - }) - ->paginate($perPage, ['*'], $pageName, $page); + return $this->searchWithTrigramFallback( + $builder, + fn (Builder $builder): LengthAwarePaginatorContract => $this->orderSearchQuery($builder, $this->buildSearchQuery($builder)) + ->paginate($perPage, ['*'], $pageName, $page), + fn (LengthAwarePaginatorContract $paginator): bool => $paginator->total() === 0, + ); } /** @@ -139,19 +146,16 @@ public function simplePaginateUsingDatabase( return $this->simplePaginateSemanticSearch($builder, $perPage, $pageName, $page); } - return $this->buildSearchQuery($builder) - ->when(count($builder->orders) > 0, function (EloquentBuilder $query) use ($builder): void { - foreach ($builder->orders as $order) { - $query->orderBy($order['column'], $order['direction']); - } - }) - ->when(count($this->getFullTextColumns($builder)) === 0, function (EloquentBuilder $query) use ($builder): void { - $query->orderBy($builder->model->getQualifiedKeyName(), 'desc'); - }) - ->when($this->shouldOrderByRelevance($builder), function (EloquentBuilder $query) use ($builder): void { - $this->orderByRelevance($builder, $query); - }) - ->simplePaginate($perPage, ['*'], $pageName, $page); + return $this->searchWithTrigramFallback( + $builder, + fn (Builder $builder): PaginatorContract => $this->orderSearchQuery($builder, $this->buildSearchQuery($builder)) + ->simplePaginate($perPage, ['*'], $pageName, $page), + // Unlike upstream, an empty later page falls back only when the search matched nothing at all, so every page + // belongs to the same result set; otherwise a page past the last full-text match would return trigram matches. + // The probe reads the first page, since pagination replaces any offset and limit on the query. + fn (PaginatorContract $paginator): bool => $paginator->isEmpty() + && ($paginator->currentPage() === 1 || ! $this->buildSearchQuery($builder)->forPage(1, 1)->exists()), + ); } /** @@ -171,22 +175,15 @@ protected function searchModels(Builder $builder, ?int $page = null, ?int $perPa } /** @var EloquentCollection */ - return $this->buildSearchQuery($builder) - ->when($page !== null && $perPage !== null, function (EloquentBuilder $query) use ($page, $perPage): void { - $query->forPage($page, $perPage); - }) - ->when(count($builder->orders) > 0, function (EloquentBuilder $query) use ($builder): void { - foreach ($builder->orders as $order) { - $query->orderBy($order['column'], $order['direction']); - } - }) - ->when(count($this->getFullTextColumns($builder)) === 0, function (EloquentBuilder $query) use ($builder): void { - $query->orderBy($builder->model->getQualifiedKeyName(), 'desc'); - }) - ->when($this->shouldOrderByRelevance($builder), function (EloquentBuilder $query) use ($builder): void { - $this->orderByRelevance($builder, $query); - }) - ->get(); + return $this->searchWithTrigramFallback( + $builder, + fn (Builder $builder): EloquentCollection => $this->orderSearchQuery($builder, $this->buildSearchQuery($builder)) + ->when($page !== null && $perPage !== null, function (EloquentBuilder $query) use ($page, $perPage): void { + $query->forPage($page, $perPage); + }) + ->get(), + fn (EloquentCollection $models): bool => $models->isEmpty(), + ); } /** @@ -306,18 +303,202 @@ protected function addTextSearchConstraints( ); } - if (count($fullTextColumns) > 0) { - $qualifiedColumns = array_map( - fn (string $column): string => $builder->model->qualifyColumn($column), - $fullTextColumns - ); - + if ($this->searchesStoredVectors($builder)) { + foreach ($fullTextColumns as $column) { + $query->orWhereRaw( + "{$this->wrapColumn($builder, $column)} @@ {$this->tsQueryExpression($builder)}", + $this->tsQueryBindings($builder) + ); + } + } elseif (count($fullTextColumns) > 0 && $connectionType === 'pgsql') { + // Unlike upstream, each vector of a multi-column search is coalesced, so a null column doesn't hide the + // row; this matches the multi-column full-text index expression the schema grammar creates. + $vectors = collect($fullTextColumns) + ->map(fn (string $column): string => count($fullTextColumns) > 1 + ? "coalesce({$this->tsVectorExpression($builder, $column)}, ''::tsvector)" + : $this->tsVectorExpression($builder, $column)) + ->implode(' || '); + + $query->orWhereRaw("({$vectors}) @@ {$this->tsQueryExpression($builder)}", $this->tsQueryBindings($builder)); + } elseif (count($fullTextColumns) > 0) { $query->orWhereFullText( - $qualifiedColumns, + array_map(fn (string $column): string => $builder->model->qualifyColumn($column), $fullTextColumns), $builder->query, $this->getFullTextOptions($builder) ); } + + foreach ($this->getTrigramColumns($builder) as $column) { + $query->orWhereRaw("{$this->wrapColumn($builder, $column)} % ?", [$builder->query]); + } + }); + } + + /** + * Run the given search, retrying with trigram matching when it finds nothing. + * + * @template TResult + * + * @param Closure(Builder): TResult $search + * @param Closure(TResult): bool $isEmpty + * @return TResult + */ + protected function searchWithTrigramFallback(Builder $builder, Closure $search, Closure $isEmpty): mixed + { + if ($builder->modelConnectionType() === 'pgsql') { + $this->ensureValidFullTextOptions($this->getFullTextOptions($builder)); + } + + $fallback = $this->shouldFallbackToTrigram($builder); + + $results = $search($builder); + + if (! $fallback || ! $isEmpty($results)) { + return $results; + } + + return $search((clone $builder)->options(array_merge($builder->options, ['trigram_mode' => 'blend']))); + } + + /** + * Determine if an empty search should be retried using trigram matching. + */ + protected function shouldFallbackToTrigram(Builder $builder): bool + { + $options = $this->getFullTextOptions($builder); + + return ($options['trigram_mode'] ?? 'fallback') === 'fallback' + && $this->usesTrigram($builder); + } + + /** + * Get the columns that should be matched using trigram similarity for the current search. + * + * @return array + */ + protected function getTrigramColumns(Builder $builder): array + { + $options = $this->getFullTextOptions($builder); + + if (($options['trigram_mode'] ?? 'fallback') !== 'blend') { + return []; + } + + return $this->usesTrigram($builder) ? Arr::wrap($options['trigram']) : []; + } + + /** + * Determine if the given search should use trigram matching. + */ + protected function usesTrigram(Builder $builder): bool + { + if (blank($builder->query) || empty($this->getFullTextOptions($builder)['trigram']) || $builder->modelConnectionType() !== 'pgsql') { + return false; + } + + $this->ensureTrigramExtensionIsInstalled($builder); + + return true; + } + + /** + * Ensure the pg_trgm functions are reachable on the model's connection search path. + * + * @throws RuntimeException + */ + protected function ensureTrigramExtensionIsInstalled(Builder $builder): void + { + $connection = $builder->model->getConnection(); + + $key = implode('|', [ + $connection->getName(), + $connection->getDatabaseName(), + json_encode($connection->getConfig('search_path') ?? $connection->getConfig('schema'), JSON_THROW_ON_ERROR), + ]); + + if (isset($this->trigramExtensionInstalled[$key])) { + return; + } + + if (! $connection->selectOne("select to_regprocedure('similarity(text,text)') is not null as installed")->installed) { + throw new RuntimeException("Scout trigram search requires the pg_trgm extension on the [{$connection->getName()}] connection's search path."); + } + + $this->trigramExtensionInstalled[$key] = true; + } + + /** + * Determine if the full-text columns are stored tsvector columns (Postgres only). + */ + protected function searchesStoredVectors(Builder $builder): bool + { + return ! empty($this->getFullTextOptions($builder)['vector']) && $builder->modelConnectionType() === 'pgsql'; + } + + /** + * Get the tsvector SQL expression for the given full-text column. + */ + protected function tsVectorExpression(Builder $builder, string $column): string + { + if ($this->searchesStoredVectors($builder)) { + return $this->wrapColumn($builder, $column); + } + + $language = $this->getFullTextOptions($builder)['language'] ?? 'english'; + + SearchableSchema::ensureValidLanguage($language); + + return "to_tsvector('{$language}', {$this->wrapColumn($builder, $column)})"; + } + + /** + * Get the tsquery SQL expression for the search. + */ + protected function tsQueryExpression(Builder $builder): string + { + $function = match ($this->getFullTextOptions($builder)['mode'] ?? null) { + 'phrase' => 'phraseto_tsquery', + 'websearch' => 'websearch_to_tsquery', + 'raw' => 'to_tsquery', + default => 'plainto_tsquery', + }; + + return "{$function}(?::regconfig, ?)"; + } + + /** + * Get the bindings for the tsquery SQL expression. + * + * @return array{string, string} + */ + protected function tsQueryBindings(Builder $builder): array + { + return [$this->getFullTextOptions($builder)['language'] ?? 'english', $builder->query]; + } + + /** + * Qualify and wrap the given column for use in raw SQL. + */ + protected function wrapColumn(Builder $builder, string $column): string + { + return $builder->model->getConnection()->getQueryGrammar()->wrap($builder->model->qualifyColumn($column)); + } + + /** + * Add the ordering clauses to the search query. + */ + protected function orderSearchQuery(Builder $builder, EloquentBuilder $query): EloquentBuilder + { + return $query->when(count($builder->orders) > 0, function (EloquentBuilder $query) use ($builder): void { + foreach ($builder->orders as $order) { + $query->orderBy($order['column'], $order['direction']); + } + })->when(count($this->getFullTextColumns($builder)) === 0, function (EloquentBuilder $query) use ($builder): void { + $query->orderBy($builder->model->getQualifiedKeyName(), 'desc'); + })->when($this->shouldOrderByRelevance($builder), function (EloquentBuilder $query) use ($builder): void { + $this->orderByRelevance($builder, $query); + + $query->orderBy($builder->model->getQualifiedKeyName(), 'desc'); }); } @@ -338,30 +519,33 @@ protected function shouldOrderByRelevance(Builder $builder): bool */ protected function orderByRelevance(Builder $builder, EloquentBuilder $query): EloquentBuilder { - $fullTextColumns = $this->getFullTextColumns($builder); $options = $this->getFullTextOptions($builder); - $language = $options['language'] ?? 'english'; - - $vectors = collect($fullTextColumns) - ->map(fn (string $column): string => sprintf( - "to_tsvector('%s', %s)", - $language, - $builder->model->qualifyColumn($column) - )) - ->implode(' || '); - $tsQueryFunction = match ($options['mode'] ?? 'plainto_tsquery') { - 'phrase' => 'phraseto_tsquery', - 'websearch' => 'websearch_to_tsquery', - default => 'plainto_tsquery', + $rank = match ($options['rank'] ?? null) { + 'ts_rank_cd' => 'ts_rank_cd', + default => 'ts_rank', }; - $query->orderByRaw( - sprintf('ts_rank(%s, %s(?)) desc', $vectors, $tsQueryFunction), - [$builder->query] - ); + $vectors = collect($this->getFullTextColumns($builder)) + ->map(fn (string $column): string => "coalesce({$this->tsVectorExpression($builder, $column)}, ''::tsvector)") + ->implode(' || '); - return $query; + $score = "{$rank}({$vectors}, {$this->tsQueryExpression($builder)})"; + $bindings = $this->tsQueryBindings($builder); + + $trigramColumns = $this->getTrigramColumns($builder); + + if ($trigramColumns === []) { + return $query->orderByRaw("{$score} desc", $bindings); + } + + $similarity = collect($trigramColumns)->map(fn (string $column): string => "similarity({$this->wrapColumn($builder, $column)}, ?)")->implode(', '); + + $score = "{$score} + (coalesce(greatest({$similarity}), 0) * 0.25)"; + + $bindings = array_merge($bindings, array_fill(0, count($trigramColumns), $builder->query)); + + return $query->orderByRaw("{$score} desc", $bindings); } /** @@ -493,21 +677,43 @@ protected function getFullTextOptions(Builder $builder): array { $modelClass = $builder->model::class; - if (array_key_exists($modelClass, $this->fullTextOptions)) { - return $this->fullTextOptions[$modelClass]; + if (! array_key_exists($modelClass, $this->fullTextOptions)) { + $options = []; + + $reflection = new ReflectionMethod($builder->model, 'toSearchableArray'); + + foreach ($reflection->getAttributes(SearchUsingFullText::class) as $attribute) { + /** @var SearchUsingFullText $attributeInstance */ + $attributeInstance = $attribute->newInstance(); + $options = array_merge($options, $attributeInstance->options); + } + + $this->fullTextOptions[$modelClass] = $options; } - $options = []; + if ($builder->options === []) { + return $this->fullTextOptions[$modelClass]; + } - $reflection = new ReflectionMethod($builder->model, 'toSearchableArray'); + return array_merge($this->fullTextOptions[$modelClass], Arr::only($builder->options, ['mode', 'language', 'trigram', 'trigram_mode', 'rank', 'expanded'])); + } - foreach ($reflection->getAttributes(SearchUsingFullText::class) as $attribute) { - /** @var SearchUsingFullText $attributeInstance */ - $attributeInstance = $attribute->newInstance(); - $options = array_merge($options, $attributeInstance->options); + /** + * Ensure the given PostgreSQL full-text options are supported. + * + * @param array $options + * + * @throws InvalidArgumentException + */ + protected function ensureValidFullTextOptions(array $options): void + { + if (isset($options['rank']) && ! in_array($options['rank'], ['ts_rank', 'ts_rank_cd'], true)) { + throw new InvalidArgumentException('Invalid full-text search option: [rank] must be one of [ts_rank, ts_rank_cd].'); } - return $this->fullTextOptions[$modelClass] = $options; + if (isset($options['trigram_mode']) && ! in_array($options['trigram_mode'], ['fallback', 'blend'], true)) { + throw new InvalidArgumentException('Invalid full-text search option: [trigram_mode] must be one of [fallback, blend].'); + } } /** diff --git a/src/scout/src/ScoutServiceProvider.php b/src/scout/src/ScoutServiceProvider.php index ffaa080923..c9724f5a0f 100644 --- a/src/scout/src/ScoutServiceProvider.php +++ b/src/scout/src/ScoutServiceProvider.php @@ -52,6 +52,8 @@ public function register(): void */ public function boot(): void { + SearchableSchema::register(); + $this->configureAlgoliaSdk(); if ($this->app->runningInConsole()) { diff --git a/src/scout/src/SearchableSchema.php b/src/scout/src/SearchableSchema.php new file mode 100644 index 0000000000..63b72ce55a --- /dev/null +++ b/src/scout/src/SearchableSchema.php @@ -0,0 +1,106 @@ + 'create extension if not exists pg_trgm'); + + Blueprint::macro('searchable', function (array $columns, string $language = 'english', array $trigram = [], string $tsvector = 'search_vector'): void { + $this->addColumn('tsvector', $tsvector)->storedAs( + SearchableSchema::vectorExpression($columns, $language) + ); + + $this->index($tsvector, null, 'gin'); + + if ($trigram === []) { + return; + } + + $this->addCommand('scoutTrigramExtension'); + + foreach ($trigram as $column) { + SearchableSchema::ensureValidIdentifier($column); + + $this->rawIndex("\"{$column}\" gin_trgm_ops", $this->createIndexName('trigram', [$column]))->algorithm('gin'); + } + }); + + Blueprint::macro('dropSearchable', function (array $trigram = [], string $tsvector = 'search_vector'): void { + $table = $this->getTable(); + $schema = str_contains($table, '.') ? Str::beforeLast($table, '.') . '.' : ''; + + foreach ($trigram as $column) { + $this->dropIndex($schema . $this->createIndexName('trigram', [$column])); + } + + $this->dropColumn($tsvector); + }); + } + + /** + * Build the weighted tsvector expression for the given columns. + * + * @internal + * + * @param array $columns column names, optionally keyed by column with an A-D weight value + */ + public static function vectorExpression(array $columns, string $language): string + { + if ($columns === []) { + throw new InvalidArgumentException('At least one searchable column is required.'); + } + + static::ensureValidLanguage($language); + + return collect($columns)->map(function (string $weight, int|string $column) use ($language): string { + [$column, $weight] = is_int($column) ? [$weight, 'D'] : [$column, strtoupper($weight)]; + + static::ensureValidIdentifier($column); + + if (! in_array($weight, ['A', 'B', 'C', 'D'], true)) { + throw new InvalidArgumentException("Invalid full-text weight [{$weight}] for column [{$column}]. Use A, B, C, or D."); + } + + return "setweight(to_tsvector('{$language}', coalesce(\"{$column}\"::text, '')), '{$weight}')"; + })->implode(' || '); + } + + /** + * Ensure the given text search config name is a plain, optionally schema-qualified, identifier. + * + * @internal + */ + public static function ensureValidLanguage(string $language): void + { + if (! preg_match('/^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?$/', $language)) { + throw new InvalidArgumentException("Invalid full-text search language [{$language}]."); + } + } + + /** + * Ensure the given column name is a plain identifier. + * + * @internal + */ + public static function ensureValidIdentifier(string $column): void + { + if (! preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $column)) { + throw new InvalidArgumentException("Invalid searchable column [{$column}]."); + } + } +} diff --git a/tests/Integration/Scout/Database/Postgres/DatabasePgsqlSearchTest.php b/tests/Integration/Scout/Database/Postgres/DatabasePgsqlSearchTest.php new file mode 100644 index 0000000000..5250f0fa8b --- /dev/null +++ b/tests/Integration/Scout/Database/Postgres/DatabasePgsqlSearchTest.php @@ -0,0 +1,404 @@ +make('config'); + $connection = $config->array('database.connections.' . $config->string('database.default')); + + $config->set('scout.driver', 'database'); + $config->set('database.connections.pgsql_prefixed', [...$connection, 'prefix' => 'pre_', 'prefix_indexes' => true]); + $config->set('database.connections.pgsql_tenant', [...$connection, 'search_path' => 'scout_tenant']); + } + + /** + * Commit the extension before the test transaction begins, so another connection's + * "create extension if not exists" doesn't wait for that transaction. + */ + protected function beforeRefreshingDatabase(): void + { + DB::statement('create extension if not exists pg_trgm'); + } + + /** + * Create the searchable table inside the test transaction. + */ + protected function afterRefreshingDatabase(): void + { + Schema::create('scout_posts', function (Blueprint $table): void { + $table->id(); + $table->string('title'); + $table->text('body')->nullable(); + $table->timestamps(); + $table->softDeletes(); + $table->searchable(['title' => 'A', 'body' => 'D'], 'english', ['title']); + }); + } + + /** + * Drop the table the prefixed connection created outside the test transaction. + */ + protected function tearDownInCoroutine(): void + { + Schema::connection('pgsql_prefixed')->dropIfExists('scout_posts'); + } + + public function testSearchableCreatesGeneratedVectorAndGinIndexes(): void + { + $this->assertSame( + ['scout_posts_search_vector_index' => 'CREATE INDEX scout_posts_search_vector_index ON public.scout_posts USING gin (search_vector)', + 'scout_posts_title_trigram' => 'CREATE INDEX scout_posts_title_trigram ON public.scout_posts USING gin (title gin_trgm_ops)'], + $this->indexes() + ); + + Schema::table('scout_posts', fn (Blueprint $table) => $table->dropSearchable(['title'])); + + $this->assertSame([], $this->indexes()); + $this->assertFalse(Schema::hasColumn('scout_posts', 'search_vector')); + } + + public function testSearchableCanBeAddedToAnExistingTable(): void + { + Schema::table('scout_posts', fn (Blueprint $table) => $table->dropSearchable(['title'])); + PgsqlPost::create(['title' => 'Laravel Scout']); + + Schema::table('scout_posts', fn (Blueprint $table) => $table->searchable(['title', 'body'])); + + $this->assertSame(['scout_posts_search_vector_index'], array_keys($this->indexes())); + $this->assertSame(['Laravel Scout'], PgsqlPost::search('scout')->get()->pluck('title')->all()); + } + + public function testSearchableCreatesTheTrigramExtension(): void + { + DB::statement('drop extension pg_trgm cascade'); + Schema::table('scout_posts', fn (Blueprint $table) => $table->dropSearchable()); + + try { + Schema::table('scout_posts', fn (Blueprint $table) => $table->searchable(['title'], 'english', ['title'])); + + $this->assertTrue(DB::selectOne("select exists(select 1 from pg_extension where extname = 'pg_trgm') as installed")->installed); + } finally { + DB::statement('create extension if not exists pg_trgm'); + } + } + + public function testWeightedVectorSearchRanksTitleMatchesFirstAndMatchesNullColumns(): void + { + PgsqlPost::create(['title' => 'Queues', 'body' => 'Scout keeps search indexes in sync']); + PgsqlPost::create(['title' => 'Laravel Scout', 'body' => null]); + PgsqlPost::create(['title' => 'Horizon', 'body' => 'Queue dashboard']); + + $this->assertSame(['Laravel Scout', 'Queues'], PgsqlPost::search('scout')->get()->pluck('title')->all()); + } + + public function testQueryModes(): void + { + PgsqlPost::create(['title' => 'Shah Rukh Khan', 'body' => 'Romance']); + PgsqlPost::create(['title' => 'Salman Khan', 'body' => 'Action']); + + $this->assertSame(['Shah Rukh Khan'], PgsqlWebsearchPost::search('khan -salman')->get()->pluck('title')->all()); + $this->assertSame(['Shah Rukh Khan'], PgsqlPhrasePost::search('rukh khan')->get()->pluck('title')->all()); + $this->assertSame([], PgsqlPhrasePost::search('khan rukh')->get()->all()); + } + + public function testPerQueryOptionsOverrideTheModelOptions(): void + { + PgsqlPost::create(['title' => 'Shah Rukh Khan', 'body' => 'Romance']); + PgsqlPost::create(['title' => 'Salman Khan', 'body' => 'Action']); + + $this->assertSame(['Shah Rukh Khan'], PgsqlPost::search('khan -salman')->options(['mode' => 'websearch'])->get()->pluck('title')->all()); + $this->assertSame(['Salman Khan'], PgsqlPhrasePost::search('salman')->options(['mode' => 'plain'])->get()->pluck('title')->all()); + $this->assertSame(['Salman Khan'], PgsqlPost::search('salman khn')->options(['mode' => 'websearch'])->get()->pluck('title')->all()); + $this->assertCount(2, PgsqlPost::search('khan')->options(['vector' => [0.1, 0.2], 'hybrid' => ['semanticRatio' => 0.5]])->get()); + $this->assertCount(2, PgsqlExpressionPost::search('khan')->options(['vector' => [0.1, 0.2]])->get()); + } + + public function testInvalidPerQueryOptionsThrow(): void + { + $this->expectException(InvalidArgumentException::class); + + PgsqlPost::search('khan')->options(['rank' => 'pg_sleep(1) + ts_rank'])->get(); + } + + public function testExpressionSearchUsesACustomTextSearchConfig(): void + { + DB::statement('drop text search configuration if exists public.scout_simple'); + DB::statement('create text search configuration public.scout_simple (copy = simple)'); + + PgsqlExpressionPost::create(['title' => 'The Godfather', 'body' => 'Crime']); + + $this->assertSame([], PgsqlExpressionPost::search('the')->get()->all()); + $this->assertSame(['The Godfather'], PgsqlExpressionPost::search('the')->options(['language' => 'public.scout_simple'])->get()->pluck('title')->all()); + } + + public function testMissingTrigramExtensionThrows(): void + { + DB::statement('create schema scout_tenant'); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('requires the pg_trgm extension'); + + PgsqlSearchPathPost::search('godfather')->get(); + } + + public function testBlendModeAndTsRankCd(): void + { + PgsqlPost::create(['title' => 'The Godfather']); + PgsqlPost::create(['title' => 'Godfather cameo']); + PgsqlPost::create(['title' => 'Godzilla']); + + // Equal full-text ranks fall back to the newest row, unless trigram similarity breaks the tie... + $this->assertSame(['Godfather cameo', 'The Godfather'], PgsqlPost::search('godfather')->get()->pluck('title')->all()); + $this->assertSame(['The Godfather', 'Godfather cameo'], PgsqlPost::search('godfather')->options(['trigram_mode' => 'blend'])->get()->pluck('title')->all()); + $this->assertCount(2, PgsqlPost::search('godfather')->options(['rank' => 'ts_rank_cd'])->get()); + $this->assertSame(['The Godfather', 'Godfather cameo'], PgsqlPost::search('godfathr')->options(['trigram_mode' => 'blend'])->get()->pluck('title')->all()); + } + + public function testSimplePaginationKeepsFallingBackOnLaterPages(): void + { + PgsqlPost::create(['title' => 'The Godfather']); + PgsqlPost::create(['title' => 'The Godfather Part II']); + + $this->assertSame(['The Godfather Part II'], PgsqlPost::search('godfathr')->simplePaginate(1, 'page', 2)->pluck('title')->all()); + } + + public function testSimplePaginationDoesNotFallBackPastTheLastFullTextMatch(): void + { + PgsqlPost::create(['title' => 'Godfather']); + PgsqlPost::create(['title' => 'Godfarther']); + + $first = PgsqlPost::search('godfather')->simplePaginate(1, 'page', 1); + + $this->assertSame(['Godfather'], $first->pluck('title')->all()); + $this->assertFalse($first->hasMorePages()); + $this->assertSame([], PgsqlPost::search('godfather')->simplePaginate(1, 'page', 2)->items()); + + // Pagination replaces an offset set on the query, so it must not hide the first match either... + $offset = fn (EloquentBuilder $query) => $query->offset(1); + + $this->assertSame(['Godfather'], PgsqlPost::search('godfather')->query($offset)->simplePaginate(1, 'page', 1)->pluck('title')->all()); + $this->assertSame([], PgsqlPost::search('godfather')->query($offset)->simplePaginate(1, 'page', 2)->items()); + } + + public function testEquallyRankedResultsPaginateWithoutDuplicates(): void + { + foreach (range(1, 5) as $i) { + PgsqlPost::create(['title' => 'Same title']); + } + + $ids = collect(range(1, 5))->flatMap( + fn (int $page) => PgsqlPost::search('same')->paginate(1, 'page', $page)->pluck('id') + ); + + $this->assertSame([5, 4, 3, 2, 1], $ids->all()); + } + + public function testTrigramFallbackRunsOnlyWhenFullTextFindsNothing(): void + { + PgsqlPost::create(['title' => 'The Godfather']); + PgsqlPost::create(['title' => 'Godzilla']); + + $queries = []; + DB::listen(function (QueryExecuted $query) use (&$queries): void { + $queries[] = $query->sql; + }); + + $this->assertSame(['The Godfather'], PgsqlPost::search('godfather')->get()->pluck('title')->all()); + $this->assertCount(0, array_filter($queries, fn (string $sql): bool => str_contains($sql, '% ?'))); + + $this->assertSame(['The Godfather'], PgsqlPost::search('godfathr')->get()->pluck('title')->all()); + + $page = PgsqlPost::search('godfathr')->paginate(10); + $this->assertSame(1, $page->total()); + $this->assertSame(['The Godfather'], $page->pluck('title')->all()); + + $this->assertSame(['The Godfather'], PgsqlPost::search('godfathr')->simplePaginate(10)->pluck('title')->all()); + $this->assertSame([], PgsqlPost::search('nothing like it')->get()->all()); + } + + public function testSearchWithoutAVectorColumnUsesExpressionRanking(): void + { + PgsqlExpressionPost::create(['title' => 'Queues', 'body' => 'Scout keeps search indexes in sync']); + PgsqlExpressionPost::create(['title' => 'Laravel Scout', 'body' => 'Scout docs']); + PgsqlExpressionPost::create(['title' => 'Horizon', 'body' => 'Queue dashboard']); + + $this->assertSame(['Laravel Scout', 'Queues'], PgsqlExpressionPost::search('scout')->get()->pluck('title')->all()); + } + + public function testExpressionSearchMatchesARowWithANullColumn(): void + { + PgsqlExpressionPost::create(['title' => 'Laravel Scout', 'body' => null]); + + $this->assertSame(['Laravel Scout'], PgsqlExpressionPost::search('scout')->get()->pluck('title')->all()); + } + + public function testVectorColumnsMatchWithACustomTextSearchConfig(): void + { + DB::statement('drop text search configuration if exists public.scout_simple'); + DB::statement('create text search configuration public.scout_simple (copy = simple)'); + Schema::table('scout_posts', fn (Blueprint $table) => $table->dropSearchable(['title'])); + Schema::table('scout_posts', fn (Blueprint $table) => $table->searchable(['title'], 'public.scout_simple')); + + PgsqlPost::create(['title' => 'Running fast']); + + // The "english" config would stem the query to "run" and miss the unstemmed "running" lexeme... + $this->assertSame(['Running fast'], PgsqlPost::search('running')->options(['language' => 'public.scout_simple'])->get()->pluck('title')->all()); + } + + public function testSoftDeletedAndExplicitlyOrderedSearches(): void + { + PgsqlPost::create(['title' => 'Scout A', 'body' => 'scout scout scout']); + PgsqlPost::create(['title' => 'Scout B'])->delete(); + PgsqlPost::create(['title' => 'Scout C']); + + $this->assertSame(['Scout A', 'Scout C'], PgsqlPost::search('scout')->get()->pluck('title')->all()); + $this->assertSame(['Scout B'], PgsqlPost::search('scout')->onlyTrashed()->get()->pluck('title')->all()); + $this->assertSame(['Scout C', 'Scout A'], PgsqlPost::search('scout')->orderBy('title', 'desc')->get()->pluck('title')->all()); + } + + public function testSchemaQualifiedTables(): void + { + DB::statement('create schema scout_tenant'); + + $this->assertSearchableTableWorks(Schema::connection(DB::getDefaultConnection()), 'scout_tenant.scout_posts', new PgsqlTenantPost); + } + + public function testPrefixedTablesOnANamedConnection(): void + { + $default = DB::getDefaultConnection(); + + // Any accidental use of the default connection now fails... + config(['database.default' => 'missing']); + + try { + $this->assertSearchableTableWorks(Schema::connection('pgsql_prefixed'), 'scout_posts', new PgsqlPrefixedPost); + } finally { + // The test transaction is rolled back through the default connection. + config(['database.default' => $default]); + } + } + + /** + * Assert that a searchable table created through the given schema builder can be searched and dropped. + */ + protected function assertSearchableTableWorks(SchemaBuilder $schema, string $table, Model $model): void + { + $schema->create($table, function (Blueprint $table): void { + $table->id(); + $table->string('title'); + $table->text('body')->nullable(); + $table->timestamps(); + $table->softDeletes(); + $table->searchable(['title' => 'A', 'body' => 'D'], 'english', ['title']); + }); + + $model->newQuery()->create(['title' => 'The Godfather']); + $model->newQuery()->create(['title' => 'Godzilla', 'body' => 'godfather cameo']); + + $this->assertSame(['The Godfather', 'Godzilla'], $model::search('godfather')->get()->pluck('title')->all()); + $this->assertSame(['The Godfather'], $model::search('godfathr')->get()->pluck('title')->all()); + $this->assertSame(1, $model::search('godfathr')->paginate()->total()); + + $schema->table($table, fn (Blueprint $table) => $table->dropSearchable(['title'])); + + $this->assertFalse($schema->hasColumn($table, 'search_vector')); + } + + /** + * Get the definitions of the searchable table's indexes, keyed by name. + * + * @return array + */ + protected function indexes(): array + { + return collect(DB::select("select indexname, indexdef from pg_indexes where tablename = 'scout_posts' and indexname <> 'scout_posts_pkey' order by indexname")) + ->pluck('indexdef', 'indexname') + ->all(); + } +} + +class PgsqlPost extends Model +{ + use Searchable; + use SoftDeletes; + + protected ?string $table = 'scout_posts'; + + protected array $guarded = []; + + #[SearchUsingFullText(['search_vector'], ['vector' => true, 'trigram' => ['title']])] + public function toSearchableArray(): array + { + return ['search_vector' => null]; + } +} + +class PgsqlExpressionPost extends PgsqlPost +{ + #[SearchUsingFullText(['title', 'body'], ['language' => 'english'])] + public function toSearchableArray(): array + { + return ['title' => $this->title, 'body' => $this->body]; + } +} + +class PgsqlPhrasePost extends PgsqlPost +{ + #[SearchUsingFullText(['search_vector'], ['vector' => true, 'mode' => 'phrase'])] + public function toSearchableArray(): array + { + return ['search_vector' => null]; + } +} + +class PgsqlWebsearchPost extends PgsqlPost +{ + #[SearchUsingFullText(['search_vector'], ['vector' => true, 'mode' => 'websearch'])] + public function toSearchableArray(): array + { + return ['search_vector' => null]; + } +} + +class PgsqlTenantPost extends PgsqlPost +{ + protected ?string $table = 'scout_tenant.scout_posts'; +} + +class PgsqlSearchPathPost extends PgsqlPost +{ + protected UnitEnum|string|null $connection = 'pgsql_tenant'; +} + +class PgsqlPrefixedPost extends PgsqlPost +{ + protected UnitEnum|string|null $connection = 'pgsql_prefixed'; +} diff --git a/tests/Scout/Feature/DatabaseEngineFullTextOptionsTest.php b/tests/Scout/Feature/DatabaseEngineFullTextOptionsTest.php new file mode 100644 index 0000000000..4d2a28b7f2 --- /dev/null +++ b/tests/Scout/Feature/DatabaseEngineFullTextOptionsTest.php @@ -0,0 +1,73 @@ +assertSame( + ['mode' => 'natural', 'rank' => 'foo', 'expand' => true], + $this->fullTextOptions(new LegacyFullTextOptionsModel) + ); + } + + #[RequiresDatabase(['sqlite', 'mysql', 'mariadb'])] + public function testSearchableSchemaRequiresPostgres(): void + { + SearchableSchema::register(); + + $this->expectException(RuntimeException::class); + + Schema::create('posts', fn (Blueprint $table) => $table->searchable(['title'])); + } + + #[DataProvider('invalidOptions')] + public function testInvalidOptionsThrow(array $options, string $message): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($message); + + (fn () => $this->ensureValidFullTextOptions($options))->call(new DatabaseEngine); + } + + public static function invalidOptions(): array + { + return [ + 'rank' => [['rank' => 'pg_sleep(1) + ts_rank'], '[rank] must be one of'], + 'trigram mode' => [['trigram_mode' => 'off'], '[trigram_mode] must be one of'], + ]; + } + + /** + * Get the full-text options the database engine resolves for the given model. + */ + protected function fullTextOptions(Model $model): array + { + return (fn () => $this->getFullTextOptions(new Builder($model, 'query')))->call(new DatabaseEngine); + } +} + +class LegacyFullTextOptionsModel extends Model +{ + #[SearchUsingFullText(['title'], ['mode' => 'natural', 'rank' => 'foo', 'expand' => true])] + public function toSearchableArray(): array + { + return []; + } +} diff --git a/tests/Scout/Unit/SearchableSchemaTest.php b/tests/Scout/Unit/SearchableSchemaTest.php new file mode 100644 index 0000000000..982d17d362 --- /dev/null +++ b/tests/Scout/Unit/SearchableSchemaTest.php @@ -0,0 +1,37 @@ +assertSame( + "setweight(to_tsvector('english', coalesce(\"title\"::text, '')), 'A') || setweight(to_tsvector('english', coalesce(\"body\"::text, '')), 'D')", + SearchableSchema::vectorExpression(['title' => 'a', 'body'], 'english') + ); + } + + public function testVectorExpressionAcceptsSchemaQualifiedLanguages(): void + { + $this->assertStringContainsString("to_tsvector('public.english_unaccent'", SearchableSchema::vectorExpression(['title'], 'public.english_unaccent')); + } + + public function testVectorExpressionRejectsInvalidInput(): void + { + foreach ([[[], 'english'], [['title'], "english'); drop table users; --"], [['title' => 'E'], 'english'], [['ti"tle'], 'english']] as [$columns, $language]) { + try { + SearchableSchema::vectorExpression($columns, $language); + $this->fail('Expected an InvalidArgumentException.'); + } catch (InvalidArgumentException) { + $this->addToAssertionCount(1); + } + } + } +} From 99d9a496b61dbf2a84c5e50b58a6761aafe33172 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:40:01 +0000 Subject: [PATCH 06/30] Read the first-party schema macros in PHPStan's migration reader PHPStan's migration reader treats Blueprint macros as unknown, so a migration calling one lost every column fact of its table, and attributes of models on that table read as mixed. That covered Scout's documented searchable() macro and nested set's schema macros. The reader now applies the columns these first-party macros add and drop: Scout's generated tsvector column, by its default or literal name, and nested set's bound, depth and parent columns. It only does so when the booted application's macro is the one Scout or nested set registered; an application's own macro of the same name stays unknown. Which macros are recognized is part of the migration cache input, and the cache format version is bumped, so replacing a macro invalidates cached schemas. Tests: schema aggregator tests for both packages' macros, a dynamic column name and an application macro of the same name, and a migration cache test that replaces the macro after the cache is warm. The PHPStan test bootstrap registers the macros as their providers would. The static analysis docs name the exception. Validation: the PHPStan analyzer suite passes; formatting and full static analysis pass. --- src/docs/static-analysis.md | 2 +- .../Properties/ConnectionConfigResolver.php | 3 +- src/phpstan/src/Properties/MigrationCache.php | 2 +- .../src/Properties/SchemaAggregator.php | 100 ++++++++++++++++++ tests/PHPStan/Unit/MigrationCacheTest.php | 30 ++++++ tests/PHPStan/Unit/SchemaAggregatorTest.php | 84 +++++++++++++++ tests/PHPStan/phpunit-bootstrap.php | 6 ++ 7 files changed, 224 insertions(+), 3 deletions(-) diff --git a/src/docs/static-analysis.md b/src/docs/static-analysis.md index 23b0c11ca6..573dc70f11 100644 --- a/src/docs/static-analysis.md +++ b/src/docs/static-analysis.md @@ -368,7 +368,7 @@ $schema->create('events', function (Blueprint $table) { }); ``` -Dynamic migration code can limit attribute typing. Blueprint macros, loops and helpers that receive a blueprint make its table unknown. A `switch` statement whose case can end early, such as through a `break` inside an `if`, makes the tables it changes unknown. When a migration, its base class or one of its traits assigns the `$connection` property outside the `down` method, such as in a constructor, the connection its calls use by default is unknown. A trait method a migration renames with `as` or selects with `insteadof`, or a method called by a dynamic name such as `$this->{$method}()`, makes every connection's schema unknown. A dynamic method name on the `Schema` facade or a schema builder, such as `Schema::{$method}()`, makes that connection's schema unknown. Unknown column names leave room for additional columns; unknown table names make the connection's schema unknown, and unknown connection names make every connection's schema unknown. A column created in only some branches reads as `mixed`. +Dynamic migration code can limit attribute typing. Blueprint macros, loops and helpers that receive a blueprint make its table unknown. The nested set and Scout `searchable` schema macros are the exception: their columns are read like any other column. A `switch` statement whose case can end early, such as through a `break` inside an `if`, makes the tables it changes unknown. When a migration, its base class or one of its traits assigns the `$connection` property outside the `down` method, such as in a constructor, the connection its calls use by default is unknown. A trait method a migration renames with `as` or selects with `insteadof`, or a method called by a dynamic name such as `$this->{$method}()`, makes every connection's schema unknown. A dynamic method name on the `Schema` facade or a schema builder, such as `Schema::{$method}()`, makes that connection's schema unknown. Unknown column names leave room for additional columns; unknown table names make the connection's schema unknown, and unknown connection names make every connection's schema unknown. A column created in only some branches reads as `mixed`. Use the `Schema` and `DB` facades for schema changes. Unlike a schema builder, a database connection stored in a variable is not followed, so `$connection = DB::connection('analytics')` makes that connection's schema unknown, even if the connection only updates rows. Schema changes through connections obtained in other ways, such as from a property or `app('db')`, are not followed either. To target another database or schema, configure a connection and select it instead of relying on `USE` or `SET search_path` across separate SQL calls. diff --git a/src/phpstan/src/Properties/ConnectionConfigResolver.php b/src/phpstan/src/Properties/ConnectionConfigResolver.php index 10e02d6f82..7fee74301e 100644 --- a/src/phpstan/src/Properties/ConnectionConfigResolver.php +++ b/src/phpstan/src/Properties/ConnectionConfigResolver.php @@ -130,7 +130,7 @@ public function getConsumedKeys(): array } /** - * Get the normalized configuration that schema aggregation reads. + * Get the normalized configuration and boot state that schema aggregation reads. * * Fetch options and custom resolvers only affect lookups, so changing them keeps the aggregated schema. * @@ -156,6 +156,7 @@ public function getSchemaConfiguration(): array ]; }, $names)), 'morphKeyType' => Builder::$defaultMorphKeyType, + 'macros' => SchemaAggregator::getKnownMacros(), ]; } diff --git a/src/phpstan/src/Properties/MigrationCache.php b/src/phpstan/src/Properties/MigrationCache.php index 1c6ef4fb7f..b0672c1d1a 100644 --- a/src/phpstan/src/Properties/MigrationCache.php +++ b/src/phpstan/src/Properties/MigrationCache.php @@ -44,7 +44,7 @@ class MigrationCache private const string CACHE_PREFIX = 'hypervel_migrations_'; /** Bump when the serialized entry, SchemaConnection/SchemaTable/SchemaColumn or the aggregation logic changes. */ - private const string CACHE_FORMAT_VERSION = '9'; + private const string CACHE_FORMAT_VERSION = '10'; /** * Create a new migration cache instance. diff --git a/src/phpstan/src/Properties/SchemaAggregator.php b/src/phpstan/src/Properties/SchemaAggregator.php index 87baa82a67..7ee82ad742 100644 --- a/src/phpstan/src/Properties/SchemaAggregator.php +++ b/src/phpstan/src/Properties/SchemaAggregator.php @@ -4,15 +4,19 @@ namespace Hypervel\PHPStan\Properties; +use Closure; use Hypervel\Database\Eloquent\Concerns\HasUlids; use Hypervel\Database\Eloquent\Model; use Hypervel\Database\Migrations\Migration; use Hypervel\Database\Schema\Blueprint; use Hypervel\Database\Schema\Builder; +use Hypervel\NestedSet\NestedSet; +use Hypervel\NestedSet\NestedSetServiceProvider; use Hypervel\PHPStan\Reflection\SourceFiles; use Hypervel\SchemaParser\ColumnType; use Hypervel\SchemaParser\ParseException; use Hypervel\SchemaParser\View; +use Hypervel\Scout\SearchableSchema; use Hypervel\Support\Facades\Config; use Hypervel\Support\Facades\DB; use Hypervel\Support\Facades\Schema; @@ -24,6 +28,8 @@ use PHPStan\Reflection\InitializerExprTypeResolver; use PHPStan\Reflection\ReflectionProvider; use ReflectionException; +use ReflectionFunction; +use ReflectionProperty; use WeakMap; use function array_any; @@ -46,6 +52,8 @@ use function str_starts_with; use function strtolower; +use const ARRAY_FILTER_USE_BOTH; + /** * Applies migrations to the schema facts of the connections they run on. * @@ -128,6 +136,14 @@ class SchemaAggregator /** Blueprint methods whose third parameter decides whether the column is unsigned. */ private const array SIGNABLE_COLUMN_METHODS = ['biginteger', 'integer', 'mediuminteger', 'smallinteger', 'tinyinteger']; + /** The first-party Blueprint macros whose schema changes are known, with the class registering each. */ + private const array KNOWN_MACROS = [ + 'dropNestedSet' => NestedSetServiceProvider::class, 'integerNestedSet' => NestedSetServiceProvider::class, + 'nestedSet' => NestedSetServiceProvider::class, 'ulidNestedSet' => NestedSetServiceProvider::class, + 'uuidNestedSet' => NestedSetServiceProvider::class, 'dropSearchable' => SearchableSchema::class, + 'searchable' => SearchableSchema::class, + ]; + /** Blueprint methods that add a nullable column. */ private const array NULLABLE_COLUMN_METHODS = ['remembertoken', 'softdeletes', 'softdeletesdatetime', 'softdeletestz']; @@ -186,6 +202,9 @@ class SchemaAggregator /** The type of the key columns morphs() adds, or null when a migration changed it to an unknown value. */ private ?string $morphKeyType; + /** @var list the known first-party Blueprint macros the booted application registers */ + private array $knownMacros; + /** @var array the declarations of the helper methods being applied, to stop recursion */ private array $appliedMethods = []; @@ -218,6 +237,26 @@ public function __construct( public array $connections = [], ) { $this->morphKeyType = Builder::$defaultMorphKeyType; + $this->knownMacros = self::getKnownMacros(); + } + + /** + * Get the known first-party Blueprint macros the booted application registers. + * + * An application may replace one with its own macro, whose schema changes are unknown. + * + * @return list + */ + public static function getKnownMacros(): array + { + $macros = (new ReflectionProperty(Blueprint::class, 'macros'))->getValue(); + + return array_keys(array_filter( + self::KNOWN_MACROS, + static fn (string $class, string $name): bool => ($macros[$name] ?? null) instanceof Closure + && (new ReflectionFunction($macros[$name]))->getClosureScopeClass()?->getName() === $class, + ARRAY_FILTER_USE_BOTH, + )); } /** @@ -2138,6 +2177,43 @@ private function processStatementAlterMethod( $this->setColumn($connection, $table, 'created_at', $type, false, true); $this->setColumn($connection, $table, 'updated_at', $type, false, true); + return true; + case 'nestedset': + case 'integernestedset': + case 'uuidnestedset': + case 'ulidnestedset': + // Nested-set's scopes only choose the indexes. + if (! $this->callsKnownMacro($call)) { + return false; + } + + $this->setColumn($connection, $table, NestedSet::LFT, 'integer', true, false); + $this->setColumn($connection, $table, NestedSet::RGT, 'integer', true, false); + $this->setColumn($connection, $table, NestedSet::DEPTH, 'smallinteger', true, false); + [$parentType, $parentUnsigned] = match ($method) { + 'nestedset' => ['biginteger', true], + 'integernestedset' => ['integer', true], + 'uuidnestedset' => ['uuid', false], + default => ['char', false], + }; + + $this->setColumn($connection, $table, NestedSet::PARENT_ID, $parentType, $parentUnsigned, true); + + return true; + case 'searchable': + // Scout's indexes and the pg_trgm extension add no columns or relations. + if (! $this->callsKnownMacro($call)) { + return false; + } + + $name = $this->getNameArgument($call, 3, 'tsvector', 'search_vector', $table); + + if ($name === null) { + return $this->addUnnamedColumn($table, false); + } + + $this->setColumn($connection, $table, $name, 'tsvector', false, false); + return true; case 'renamecolumn': $from = $this->getNameArgument($call, 0, 'from', table: $table); @@ -2192,6 +2268,22 @@ private function processStatementAlterMethod( $name = $this->getNameArgument($call, 0, 'name', table: $table); $columns = $name === null ? [null] : ["{$name}_type", "{$name}_id"]; + break; + case 'dropnestedset': + if (! $this->callsKnownMacro($call)) { + return false; + } + + $columns = NestedSet::getDefaultColumns(); + + break; + case 'dropsearchable': + if (! $this->callsKnownMacro($call)) { + return false; + } + + $columns = [$this->getNameArgument($call, 1, 'tsvector', 'search_vector', $table)]; + break; default: $columns = match ($method) { @@ -2295,6 +2387,14 @@ private function addUnnamedColumn(SchemaTable $table, bool $change): bool return true; } + /** + * Determine if a Blueprint call reaches a known first-party macro. + */ + private function callsKnownMacro(PhpParser\Node\Expr\MethodCall $call): bool + { + return $call->name instanceof PhpParser\Node\Identifier && in_array($call->name->toString(), $this->knownMacros, true); + } + /** * Get the value the last nullable() or unsigned() modifier of a chain sets, the default without one, or null when it is not a literal. * diff --git a/tests/PHPStan/Unit/MigrationCacheTest.php b/tests/PHPStan/Unit/MigrationCacheTest.php index 1e0aa20475..a55d25f176 100644 --- a/tests/PHPStan/Unit/MigrationCacheTest.php +++ b/tests/PHPStan/Unit/MigrationCacheTest.php @@ -6,6 +6,7 @@ use Hypervel\Config\Repository; use Hypervel\Container\Container; +use Hypervel\Database\Schema\Blueprint; use Hypervel\Filesystem\Filesystem; use Hypervel\PHPStan\Properties\ConnectionConfigResolver; use Hypervel\PHPStan\Properties\MigrationCache; @@ -13,6 +14,7 @@ use Hypervel\PHPStan\Properties\SchemaTable; use Hypervel\Testing\ParallelTesting; use Hypervel\Tests\PHPStan\TestCase; +use ReflectionProperty; use SplFileInfo; use function clearstatcache; @@ -262,6 +264,34 @@ public function testCacheHitConsumesTheConfigKeysTheMigrationsRead(): void $this->assertSame(['analytics.connection'], $resolver->getConsumedKeys()); } + /** + * An application macro replacing a first-party schema macro makes its changes unknown, so the entry + * aggregated with the first-party macro no longer applies. + */ + public function testReplacingAKnownSchemaMacroInvalidatesTheEntry(): void + { + $migrationFiles = [new SplFileInfo($this->createTempFile('mig1.php'))]; + $calls = 0; + $aggregate = static function () use (&$calls): array { + ++$calls; + + return self::schema([]); + }; + + $this->cache->remember($migrationFiles, [], $this->resolver->getSchemaConfiguration(), $aggregate); + + $macro = (new ReflectionProperty(Blueprint::class, 'macros'))->getValue()['searchable']; + Blueprint::macro('searchable', function (): void {}); + + try { + $this->cache->remember($migrationFiles, [], $this->resolver->getSchemaConfiguration(), $aggregate); + } finally { + Blueprint::macro('searchable', $macro); + } + + $this->assertSame(2, $calls); + } + /** * The PHP files the migrations read, such as one declaring a constant, invalidate the entry when they * change or are deleted, and a hit returns them so their dependencies are declared again. diff --git a/tests/PHPStan/Unit/SchemaAggregatorTest.php b/tests/PHPStan/Unit/SchemaAggregatorTest.php index fb723c43f1..2d5593b8a6 100644 --- a/tests/PHPStan/Unit/SchemaAggregatorTest.php +++ b/tests/PHPStan/Unit/SchemaAggregatorTest.php @@ -6,6 +6,7 @@ use Hypervel\Database\Eloquent\Concerns\HasUniqueStringIds; use Hypervel\Database\Eloquent\Concerns\HasUuids; +use Hypervel\Database\Schema\Blueprint; use Hypervel\PHPStan\Properties\ConnectionConfigResolver; use Hypervel\PHPStan\Properties\SchemaAggregator; use Hypervel\PHPStan\Properties\SchemaConnection; @@ -26,6 +27,7 @@ use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Test; use ReflectionClass; +use ReflectionProperty; use function array_keys; use function array_map; @@ -359,6 +361,88 @@ public function itReadsForeignIdForColumnsOfUnknownModels(): void $this->assertFalse($this->table($aggregator, 'cities')->complete); } + #[Test] + public function itReadsTheColumnsOfNestedSetMacros(): void + { + $aggregator = $this->aggregate(<<<'PHP' + Schema::create('categories', function (Blueprint $table) { + $table->string('name'); + $table->nestedSet(); + }); + + Schema::create('integer_categories', fn (Blueprint $table) => $table->integerNestedSet()); + Schema::create('uuid_categories', fn (Blueprint $table) => $table->uuidNestedSet(['tenant_id'])); + Schema::create('ulid_categories', fn (Blueprint $table) => $table->ulidNestedSet()); + + Schema::create('flat_categories', function (Blueprint $table) { + $table->string('name'); + $table->nestedSet(); + }); + + Schema::table('flat_categories', fn (Blueprint $table) => $table->dropNestedSet()); + PHP); + + $bounds = ['_lft' => ['int<0, max>', false], '_rgt' => ['int<0, max>', false], 'depth' => ['int<0, max>', false]]; + + $this->assertSame(['name' => ['string', false]] + $bounds + ['parent_id' => ['int<0, max>|numeric-string', true]], $this->columns($aggregator, 'categories')); + $this->assertSame($bounds + ['parent_id' => ['int<0, max>', true]], $this->columns($aggregator, 'integer_categories')); + $this->assertSame($bounds + ['parent_id' => ['string', true]], $this->columns($aggregator, 'uuid_categories')); + $this->assertSame($bounds + ['parent_id' => ['string', true]], $this->columns($aggregator, 'ulid_categories')); + $this->assertSame(['name' => ['string', false]], $this->columns($aggregator, 'flat_categories')); + $this->assertTrue($this->table($aggregator, 'flat_categories')->complete); + } + + #[Test] + public function itReadsTheVectorColumnsOfScoutSearchableMacros(): void + { + $aggregator = $this->aggregate(<<<'PHP' + Schema::create('posts', function (Blueprint $table) { + $table->string('title'); + $table->searchable(['title']); + $table->searchable(['title'], tsvector: 'title_vector'); + $table->searchable(['title'], tsvector: 'body_vector'); + }); + + Schema::table('posts', fn (Blueprint $table) => $table->dropSearchable(tsvector: 'title_vector')); + Schema::table('posts', fn (Blueprint $table) => $table->dropSearchable()); + PHP, 'pgsql'); + + $this->assertSame(['title' => ['string', false], 'body_vector' => ['string', false]], $this->columns($aggregator, 'posts', 'pgsql')); + $this->assertTrue($this->table($aggregator, 'posts', 'pgsql')->complete); + + $aggregator = $this->aggregate(<<<'PHP' + Schema::create('posts', function (Blueprint $table) { + $table->string('title'); + $table->searchable(['title'], tsvector: $this->column); + }); + PHP, 'pgsql', members: 'public string $column = "search_vector";'); + + $this->assertSame(['title' => ['string', false]], $this->columns($aggregator, 'posts', 'pgsql')); + $this->assertFalse($this->table($aggregator, 'posts', 'pgsql')->complete); + } + + #[Test] + public function itForgetsTablesAnotherSearchableMacroChanges(): void + { + $macro = (new ReflectionProperty(Blueprint::class, 'macros'))->getValue()['searchable']; + + Blueprint::macro('searchable', function (): void {}); + + try { + $aggregator = $this->aggregate(<<<'PHP' + Schema::create('posts', function (Blueprint $table) { + $table->string('title'); + $table->searchable(['title']); + }); + PHP, 'pgsql'); + } finally { + Blueprint::macro('searchable', $macro); + } + + $this->assertSame([], $this->columns($aggregator, 'posts', 'pgsql')); + $this->assertFalse($this->table($aggregator, 'posts', 'pgsql')->complete); + } + /** * Unlike upstream, which skipped the calls, names that are not strings make what they change unknown. */ diff --git a/tests/PHPStan/phpunit-bootstrap.php b/tests/PHPStan/phpunit-bootstrap.php index cd6f59d265..fecdd3fb1f 100644 --- a/tests/PHPStan/phpunit-bootstrap.php +++ b/tests/PHPStan/phpunit-bootstrap.php @@ -14,6 +14,8 @@ use Hypervel\Database\Eloquent\Relations\Relation; use Hypervel\Filesystem\Filesystem; use Hypervel\Foundation\Console\ClosureCommand; +use Hypervel\NestedSet\NestedSetServiceProvider; +use Hypervel\Scout\SearchableSchema; use Hypervel\Testbench\Bootstrapper; use Hypervel\Tests\PHPStan\Fixtures\Config\ConfigFixtureServiceProvider; use Hypervel\Tests\PHPStan\Fixtures\Config\OverlappingConfigServiceProvider; @@ -67,6 +69,10 @@ // loadMorph() reads by class first and MorphTo by alias first. Relation::morphMap(['post' => Post::class, Group::class => Role::class]); +// First-party schema macros, registered as their providers would. +SearchableSchema::register(); +Container::getInstance()->register(NestedSetServiceProvider::class); + // Package configuration sources, which the config analysis reads from the loaded providers. Container::getInstance()->register(ConfigFixtureServiceProvider::class); Container::getInstance()->register(OverlappingConfigServiceProvider::class); From 6651444c1d911e9f2a544b378c358e61da9f41a3 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:40:09 +0000 Subject: [PATCH 07/30] Quote Typesense filter values that contain special characters Ports laravel/scout#1023 from laravel/scout at bb583fec. Typesense filter values containing &&, ||, parentheses, commas or backticks broke the filter_by expression. Where, whereIn and whereNotIn values containing them are now wrapped in backticks, with any backticks removed from the value: Typesense can't parse escaped backticks, and it ignores special characters when matching, so the stripped value still matches the stored one. Array where values are raw filter expressions again, as upstream: an equality filter is sent as key:value, so geo filters reach Typesense as location:(...). Hypervel had sent location:=(...), which Typesense rejects. Upstream defect fixed: upstream also drops explicit operators for array values, so a != filter became a positive match; Hypervel keeps the operator and still rejects unsupported operators. Tests: ported the Typesense quoting test and Meilisearch's escaped where values test, restored two upstream Typesense assertions Hypervel had dropped, and added an integration test against a live Typesense server plus array cases for explicit and unsupported operators. Validation: the Scout suite and the Typesense integration tests pass; formatting and full static analysis pass. --- src/scout/src/Engines/TypesenseEngine.php | 37 ++++++++++++--- .../TypesenseFilteringIntegrationTest.php | 22 +++++++++ .../Unit/Engines/MeilisearchEngineTest.php | 22 +++++++++ .../Unit/Engines/TypesenseEngineTest.php | 46 ++++++++++++++++++- 4 files changed, 119 insertions(+), 8 deletions(-) diff --git a/src/scout/src/Engines/TypesenseEngine.php b/src/scout/src/Engines/TypesenseEngine.php index ac49d9e1d0..71ddb93fc7 100644 --- a/src/scout/src/Engines/TypesenseEngine.php +++ b/src/scout/src/Engines/TypesenseEngine.php @@ -21,6 +21,7 @@ use Hypervel\Support\Attributes\Unsupported; use Hypervel\Support\Collection; use Hypervel\Support\LazyCollection; +use Hypervel\Support\Str; use InvalidArgumentException; use stdClass; use Typesense\Client as Typesense; @@ -728,6 +729,20 @@ protected function parseFilterValue(array|string|bool|int|float|BackedEnum $valu return $value; } + /** + * Wrap the given filter value in backticks if it contains any of the given characters. + * + * @param array $characters + */ + protected function quoteFilterValue(mixed $value, array $characters): mixed + { + if (! is_string($value) || ! Str::contains($value, $characters)) { + return $value; + } + + return '`' . Str::remove('`', $value) . '`'; + } + /** * Create a "where" filter string. * @@ -739,7 +754,9 @@ protected function parseWhereFilter( string $operator = '=' ): string { $operator = match ($operator) { - '=' => ':=', + // Array values are raw filter expressions, such as geo filters, so equality leaves the operator out. Unlike + // upstream, explicit operators are kept for array values instead of being dropped. + '=' => is_array($value) ? ':' : ':=', '!=' => ':!=', '<' => ':<', '>' => ':>', @@ -748,9 +765,13 @@ protected function parseWhereFilter( default => throw new InvalidArgumentException("Unsupported Typesense filter operator [{$operator}]."), }; - return is_array($value) - ? sprintf('%s%s%s', $key, $operator, implode('', $value)) - : sprintf('%s%s%s', $key, $operator, $value); + if (is_array($value)) { + return sprintf('%s%s%s', $key, $operator, collect($value) + ->map(fn (mixed $value): mixed => $this->quoteFilterValue($value, ['&&', '||', '`'])) + ->implode('')); + } + + return sprintf('%s%s%s', $key, $operator, $this->quoteFilterValue($value, ['&&', '||', '`', '(', ')'])); } /** @@ -760,7 +781,9 @@ protected function parseWhereFilter( */ protected function parseWhereInFilter(array $value, string $key): string { - return sprintf('%s:=[%s]', $key, implode(', ', $value)); + return sprintf('%s:=[%s]', $key, collect($value) + ->map(fn (mixed $value): mixed => $this->quoteFilterValue($value, ['&&', '||', '`', '(', ')', ','])) + ->implode(', ')); } /** @@ -770,7 +793,9 @@ protected function parseWhereInFilter(array $value, string $key): string */ protected function parseWhereNotInFilter(array $value, string $key): string { - return sprintf('%s:!=[%s]', $key, implode(', ', $value)); + return sprintf('%s:!=[%s]', $key, collect($value) + ->map(fn (mixed $value): mixed => $this->quoteFilterValue($value, ['&&', '||', '`', '(', ')', ','])) + ->implode(', ')); } /** diff --git a/tests/Integration/Scout/Typesense/TypesenseFilteringIntegrationTest.php b/tests/Integration/Scout/Typesense/TypesenseFilteringIntegrationTest.php index 59b390a77d..c80d0ed301 100644 --- a/tests/Integration/Scout/Typesense/TypesenseFilteringIntegrationTest.php +++ b/tests/Integration/Scout/Typesense/TypesenseFilteringIntegrationTest.php @@ -79,6 +79,28 @@ public function testWhereNotInExcludesSpecifiedValues(): void $this->assertSame($model2->id, $results->first()->id); } + public function testFiltersMatchValuesContainingSpecialCharacters(): void + { + foreach (['Laravel Framework', 'Rock || Roll', 'Acme (UK)', 'Acme, Inc', 'Acme'] as $title) { + TypesenseSearchableModel::create(['title' => $title, 'body' => 'Body']); + } + + TypesenseSearchableModel::query()->get()->each(fn ($m) => $this->engine->update(new EloquentCollection([$m]))); + + $this->assertSame(['Rock || Roll'], TypesenseSearchableModel::search('')->where('title', 'Rock || Roll')->get()->pluck('title')->all()); + $this->assertSame(['Acme (UK)'], TypesenseSearchableModel::search('')->where('title', 'Acme (UK)')->get()->pluck('title')->all()); + $this->assertEqualsCanonicalizing( + ['Laravel Framework', 'Acme, Inc'], + TypesenseSearchableModel::search('')->whereIn('title', ['Laravel Framework', 'Acme, Inc'])->get()->pluck('title')->all() + ); + + $titles = TypesenseSearchableModel::search('')->whereNotIn('title', ['Rock || Roll', 'Acme, Inc'])->get()->pluck('title')->all(); + + $this->assertNotContains('Rock || Roll', $titles); + $this->assertNotContains('Acme, Inc', $titles); + $this->assertContains('Acme', $titles); + } + public function testMultipleWhereClausesAreCombinedWithAnd(): void { TypesenseSearchableModel::create(['title' => 'PHP Guide', 'body' => 'Content A']); diff --git a/tests/Scout/Unit/Engines/MeilisearchEngineTest.php b/tests/Scout/Unit/Engines/MeilisearchEngineTest.php index 72d5af5ef7..661047aa80 100644 --- a/tests/Scout/Unit/Engines/MeilisearchEngineTest.php +++ b/tests/Scout/Unit/Engines/MeilisearchEngineTest.php @@ -401,6 +401,28 @@ public function testSearchWithFilters(): void $engine->search($builder); } + public function testWhereValuesAreEscaped(): void + { + $client = Double::for(Client::class)->strict(); + $index = Double::for(Indexes::class)->strict(); + $client->expects('index')->with('test_index')->returns($index); + $index->expects('rawSearch')->with('', [ + 'filter' => 'size="12\" pipe" AND path="C:\\\dir\\\"', + ])->returns(['hits' => [], 'totalHits' => 0]); + + $engine = new MeilisearchEngine($client); + + $model = Double::for(MeilisearchTestSearchableModel::class)->strict(); + $model->allows('searchableAs')->returns('test_index'); + $model->allows('getScoutKeyName')->returns('id'); + + $builder = new Builder($model, ''); + $builder->where('size', '12" pipe'); + $builder->where('path', 'C:\dir\\'); + + $engine->search($builder); + } + public function testWhereInAndWhereNotInValuesAreEscaped(): void { $client = Double::for(Client::class)->strict(); diff --git a/tests/Scout/Unit/Engines/TypesenseEngineTest.php b/tests/Scout/Unit/Engines/TypesenseEngineTest.php index 8db179920a..3e38a5611a 100644 --- a/tests/Scout/Unit/Engines/TypesenseEngineTest.php +++ b/tests/Scout/Unit/Engines/TypesenseEngineTest.php @@ -110,6 +110,36 @@ public function testFiltersMethod(): void $this->assertStringContainsString('brand:!=[apple, samsung]', $result); } + public function testFilterValuesWithSpecialCharactersAreQuoted(): void + { + $engine = $this->createEngine(); + + $builder = Double::for(Builder::class)->strict(); + $builder->wheres = [ + ['field' => 'band', 'value' => 'Rock || Roll', 'operator' => '='], + ['field' => 'company', 'value' => 'Acme (UK)', 'operator' => '!='], + ['field' => 'title', 'value' => 'It`s R&&D', 'operator' => '='], + ['field' => 'name', 'value' => 'New*', 'operator' => '='], + ['field' => 'price', 'value' => '[10..100]', 'operator' => '='], + ['field' => 'location', 'value' => ['(48.85, 2.34, 5 km)'], 'operator' => '='], + ['field' => 'status', 'value' => ['Rock || Roll'], 'operator' => '='], + ]; + $builder->whereIns = [ + 'company' => ['Laravel', 'Acme (UK)', 'Acme, Inc'], + ]; + $builder->whereNotIns = [ + 'band' => ['Rock || Roll'], + ]; + + $result = $this->invokeMethod($engine, 'filters', [$builder]); + + $this->assertSame( + 'band:=`Rock || Roll` && company:!=`Acme (UK)` && title:=`Its R&&D` && name:=New* && price:=[10..100] && location:(48.85, 2.34, 5 km)' + . ' && status:`Rock || Roll` && company:=[Laravel, `Acme (UK)`, `Acme, Inc`] && band:!=[`Rock || Roll`]', + $result + ); + } + public function testParseFilterValueMethod(): void { $engine = $this->createEngine(); @@ -135,16 +165,27 @@ public function testParseWhereFilterMethod(): void $this->assertEquals('age:=25', $this->invokeMethod($engine, 'parseWhereFilter', [25, 'age'])); $this->assertEquals('age:>=25', $this->invokeMethod($engine, 'parseWhereFilter', [25, 'age', '>='])); $this->assertEquals('status:!=inactive', $this->invokeMethod($engine, 'parseWhereFilter', ['inactive', 'status', '!='])); + $this->assertEquals('tags:tag1tag2tag3', $this->invokeMethod($engine, 'parseWhereFilter', [['tag1', 'tag2', 'tag3'], 'tags'])); + $this->assertEquals('status:!=inactive', $this->invokeMethod($engine, 'parseWhereFilter', [['inactive'], 'status', '!='])); } - public function testParseWhereFilterRejectsUnsupportedOperators(): void + #[DataProvider('unsupportedOperatorValues')] + public function testParseWhereFilterRejectsUnsupportedOperators(array|string $value): void { $engine = $this->createEngine(); $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessageIs('Unsupported Typesense filter operator [LIKE].'); - $this->invokeMethod($engine, 'parseWhereFilter', ['active', 'status', 'LIKE']); + $this->invokeMethod($engine, 'parseWhereFilter', [$value, 'status', 'LIKE']); + } + + public static function unsupportedOperatorValues(): array + { + return [ + 'scalar' => ['active'], + 'array' => [['active']], + ]; } public function testParseWhereInFilterMethod(): void @@ -155,6 +196,7 @@ public function testParseWhereInFilterMethod(): void 'category:=[electronics, books]', $this->invokeMethod($engine, 'parseWhereInFilter', [['electronics', 'books'], 'category']) ); + $this->assertEquals('id:=[1, 2, 3]', $this->invokeMethod($engine, 'parseWhereInFilter', [[1, 2, 3], 'id'])); } public function testParseWhereNotInFilterMethod(): void From 0c410a6031ef90c388d1333d6a1bdc3a696da91f Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:40:19 +0000 Subject: [PATCH 08/30] Add the SearchableAs attribute for configuring a model's index name Ports laravel/scout#1026 from laravel/scout at bb583fec. A model can set its search index name with #[SearchableAs('posts_index')] instead of overriding searchableAs(). The name is prefixed with scout.prefix like the table name, and child models inherit it. As upstream, only the model class and its parents are searched; attributes on traits are ignored. Adaptation: a cached class-attribute presence check skips reflection on each searchableAs() call for models without the attribute. Scout now declares hypervel/reflection, which provides that cache. Tests: ported upstream's SearchableAs tests and fixtures, folded Hypervel's duplicate default-name test into upstream's, and added a test that a trait's attribute doesn't override the parent's. Upstream has no docs; the Scout index configuration docs now show the attribute. Validation: the Scout suite passes; formatting and full static analysis pass. --- src/docs/scout.md | 12 +++++ src/scout/composer.json | 1 + src/scout/src/Attributes/SearchableAs.php | 18 +++++++ src/scout/src/Searchable.php | 27 ++++++++++- tests/Scout/Feature/SearchableModelTest.php | 47 +++++++++++++++++-- .../Models/SearchableAsChildModel.php | 9 ++++ .../Fixtures/Models/SearchableAsModel.php | 20 ++++++++ 7 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 src/scout/src/Attributes/SearchableAs.php create mode 100644 tests/Scout/Fixtures/Models/SearchableAsChildModel.php create mode 100644 tests/Scout/Fixtures/Models/SearchableAsModel.php diff --git a/src/docs/scout.md b/src/docs/scout.md index d67ed00580..18c37841c2 100644 --- a/src/docs/scout.md +++ b/src/docs/scout.md @@ -589,6 +589,18 @@ class Post extends Model } ``` +Instead of overriding the method, you may use the `SearchableAs` attribute. Like the table name, the attribute's index name is prefixed with your configured `scout.prefix`, and child models inherit it: + +```php +use Hypervel\Scout\Attributes\SearchableAs; + +#[SearchableAs('posts_index')] +class Post extends Model +{ + use Searchable; +} +``` + > [!NOTE] > The `searchableAs` method has no effect when using the database engine, which always searches the model's database table directly. diff --git a/src/scout/composer.json b/src/scout/composer.json index 466db8a4df..e8ffff2840 100644 --- a/src/scout/composer.json +++ b/src/scout/composer.json @@ -53,6 +53,7 @@ "hypervel/macroable": "^0.4", "hypervel/pagination": "^0.4", "hypervel/queue": "^0.4", + "hypervel/reflection": "^0.4", "hypervel/support": "^0.4", "psr/http-client": "^1.0", "psr/http-message": "^2.0", diff --git a/src/scout/src/Attributes/SearchableAs.php b/src/scout/src/Attributes/SearchableAs.php new file mode 100644 index 0000000000..faeb1385ca --- /dev/null +++ b/src/scout/src/Attributes/SearchableAs.php @@ -0,0 +1,18 @@ +string('scout.prefix') . $this->getTable(); + return config()->string('scout.prefix') . ($this->resolveSearchableAsFromAttribute() ?? $this->getTable()); + } + + /** + * Resolve the index name from the SearchableAs attribute. + */ + public function resolveSearchableAsFromAttribute(): ?string + { + // Unlike upstream, a cached presence check skips reflection for models without the attribute. + if (! ClassMetadataCache::hasClassAttribute(static::class, SearchableAs::class, ascend: true)) { + return null; + } + + $reflection = ClassMetadataCache::reflectClass(static::class); + + do { + $attributes = $reflection->getAttributes(SearchableAs::class); + + if ($attributes !== []) { + return $attributes[0]->newInstance()->index; + } + } while ($reflection = $reflection->getParentClass()); + + return null; } /** diff --git a/tests/Scout/Feature/SearchableModelTest.php b/tests/Scout/Feature/SearchableModelTest.php index 6a4d424203..d54f57a0a2 100644 --- a/tests/Scout/Feature/SearchableModelTest.php +++ b/tests/Scout/Feature/SearchableModelTest.php @@ -5,11 +5,15 @@ namespace Hypervel\Tests\Scout\Feature; use Hypervel\Database\Eloquent\Model; +use Hypervel\Scout\Attributes\SearchableAs; use Hypervel\Scout\Builder; use Hypervel\Scout\EngineManager; use Hypervel\Scout\Engines\Engine; use Hypervel\Scout\Scout; +use Hypervel\Scout\Searchable; use Hypervel\Support\Collection as BaseCollection; +use Hypervel\Tests\Scout\Fixtures\Models\SearchableAsChildModel; +use Hypervel\Tests\Scout\Fixtures\Models\SearchableAsModel; use Hypervel\Tests\Scout\Fixtures\Models\SearchableModel; use Hypervel\Tests\Scout\Fixtures\Models\SoftDeletableSearchableModel; use Hypervel\Tests\Scout\ScoutTestCase; @@ -58,11 +62,15 @@ public function testSearchHonorsBuilderContainerSubstitution(): void $this->assertInstanceOf(ContainerScoutBuilder::class, SearchableModel::search('test')); } - public function testSearchableAsReturnsTableName(): void + public function testSearchableAsDefaultsToTheTableName(): void { - $model = new SearchableModel; + $model = new class extends Model { + use Searchable; + + protected ?string $table = 'posts'; + }; - $this->assertSame('searchable_models', $model->searchableAs()); + $this->assertSame('posts', $model->searchableAs()); } public function testSearchableAsReturnsTableNameWithPrefix(): void @@ -76,6 +84,29 @@ public function testSearchableAsReturnsTableNameWithPrefix(): void $this->assertSame('test_searchable_models', $model->searchableAs()); } + public function testSearchableAsUsesTheSearchableAsAttribute(): void + { + $this->assertSame('posts_index', (new SearchableAsModel)->searchableAs()); + $this->assertSame('posts_index', (new SearchableAsModel)->indexableAs()); + } + + public function testSearchableAsAttributeIsPrefixed(): void + { + config(['scout.prefix' => 'prod_']); + + $this->assertSame('prod_posts_index', (new SearchableAsModel)->searchableAs()); + } + + public function testSearchableAsAttributeIsInherited(): void + { + $this->assertSame('posts_index', (new SearchableAsChildModel)->searchableAs()); + } + + public function testSearchableAsIgnoresAttributesOnTraits(): void + { + $this->assertSame('posts_index', (new SearchableAsTraitChildModel)->searchableAs()); + } + public function testToSearchableArrayReturnsModelArray(): void { $model = SearchableModel::create([ @@ -378,3 +409,13 @@ class CustomScoutBuilder extends Builder class ContainerScoutBuilder extends Builder { } + +#[SearchableAs('trait_index')] +trait SearchableAsTrait +{ +} + +class SearchableAsTraitChildModel extends SearchableAsModel +{ + use SearchableAsTrait; +} diff --git a/tests/Scout/Fixtures/Models/SearchableAsChildModel.php b/tests/Scout/Fixtures/Models/SearchableAsChildModel.php new file mode 100644 index 0000000000..ec0de8037d --- /dev/null +++ b/tests/Scout/Fixtures/Models/SearchableAsChildModel.php @@ -0,0 +1,9 @@ + Date: Fri, 20 Nov 2026 05:04:00 +0000 Subject: [PATCH 09/30] Collapse multi-line Wayfinder arguments that contain parentheses Ports laravel/wayfinder#329 from laravel/wayfinder at 83c56ac1. TypeScript::cleanUp() collapses a generated function's multi-line argument list onto one line. Its pattern stopped at the first closing parenthesis, so an argument type containing parentheses, such as import('@/types').Key<...>, was left spread over several lines. The pattern now matches balanced nested parentheses. Hypervel's deterministic-output replacements around it are unchanged. Tests: ported upstream's TypeScriptCleanUpTest. Validation: the Wayfinder PHP tests and both TypeScript runs (standard and cached routes) pass; formatting and full static analysis pass. --- src/wayfinder/src/TypeScript.php | 2 +- tests/Wayfinder/TypeScriptCleanUpTest.php | 31 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 tests/Wayfinder/TypeScriptCleanUpTest.php diff --git a/src/wayfinder/src/TypeScript.php b/src/wayfinder/src/TypeScript.php index 2fa2dd23df..9bee7ad855 100644 --- a/src/wayfinder/src/TypeScript.php +++ b/src/wayfinder/src/TypeScript.php @@ -128,7 +128,7 @@ public static function cleanUp(string $view): string return str($view) ->pipe(function (Stringable $str) use ($argumentReplacements): Stringable { // Clean up function arguments - $matches = $str->matchAll('/ = \(([^)]+\))/') + $matches = $str->matchAll('/ = (\((?:[^()]++|(?1))*\))/') ->concat($str->matchAll('/\.url\(\s*args,\s+\{/')) ->concat($str->matchAll('/\.url\(\s*args,\s+options\s*\)/')) ->concat($str->matchAll('/\.url\(\s*options\s*\)/')) diff --git a/tests/Wayfinder/TypeScriptCleanUpTest.php b/tests/Wayfinder/TypeScriptCleanUpTest.php new file mode 100644 index 0000000000..e16f6e63ac --- /dev/null +++ b/tests/Wayfinder/TypeScriptCleanUpTest.php @@ -0,0 +1,31 @@ + } | [project: string],", + ' options?: RouteQueryOptions,', + "): RouteDefinition<'get'> => ({", + ' url: show.url(args, options),', + " method: 'get',", + '})', + ]); + + $this->assertSame(implode(PHP_EOL, [ + "export const show = (args: { project: import('@/types').Key<'Project', string> } | [project: string], options?: RouteQueryOptions,): RouteDefinition<'get'> => ({", + ' url: show.url(args, options),', + " method: 'get',", + '})', + ]), TypeScript::cleanUp($view)); + } +} From decd604321e101b4d98517a481f9f7665cc247ca Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:04:05 +0000 Subject: [PATCH 10/30] Test QUERY on Wayfinder redirect routes registered for every verb Ports laravel/wayfinder#330 from laravel/wayfinder at 83c56ac1. Route::redirect() registers every router verb, including QUERY. Upstream added coverage after the generated definition for such a route failed to compile. Hypervel already generates the query method for these routes; this adds the coverage. Tests: added upstream's photos.redirect fixture route and the redirect route case in QueryMethod.test.ts, which calls the generated helper for each registered verb. Validation: both TypeScript runs (standard and cached routes) and the Wayfinder type check pass. --- src/wayfinder/tests/QueryMethod.test.ts | 12 ++++++++++++ tests/Wayfinder/Fixtures/routes.php | 2 ++ 2 files changed, 14 insertions(+) diff --git a/src/wayfinder/tests/QueryMethod.test.ts b/src/wayfinder/tests/QueryMethod.test.ts index e5d7000893..114cd7a9f6 100644 --- a/src/wayfinder/tests/QueryMethod.test.ts +++ b/src/wayfinder/tests/QueryMethod.test.ts @@ -4,6 +4,7 @@ import { show, } from "./.generated/actions/Hypervel/Tests/Wayfinder/Fixtures/Controllers/SearchController"; import search from "./.generated/routes/search"; +import photos from "./.generated/routes/photos"; describe("query only route", () => { test("properties", () => { @@ -105,3 +106,14 @@ describe("get and query route", () => { }); }); }); + +describe("redirect route", () => { + test("every registered verb", () => { + for (const method of photos.redirect.definition.methods) { + expect(photos.redirect[method]()).toEqual({ + url: "/old-photos", + method, + }); + } + }); +}); diff --git a/tests/Wayfinder/Fixtures/routes.php b/tests/Wayfinder/Fixtures/routes.php index 858d430a2d..cdbdbff9b8 100644 --- a/tests/Wayfinder/Fixtures/routes.php +++ b/tests/Wayfinder/Fixtures/routes.php @@ -110,6 +110,8 @@ Route::get('/albums/recent', fn (): string => 'ok')->name('albums.index.recent'); +Route::redirect('/old-photos', '/photos')->name('photos.redirect'); + Route::get('/prism/chat', [PrismChatController::class, 'index']); Route::get('/two-routes-one-action-1', [TwoRoutesSameActionController::class, 'same']); From 823973fce78448ca2ef597c41b2ba1899a6e427d Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:04:28 +0000 Subject: [PATCH 11/30] Reconcile head with upstream event, Open Graph and container changes Ports laravel/head#15, #16 and #17 and reconciles #14, from laravel/head at fc84fecc. They share one commit because the head docs and HeadManager each carry changes from more than one of them. #15: Hypervel had ported the event schema helpers from the pull request's draft. Event::location() now takes only a schema object, as merged upstream: Google requires an address for a physical location, so the draft's conversion of a string to a bare Place is gone. Online events pass a VirtualLocation. #16: Open Graph media from several defaults() calls was replaced by the last call instead of combined. Default media now merges with other default media and is still replaced by media a route, page or error defines. Adaptation: runtime defaults are merged over the registered defaults before being marked as defaults, so runtime default media still replaces registered default media, as the runtime defaults docs promise. #17: ported the protected currentContainer() extension point; state() and request() resolve through it. Adaptation: it returns the injected container instead of Container::getInstance(). Upstream's reason, Octane cloning the application per request, doesn't apply: the worker keeps one container, CurrentHead is scoped per coroutine and the request comes from the coroutine's context. #14: Hypervel already fell back to the canonical URL for og:url, so the README no longer lists it as a difference. Tests: ported upstream's virtual location and multiple defaults() calls cases, removed the draft's string location test, and extended the runtime defaults media test. Upstream's container clone test is recorded as removed; the coroutine isolation tests cover per-request state. Validation: the head suite passes; formatting and full static analysis pass. --- src/docs/head.md | 4 ++-- src/head/README.md | 2 -- src/head/src/HeadManager.php | 17 ++++++++++++++--- src/head/src/Schema/Event.php | 8 +++++--- src/head/src/Tags/OpenGraph.php | 7 ++++--- tests/Head/Feature/HeadManagerTest.php | 3 +++ tests/Head/Feature/Schema/EventSchemaTest.php | 6 +++--- tests/Head/Feature/Tags/OpenGraphTest.php | 13 +++++++++++++ 8 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/docs/head.md b/src/docs/head.md index 22166b1d45..75b0dddcf8 100644 --- a/src/docs/head.md +++ b/src/docs/head.md @@ -279,7 +279,7 @@ Head::ogImage('/images/cover.jpg', alt: 'Draft cover') ``` -Open Graph media inherited from your defaults acts as a fallback. When route, runtime, or error metadata defines its own media of the same type, the default media is replaced instead of merged, so a page's `og:image` takes precedence over a site-wide default image. +Open Graph media inherited from your defaults acts as a fallback. When route, runtime, or error metadata defines its own media of the same type, the default media is replaced instead of merged, so a page's `og:image` takes precedence over a site-wide default image. Media added across several `defaults` calls is combined. You may fluently define conditional metadata using the `when` and `unless` methods: @@ -641,7 +641,7 @@ Head::schema( ); ``` -Passing a string to `location` creates a `Place` with that name. You may also pass another schema object, such as `Schema::virtualLocation()->url($event->stream_url)`. The `performer` method accepts a person, an organization, or a list of those builders, and `image` accepts a URL or a list of URLs. +Google requires an address for a physical location, so `location` accepts a `Place` rather than a venue name. For online events, pass another schema object, such as `Schema::virtualLocation()->url($event->stream_url)`. The `performer` method accepts a person, an organization, or a list of those builders, and `image` accepts a URL or a list of URLs. For raw values or shapes outside the typed helpers, every schema builder provides a `set` method. For example, `$eventSchema->set('location', [$place, $virtualLocation])` defines multiple locations, and `$eventSchema->set('location', 'Venue')` preserves a plain text location. diff --git a/src/head/README.md b/src/head/README.md index 565530d2e4..6e4ccb69b1 100644 --- a/src/head/README.md +++ b/src/head/README.md @@ -7,8 +7,6 @@ Documentation: https://hypervel.org/docs/head Head configuration is shared for the worker lifetime. Register defaults and error pages during boot; use the runtime resolvers for request-specific values. See [runtime defaults](https://hypervel.org/docs/head#runtime-defaults). -An unset `og:url` falls back to the canonical URL when a page defines Open Graph metadata; explicit Open Graph URLs take precedence. See [Open Graph](https://hypervel.org/docs/head#open-graph). - Livewire integration is not provided. Use Blade or Inertia for page rendering. Ported from: https://github.com/laravel/head diff --git a/src/head/src/HeadManager.php b/src/head/src/HeadManager.php index ed5cc50b45..d03972e6ff 100644 --- a/src/head/src/HeadManager.php +++ b/src/head/src/HeadManager.php @@ -293,7 +293,8 @@ protected function resolve(?int $status = null): HeadData $data = (new HeadData)->merge($this->defaults); if ($this->defaultsResolver !== null) { - $data = $data->merge($this->define($this->defaultsResolver)->asDefaults()); + // Marked after merging, so runtime default media replaces the registered default media instead of joining it. + $data = $data->merge($this->define($this->defaultsResolver))->asDefaults(); } if ($route = $this->route()) { @@ -361,7 +362,7 @@ protected function schemaFactory(): SchemaFactory */ protected function state(): CurrentHead { - return $this->app->make(CurrentHead::class); + return $this->currentContainer()->make(CurrentHead::class); } /** @@ -369,7 +370,17 @@ protected function state(): CurrentHead */ protected function request(): ?Request { - return $this->app->make('request'); + return $this->currentContainer()->make('request'); + } + + /** + * Get the container handling the current request. + */ + protected function currentContainer(): Container + { + // Unlike upstream, this is the injected container rather than Container::getInstance(): the worker keeps one + // container, CurrentHead is scoped per coroutine and the request comes from the coroutine's context. + return $this->app; } /** diff --git a/src/head/src/Schema/Event.php b/src/head/src/Schema/Event.php index 40b02a548e..d0f54f8728 100644 --- a/src/head/src/Schema/Event.php +++ b/src/head/src/Schema/Event.php @@ -63,11 +63,13 @@ public function image(string|array $image): static } /** - * Set the event location. A string is treated as the name of a physical place. + * Set the event location. + * + * Google requires an address on a physical location, so this takes a Place (or a VirtualLocation) rather than a bare venue name. */ - public function location(SchemaObject|string $location): static + public function location(SchemaObject $location): static { - return $this->set('location', is_string($location) ? (new Place)->name($location) : $location); + return $this->set('location', $location); } /** diff --git a/src/head/src/Tags/OpenGraph.php b/src/head/src/Tags/OpenGraph.php index 468964e761..1f251f5e45 100644 --- a/src/head/src/Tags/OpenGraph.php +++ b/src/head/src/Tags/OpenGraph.php @@ -262,11 +262,12 @@ public function overlayOn(?TagBuilder $base): static return $this; } + // Defaults defined across several calls merge; media a higher layer defines replaces them. $overlaid = new static( array_replace($base->properties, $this->properties), - $this->overlayMedia($base->images, $this->images, $base->imagesAreDefaults), - $this->overlayMedia($base->videos, $this->videos, $base->videosAreDefaults), - $this->overlayMedia($base->audios, $this->audios, $base->audiosAreDefaults), + $this->overlayMedia($base->images, $this->images, $base->imagesAreDefaults && ! $this->imagesAreDefaults), + $this->overlayMedia($base->videos, $this->videos, $base->videosAreDefaults && ! $this->videosAreDefaults), + $this->overlayMedia($base->audios, $this->audios, $base->audiosAreDefaults && ! $this->audiosAreDefaults), ); // A page image must not turn inherited default video or audio into page media. diff --git a/tests/Head/Feature/HeadManagerTest.php b/tests/Head/Feature/HeadManagerTest.php index 4b6ba71430..470f53ba9b 100644 --- a/tests/Head/Feature/HeadManagerTest.php +++ b/tests/Head/Feature/HeadManagerTest.php @@ -20,6 +20,9 @@ public function testRendersTheHeadAsAnHtmlString(): void $this->assertInstanceOf(HtmlString::class, Head::render()); } + // REMOVED: upstream's "keeps request head data in the container handling the request", which clones the container + // per request as Octane does. CurrentHead is scoped per coroutine instead; CoroutineIsolationTest covers its isolation. + public function testDefinitionsStayEagerAndRuntimeDefaultsAreLazyAndFresh(): void { $eagerCalls = 0; diff --git a/tests/Head/Feature/Schema/EventSchemaTest.php b/tests/Head/Feature/Schema/EventSchemaTest.php index 0cc028ad42..ce29fb12e5 100644 --- a/tests/Head/Feature/Schema/EventSchemaTest.php +++ b/tests/Head/Feature/Schema/EventSchemaTest.php @@ -37,11 +37,11 @@ public function testRendersAnEventWithThePropertiesRequiredForRichResults(): voi $this->assertStringContainsString('"location":{"@type":"Place","name":"Music City Center","address":"201 Rep. John Lewis Way S, Nashville, TN"}', $html); } - public function testTreatsAStringLocationAsTheNameOfAPlace(): void + public function testAcceptsAVirtualLocationForOnlineEvents(): void { - $event = Schema::event()->location('Music City Center'); + $event = Schema::event()->location(Schema::virtualLocation()->url('https://example.com/stream')); - $this->assertSame(['@type' => 'Place', 'name' => 'Music City Center'], $event->toArray()['location']); + $this->assertSame(['@type' => 'VirtualLocation', 'url' => 'https://example.com/stream'], $event->toArray()['location']); } public function testSetsEventStatusAndAttendanceModeFromSchemaOrgEnumValues(): void diff --git a/tests/Head/Feature/Tags/OpenGraphTest.php b/tests/Head/Feature/Tags/OpenGraphTest.php index e35c12e03a..04958bd349 100644 --- a/tests/Head/Feature/Tags/OpenGraphTest.php +++ b/tests/Head/Feature/Tags/OpenGraphTest.php @@ -148,6 +148,16 @@ public function testReplacesDefaultLayerOgImagesWithPageLevelOgImagesWhenTheDefa $this->assertStringNotContainsString('https://example.com/brand.png', $html); } + public function testMergesOgImagesFromSeveralDefaultsCalls(): void + { + Head::defaults(fn (HeadBuilder $head): HeadBuilder => $head->ogImage('https://example.com/first.png')); + Head::defaults(fn (HeadBuilder $head): HeadBuilder => $head->ogImage('https://example.com/second.png')); + + $html = Head::toHtml(); + $this->assertStringContainsString('property="og:image" content="https://example.com/first.png">', $html); + $this->assertStringContainsString('property="og:image" content="https://example.com/second.png">', $html); + } + public function testReplacesDefaultMediaIndependentlyAcrossRouteAndPageLayers(): void { Head::defaults(fn (HeadBuilder $head): HeadBuilder => $head @@ -172,6 +182,9 @@ public function testRepeatedAndRuntimeDefaultsRetainFallbackMediaProvenance(): v Head::defaults(fn (HeadBuilder $head): HeadBuilder => $head->ogImage('/first.png')->ogAudio('/audio.mp3')); Head::defaults(fn (HeadBuilder $head): HeadBuilder => $head->ogImage('/second.png')->ogVideo('/video.mp4')); Head::resolveDefaultsUsing(fn (HeadBuilder $head): HeadBuilder => $head->ogImage('/runtime.png')); + + $this->assertSame([['url' => '/runtime.png']], Head::toArray()['openGraph']['images']); + Head::ogImage('/page.png'); $data = Head::toArray()['openGraph']; From 41058290ec9ec0a8640b18af0ae02811d47148bf Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 12/30] Clarify Mailtrap Email Sandbox and Markdown escaping in mail docs Ports laravel/docs#11431 and #11456 from laravel/docs 13.x at eabbe579. They share one commit because both change mail.md. #11431: the local development section names Mailtrap Email Sandbox and links to its current page. Hypervel's Mailpit sentence is unchanged. #11456: adds upstream's warning that Blade echoes escape HTML but not Markdown, so user input can inject Markdown into a Markdown mailable, and that Markdown::withSecuredEncoding() escapes it. Secured encoding escapes brackets, angle brackets and backslashes. Docs only; upstream changed no tests. --- src/docs/mail.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/docs/mail.md b/src/docs/mail.md index a53585113b..1cf594a0b6 100644 --- a/src/docs/mail.md +++ b/src/docs/mail.md @@ -952,6 +952,9 @@ Thanks,
> [!NOTE] > Do not use excess indentation when writing Markdown emails. Per Markdown standards, Markdown parsers will render indented content as code blocks. +> [!WARNING] +> Blade's `{{ }}` echo statements escape HTML but not Markdown, so Markdown syntax within user-supplied data, such as links and images, will be rendered in the email. To escape link and image syntax, invoke `Hypervel\Mail\Markdown::withSecuredEncoding()` in the `boot` method of your `AppServiceProvider`. + #### Button Component @@ -1614,9 +1617,9 @@ The `array` mail driver keeps sent messages in memory for the current execution. Instead of sending your emails, the `log` mail driver will write all email messages to your log files for inspection. Typically, this driver would only be used during local development. For more information on configuring your application per environment, check out the [configuration documentation](/docs/{{version}}/configuration#environment-configuration). -#### HELO / Mailtrap / Mailpit +#### HELO / Mailtrap Email Sandbox / Mailpit -Alternatively, you may use a service like [HELO](https://usehelo.com) or [Mailtrap](https://mailtrap.io) and the `smtp` driver to send your email messages to a "dummy" mailbox where you may view them in a true email client. This approach has the benefit of allowing you to actually inspect the final emails in Mailtrap's message viewer. +Alternatively, you may use a service like [HELO](https://usehelo.com) or [Mailtrap Email Sandbox](https://mailtrap.io/email-sandbox/) and the `smtp` driver to send your email messages to a "dummy" mailbox where you may view them in a true email client. This approach has the benefit of allowing you to actually inspect the final emails in Mailtrap Email Sandbox's message viewer. You may also use [Mailpit](https://github.com/axllent/mailpit) to preview your messages locally. By default, Mailpit's web interface is available at `http://localhost:8025`. From 62310289535b16517b3a3254c0d9a7b6610b5213 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 13/30] Document Eloquent after query callbacks Ports laravel/docs#11434 from laravel/docs 13.x at eabbe579. Adds the After Query Callbacks section and its contents entry after Pending Attributes, as upstream. Adaptation: upstream's local scope example sets an excerpt property on each post, which marks the model dirty so a later save tries to write an excerpt column. Eloquent's pluck() also passes the plucked values to the callback, so a scope whose callback expects models fails there. The example now hides an attribute with makeHidden() on a query that ends in get(), and the local scope sentence is dropped. Docs only; upstream changed no tests. --- src/docs/eloquent.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/docs/eloquent.md b/src/docs/eloquent.md index 7b9224f62f..0ae76a4aa9 100644 --- a/src/docs/eloquent.md +++ b/src/docs/eloquent.md @@ -35,6 +35,7 @@ - [Local Scopes](#local-scopes) - [Applying Scope Callbacks](#applying-scope-callbacks) - [Pending Attributes](#pending-attributes) + - [After Query Callbacks](#after-query-callbacks) - [Custom Eloquent Builders](#custom-eloquent-builders) - [Comparing Models](#comparing-models) - [Events](#events) @@ -1886,6 +1887,23 @@ $query->withAttributes([ ], asConditions: false); ``` + +### After Query Callbacks + +The `afterQuery` method allows you to register a closure that will be invoked with the results of the query after it has been executed: + +```php +use App\Models\Post; +use Hypervel\Database\Eloquent\Collection; + +$posts = Post::where('published', true) + ->afterQuery(function (Collection $posts) { + // Hide the full post body when the posts are serialized... + $posts->makeHidden('body'); + }) + ->get(); +``` + ### Custom Eloquent Builders From ef2a0fa56f01831244fb1e7e4b3f5152c9678964 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 14/30] Document sequence ordering with inOrderOf() Ports laravel/docs#11435 from laravel/docs 13.x at eabbe579. Hypervel's Preferred Value Ordering section is replaced by upstream's Sequence Ordering section and enum example, at upstream's position after latest and oldest. Kept Hypervel's note that values outside the sequence sort last, which the grammar guarantees. No docs linked to the old anchor. Docs only; upstream changed no tests. --- src/docs/queries.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/docs/queries.md b/src/docs/queries.md index fa0bc9d6d9..d222f01e14 100644 --- a/src/docs/queries.md +++ b/src/docs/queries.md @@ -1521,17 +1521,6 @@ $corporations = DB::table('corporations') ->get(); ``` - -#### Preferred Value Ordering - -The `inOrderOf` method orders results according to a preferred sequence of values. Values not included in the sequence are placed last: - -```php -$orders = DB::table('orders') - ->inOrderOf('status', ['pending', 'processing', 'completed']) - ->get(); -``` - #### The `latest` and `oldest` Methods @@ -1543,6 +1532,19 @@ $user = DB::table('users') ->first(); ``` + +#### Sequence Ordering + +The `inOrderOf` method allows you to order query results using a sequence of values, such as strings or enums: + +```php +$orders = DB::table('orders') + ->inOrderOf('status', [OrderStatus::Shipped, OrderStatus::Delivered, OrderStatus::Cancelled]) + ->get(); +``` + +Values not included in the sequence are placed last. + #### Random Ordering From de7a8ea99637f8ff2b8af59e2b05a68ee5329eaf Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 15/30] Note that Fortify throttles its login route automatically Ports laravel/docs#11438 from laravel/docs 13.x at eabbe579. Hypervel's Fortify docs are structured differently, so upstream's sentence goes in the Rate Limiting section: when a login rate limiter is configured, Fortify adds the matching throttle middleware to its login route. Docs only; upstream changed no tests. --- src/docs/fortify.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/docs/fortify.md b/src/docs/fortify.md index 05e0f262fd..01c6211cd4 100644 --- a/src/docs/fortify.md +++ b/src/docs/fortify.md @@ -306,6 +306,8 @@ The package defaults leave `fortify.limiters.login` and `fortify.limiters.passke The published configuration sets the login and passkey limiters to `login` and `passkeys`, and the published `App\Providers\FortifyServiceProvider` registers matching named rate limiters. +When a login rate limiter is configured, Fortify automatically applies the corresponding `throttle` middleware to its login route, so you do not need to attach this middleware manually. + Fortify's `two-factor` limiter allows five attempts per minute for each challenged account and guard. Changing the request's IP address does not reset that account's limit, and users behind the same IP address do not share a limit. If the challenged account is unexpectedly missing from the login session, the limiter falls back to that session's ID. You may set `fortify.limiters.two-factor` to a different throttle string or named limiter. Registering another limiter named `two-factor` from your application provider replaces Fortify's default. From eb9835cc9d65c8096396c955f67043bb8c3c31b9 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 16/30] Correct what a streamed parent response includes from its sub-agents Ports laravel/docs#11445 from laravel/docs 13.x at eabbe579. A sub-agent's final text is the tool call's result in the parent response's toolResults; the parent response's text, reasoning, citations and usage only include the parent's own output. Adaptation: upstream says to track sub-agent usage through AgentStreamed. A sub-agent with structured output doesn't stream, even when its parent does: AgentTool runs it through prompt(), which dispatches AgentPrompted. The docs name both events; each carries the parent's invocation ID as the prompt's parentInvocationId. Docs only; upstream changed no tests. --- src/docs/ai-sdk.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/ai-sdk.md b/src/docs/ai-sdk.md index 3ecea9f1e7..c9646f11f1 100644 --- a/src/docs/ai-sdk.md +++ b/src/docs/ai-sdk.md @@ -2101,7 +2101,7 @@ foreach ($stream as $event) { } ``` -Response values such as `text`, `usage`, and `toolResults` ignore preliminary events. The [Vercel protocol](#stream-protocols) renders them as native streaming tool output, so `useChat` displays the progress without any custom code, while the AG-UI protocol reports them as activity snapshots. The sub-agent's final text becomes a tool result for the parent agent. The parent's response usage counts only its own generation steps. +Response values such as `text`, `usage`, and `toolResults` ignore preliminary events. The [Vercel protocol](#stream-protocols) renders them as native streaming tool output, so `useChat` displays the progress without any custom code, while the AG-UI protocol reports them as activity snapshots. The sub-agent's final text becomes the tool call's result, which is available through the completed response's `toolResults`. The completed response's `text`, `reasoning`, `citations`, and `usage` only include the parent agent's own output. To track the usage of a sub-agent, you may listen for the `AgentStreamed` [event](#events), or the `AgentPrompted` event for a sub-agent that produces structured output, and compare the event's `prompt->parentInvocationId` with the parent response's `invocationId`. ### Middleware From 4255d69b209107ab7643dede670aa49cfec19a9c Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 17/30] Link MCP security practices and document structured content assertions Ports laravel/docs#11421 and the mcp.md part of direct commit 4cfebfe805 from laravel/docs 13.x at eabbe579. They share one commit because both change mcp.md. #11421: links the MCP Security Best Practices guide after the authorization section. 4cfebfe805: Hypervel's structured content assertion paragraph is replaced by upstream's section and assertion table, after assertSee. All listed assertions exist with matching signatures. Kept Hypervel's sentence that assertStructuredContent() also accepts an AssertableJson closure. Docs only; upstream changed no tests. --- src/docs/mcp.md | 61 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/src/docs/mcp.md b/src/docs/mcp.md index de621e9b0f..7ff8022d56 100644 --- a/src/docs/mcp.md +++ b/src/docs/mcp.md @@ -2001,6 +2001,8 @@ public function handle(Request $request): Response } ``` +For guidance on securing MCP clients and servers, see the MCP [Security Best Practices](https://modelcontextprotocol.io/docs/2026-07-28/tutorials/security/security_best_practices). + ## MCP Client @@ -2489,6 +2491,53 @@ You may assert that a response contains specific text using the `assertSee` meth $response->assertSee('The current weather in New York City is 72°F and sunny.'); ``` +For tools that return [structured content](#structured-responses), you may assert counts, values at specific paths, and the structure of the response: + +```php +$response + ->assertStructuredContentCount(2, 'data') + ->assertStructuredContentPath('data.0.location', 'New York City') + ->assertStructuredContentPaths([ + 'data.0.temperature' => 72, + 'data.1.temperature' => 68, + ]) + ->assertStructuredContentStructure([ + 'data' => [ + '*' => ['location', 'temperature'], + ], + ]); +``` + +Paths support "dot" notation. The `assertStructuredContentPath` method also accepts a closure to determine if the value passes a given truth test: + +```php +$response->assertStructuredContentPath('data.0.temperature', fn (int $temperature): bool => $temperature > 70); +``` + +The `assertStructuredContent` method accepts the expected structured content, or a closure that receives an `AssertableJson` instance for fluent assertions. + +The following structured content assertions are available: + +| Method | Description | +| --- | --- | +| `assertExactStructuredContent($data)` | Assert that the structured content matches the given array exactly. | +| `assertSimilarStructuredContent($data)` | Assert that the structured content matches the given array, ignoring order. | +| `assertStructuredContentCount($count, $key = null)` | Assert the number of items, optionally at a given key. | +| `assertStructuredContentPath($path, $expect)` | Assert the value at a given path. | +| `assertStructuredContentPaths($paths)` | Assert multiple path / value pairs. | +| `assertStructuredContentPathCanonicalizing($path, $expect)` | Assert the values at a given path, ignoring order. | +| `assertStructuredContentPathsCanonicalizing($paths)` | Assert multiple path / value pairs, ignoring order. | +| `assertStructuredContentFragment($data)` | Assert that a fragment appears anywhere in the structured content. | +| `assertStructuredContentFragments($data)` | Assert that each given fragment appears in the structured content. | +| `assertStructuredContentMissing($data)` | Assert that a fragment does not appear in the structured content. | +| `assertStructuredContentMissingExact($data)` | Assert that an exact fragment does not appear in the structured content. | +| `assertStructuredContentMissingPath($path)` | Assert that a path does not exist. | +| `assertStructuredContentMissingPaths($paths)` | Assert that multiple paths do not exist. | +| `assertStructuredContentStructure($structure)` | Assert that the given keys exist, allowing additional keys. | +| `assertExactStructuredContentStructure($structure)` | Assert that only the given keys exist. | +| `assertStructuredContentIsArray($key = null)` | Assert that the structured content, optionally at a given key, is a JSON array. | +| `assertStructuredContentIsObject($key = null)` | Assert that the structured content, optionally at a given key, is a JSON object. | + You may assert that a response contains an error using the `assertHasErrors` method: ```php @@ -2513,18 +2562,6 @@ $response->assertTitle('Current Weather Tool'); $response->assertDescription('Fetches the current weather forecast for a specified location.'); ``` -You may assert a tool's structured content using the `assertStructuredContent` method, which accepts the expected array or a closure that receives an `AssertableJson` instance. Structured content equivalents of the HTTP test response's JSON assertions are also available, such as `assertStructuredContentCount`, `assertStructuredContentPath`, `assertExactStructuredContent`, `assertStructuredContentFragment`, `assertStructuredContentMissingPath`, and `assertStructuredContentStructure`: - -```php -$response - ->assertStructuredContentCount(2, 'data') - ->assertStructuredContentPath('data.0.id', 123) - ->assertStructuredContentStructure([ - 'data' => ['*' => ['id', 'status']], - 'meta' => ['total'], - ]); -``` - You may assert that notifications were sent using the `assertSentNotification` and `assertNotificationCount` methods: ```php From c7cfc1ab9c209efc5bec9acf370bc7d7ff1eb77c Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 18/30] Point deployment requirements at the installation requirements Ports laravel/docs#11447 from laravel/docs 13.x at eabbe579. Adds upstream's PHP requirements subsection and php -v check to the deployment docs. Adaptation: it links to the installation requirements instead of keeping a second list. The deployment list had drifted: it gave Swoole 6.3.0 rather than the required 6.3.0RC2 and listed phpredis without its condition. The installation section also covers Swoole's build options and the use_shortname setting, which production servers need. The installation list lacked extensions that hypervel/framework requires through its packages and dependencies: Ctype (session), DOM (testing), Sockets (engine), SimpleXML (the AWS SDK), iconv (Symfony's mbstring polyfill), libxml (the CSS inliner used by mail) and PCRE. Composer refuses to install without them. Docs only; upstream changed no tests. --- src/docs/deployment.md | 37 +++++++++++-------------------------- src/docs/installation.md | 7 +++++++ 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/docs/deployment.md b/src/docs/deployment.md index 94bf0ddc50..538a3ee379 100644 --- a/src/docs/deployment.md +++ b/src/docs/deployment.md @@ -2,6 +2,7 @@ - [Introduction](#introduction) - [Server Requirements](#server-requirements) + - [Hypervel PHP Requirements](#hypervel-php-requirements) - [Server Configuration](#server-configuration) - [Nginx](#nginx) - [Nginx and WebSockets](#nginx-and-websockets) @@ -27,32 +28,16 @@ When you're ready to deploy your Hypervel application to production, there are s ## Server Requirements -The Hypervel framework has a few system requirements. Hypervel ships with its own Swoole-based application server, so your production server should run the long-running Hypervel server process instead of PHP-FPM. You should ensure that your server has the following minimum PHP version and commonly required extensions: - -
- -- PHP >= 8.4 -- Ctype PHP Extension -- cURL PHP Extension -- DOM PHP Extension -- Fileinfo PHP Extension -- Filter PHP Extension -- Hash PHP Extension -- Intl PHP Extension -- JSON PHP Extension -- Mbstring PHP Extension -- OpenSSL PHP Extension -- PCNTL PHP Extension -- PCRE PHP Extension -- PDO PHP Extension -- POSIX PHP Extension -- Redis PHP Extension >= 6.1 -- Session PHP Extension -- Swoole PHP Extension >= 6.3.0 -- Tokenizer PHP Extension -- XML PHP Extension - -
+Hypervel ships with its own Swoole-based application server, so your production server should run the long-running Hypervel server process instead of PHP-FPM. + + +### Hypervel PHP Requirements + +Your server must meet Hypervel's [installation requirements](/docs/{{version}}/installation#requirements), including the PHP version, the required PHP extensions, Swoole's build options, and the `swoole.use_shortname` setting. You can check the PHP version installed on your server with: + +```shell +php -v +``` ## Server Configuration diff --git a/src/docs/installation.md b/src/docs/installation.md index 372a25abd5..b7d3309f5a 100644 --- a/src/docs/installation.md +++ b/src/docs/installation.md @@ -31,18 +31,25 @@ The Hypervel framework has a few system requirements: - PHP >= 8.4 - Composer +- Ctype PHP Extension - cURL PHP Extension +- DOM PHP Extension - Fileinfo PHP Extension - Filter PHP Extension - Hash PHP Extension +- iconv PHP Extension - Intl PHP Extension - JSON PHP Extension +- libxml PHP Extension - Mbstring PHP Extension - OpenSSL PHP Extension - PCNTL PHP Extension +- PCRE PHP Extension - PDO PHP Extension - POSIX PHP Extension - Session PHP Extension +- SimpleXML PHP Extension +- Sockets PHP Extension - Swoole PHP Extension >= 6.3.0RC2 - Tokenizer PHP Extension From f55693dc9ee3d01faeb52b59abb1937891be9821 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 19/30] Document HtmlString for unescaped Blade output Ports laravel/docs#11450 from laravel/docs 13.x at eabbe579. Adds upstream's HtmlString paragraph and examples with Hypervel namespaces; e() returns an Htmlable's HTML unescaped. Docs only; upstream changed no tests. --- src/docs/blade.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/docs/blade.md b/src/docs/blade.md index ad6f622ce2..df9c01e2d1 100644 --- a/src/docs/blade.md +++ b/src/docs/blade.md @@ -118,6 +118,18 @@ By default, Blade `{{ }}` statements are automatically sent through PHP's `htmls Hello, {!! $name !!}. ``` +Alternatively, you may wrap a string in an instance of `Hypervel\Support\HtmlString`. Blade will not escape these instances, even when displayed using the `{{ }}` statement: + +```php +use Hypervel\Support\HtmlString; + +$name = new HtmlString('Taylor'); +``` + +```blade +Hello, {{ $name }}. +``` + > [!WARNING] > Be very careful when echoing content that is supplied by users of your application. You should typically use the escaped, double curly brace syntax to prevent XSS attacks when displaying user supplied data. From 1818302e72830f6f81390aa0dcdd40fdd28eada1 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 20/30] Document insertBefore and insertAfter middleware methods Ports laravel/docs#11454 from laravel/docs 13.x at eabbe579. Hypervel's own wording for inserting global and group middleware is replaced by upstream's, including the array argument note and the warning. The example's TrimStrings and TrustProxies are in Hypervel's default global stack. Docs only; upstream changed no tests. --- src/docs/middleware.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/docs/middleware.md b/src/docs/middleware.md index 6720ce8eb4..3995fbc584 100644 --- a/src/docs/middleware.md +++ b/src/docs/middleware.md @@ -131,15 +131,26 @@ use Hypervel\Foundation\Configuration\Middleware; The `$middleware` object provided to the `withMiddleware` closure is an instance of `Hypervel\Foundation\Configuration\Middleware` and is responsible for managing the middleware assigned to your application's routes. The `append` method adds the middleware to the end of the list of global middleware. If you would like to add a middleware to the beginning of the list, you should use the `prepend` method. -To place a middleware next to an existing global middleware, you may use the `insertBefore` and `insertAfter` methods: +If a middleware needs to run at a specific point in Hypervel's default global middleware stack, you may use the `insertBefore` and `insertAfter` methods. Both methods accept the middleware already in the stack that you want to position against, followed by the middleware you want to insert: ```php -use App\Http\Middleware\EnsureTokenIsValid; +use App\Http\Middleware\SanitizeInput; +use App\Http\Middleware\StripCrossOriginReferer; +use Hypervel\Foundation\Http\Middleware\TrimStrings; use Hypervel\Http\Middleware\TrustProxies; -$middleware->insertAfter(TrustProxies::class, EnsureTokenIsValid::class); +->withMiddleware(function (Middleware $middleware): void { + $middleware->insertAfter(TrustProxies::class, StripCrossOriginReferer::class); + + $middleware->insertBefore(TrimStrings::class, SanitizeInput::class); +}) ``` +You may also pass an array of middleware as the second argument to either method. + +> [!WARNING] +> If the middleware you are positioning against is not part of the global middleware stack, nothing will be inserted. + #### Preferring JSON Responses @@ -344,7 +355,7 @@ $middleware->web(remove: [ ]); ``` -The `insertBeforeInGroup` and `insertAfterInGroup` methods place middleware next to an existing entry in a group: +If you need a middleware to run at a particular position within a group, you may use the `insertBeforeInGroup` and `insertAfterInGroup` methods: ```php use App\Http\Middleware\EnsureUserIsSubscribed; From 26516b4227e0cd4d8d18556eb30729e37a6fb3a1 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:21:57 +0000 Subject: [PATCH 21/30] Fix bearerToken() return value wording Ports laravel/docs#11458 from laravel/docs 13.x at eabbe579. bearerToken() returns null when no bearer token is present, including when the Authorization header uses another scheme. Docs only; upstream changed no tests. --- src/docs/requests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/requests.md b/src/docs/requests.md index b57fe1b6cf..9fc77c257d 100644 --- a/src/docs/requests.md +++ b/src/docs/requests.md @@ -228,7 +228,7 @@ if ($request->hasHeader('X-Header-Name')) { } ``` -For convenience, the `bearerToken` method may be used to retrieve a bearer token from the `Authorization` header. If no such header is present, `null` will be returned: +For convenience, the `bearerToken` method may be used to retrieve a bearer token from the `Authorization` header. If no token is present, `null` will be returned: ```php $token = $request->bearerToken(); From ee742b357ac2ac4758cd785043168f7a25b95eb3 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:35:34 +0000 Subject: [PATCH 22/30] Fail tool calls when a sub-agent's validation fails Validation failures from a tool are returned to the model so it can correct its arguments and retry. A sub-agent's ValidationException comes from its own run, not from the arguments the model passed, so the prompt path returned the sub-agent's validation messages to the model as a normal tool result and never reported the failure. Agent tools now fail the tool call on a validation failure, as the streamed path already did: the model sees a failed result, the message stays hidden outside debug mode, and ToolFailed carries the original exception. Other tools still get their validation messages back. Covered in ToolFailureRecoveryTest. --- src/ai/src/Gateway/Concerns/InvokesTools.php | 22 ++++++++++-------- tests/Ai/Feature/ToolFailureRecoveryTest.php | 24 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/ai/src/Gateway/Concerns/InvokesTools.php b/src/ai/src/Gateway/Concerns/InvokesTools.php index 3566b13a56..a8e1754a6c 100644 --- a/src/ai/src/Gateway/Concerns/InvokesTools.php +++ b/src/ai/src/Gateway/Concerns/InvokesTools.php @@ -9,6 +9,7 @@ use Hypervel\Ai\Gateway\ParentInvocation; use Hypervel\Ai\Gateway\RunContext; use Hypervel\Ai\Providers\Tools\ToolSearch; +use Hypervel\Ai\Tools\AgentTool; use Hypervel\Ai\Tools\Request; use Hypervel\Ai\Tools\ToolNameResolver; use Hypervel\Coroutine\Coroutine; @@ -38,16 +39,19 @@ protected function executeTool(Tool $tool, array $arguments, ?string $toolCallId // Only the handler itself may fail the tool call, so a listener that throws is never reported as a tool failure... try { $result = $tool->handle(new Request($arguments, $toolCallId, $toolInvocationId)); - } catch (ValidationException $exception) { - // Validation failures are returned to the model so it can correct the arguments and retry... - $result = implode(' ', $exception->validator->errors()->all()) ?: $exception->getMessage(); } catch (Throwable $exception) { - // Keep the original exception, including cancellation, as previous if failure reporting throws. - try { - // Unlike upstream, cancellation is not wrapped, so the caller never recovers it as the tool's own failure. - throw Coroutine::causedByCancellation($exception) ? $exception : new ToolExecutionException($exception); - } finally { - $context?->toolFailed($tool, $arguments, $exception, $toolInvocationId, $this->elapsedMilliseconds($startedAt)); + // Validation failures are returned to the model so it can correct the arguments and retry. Unlike upstream, a + // sub-agent's validation failure comes from its own run, so it fails the tool call, as the streamed path does... + if ($exception instanceof ValidationException && ! $tool instanceof AgentTool) { + $result = implode(' ', $exception->validator->errors()->all()) ?: $exception->getMessage(); + } else { + // Keep the original exception, including cancellation, as previous if failure reporting throws. + try { + // Unlike upstream, cancellation is not wrapped, so the caller never recovers it as the tool's own failure. + throw Coroutine::causedByCancellation($exception) ? $exception : new ToolExecutionException($exception); + } finally { + $context?->toolFailed($tool, $arguments, $exception, $toolInvocationId, $this->elapsedMilliseconds($startedAt)); + } } } diff --git a/tests/Ai/Feature/ToolFailureRecoveryTest.php b/tests/Ai/Feature/ToolFailureRecoveryTest.php index 779634ba22..4c3bb1d81a 100644 --- a/tests/Ai/Feature/ToolFailureRecoveryTest.php +++ b/tests/Ai/Feature/ToolFailureRecoveryTest.php @@ -18,6 +18,7 @@ use Hypervel\Tests\Ai\Fixtures\Agents\ResearchAgent; use Hypervel\Tests\Ai\Fixtures\Agents\ThrowingToolAgent; use Hypervel\Tests\Ai\TestCase; +use Hypervel\Validation\ValidationException; use PHPUnit\Framework\Attributes\TestWith; use RuntimeException; @@ -115,6 +116,29 @@ public function testAThrowingSubAgentIsReportedOnceAndHiddenFromTheModelOutsideD Exceptions::assertReported(fn (RuntimeException $reported): bool => $reported === $exception); } + #[WithConfig('app.debug', false)] + public function testASubAgentValidationFailureFailsTheToolCallInsteadOfReturningItsMessages(): void + { + $exception = ValidationException::withMessages(['task' => 'The sub-agent rejected its input.']); + $failures = []; + + Event::listen(ToolFailed::class, function (ToolFailed $event) use (&$failures): void { + $failures[] = $event->exception; + }); + + DelegatingAgent::fake([ + new ToolCall('call_1', 'research_agent', ['task' => 'Research']), + 'The research could not be completed.', + ]); + ResearchAgent::fake(fn () => throw $exception); + + $response = (new DelegatingAgent)->prompt('Research'); + + $this->assertFalse($response->toolResults[0]->successful()); + $this->assertSame('The tool call failed.', $response->toolResults[0]->error()); + $this->assertSame([$exception], $failures); + } + public function testAnExceptionFromAToolEventListenerStillFailsTheRun(): void { Exceptions::fake(); From 999a261ad8b00483bcc7299fd7f5427c59428f0a Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:35:34 +0000 Subject: [PATCH 23/30] Keep the Anthropic code execution container across steps Anthropic only returns the container in a response when the step runs code. A step that only called local tools came back without one, so the next step sent no container and any later code ran in a new container without the files created earlier. A step response that doesn't name a container now keeps the one its request used, including a container set by request middleware. This applies to both generated and streamed steps; a stream that ends without completing its message still produces no step. Covered in ToolCallLoopTest and StreamingTest. --- .../Gateway/Anthropic/AnthropicGateway.php | 10 +++- .../Anthropic/Concerns/BuildsTextRequests.php | 16 ++++++ .../Providers/Anthropic/StreamingTest.php | 50 +++++++++++++++++++ .../Providers/Anthropic/ToolCallLoopTest.php | 49 ++++++++++++++++++ 4 files changed, 123 insertions(+), 2 deletions(-) diff --git a/src/ai/src/Gateway/Anthropic/AnthropicGateway.php b/src/ai/src/Gateway/Anthropic/AnthropicGateway.php index e3d466e9df..c3890be7d4 100644 --- a/src/ai/src/Gateway/Anthropic/AnthropicGateway.php +++ b/src/ai/src/Gateway/Anthropic/AnthropicGateway.php @@ -90,7 +90,10 @@ public function generateTextStep( $this->validateTextResponse($data); - return $this->parseTextResponse($data, $provider, filled($schema))->withRawResponse($response); + return $this->withRequestContainer( + $this->parseTextResponse($data, $provider, filled($schema))->withRawResponse($response), + $body, + ); } /** @@ -130,13 +133,16 @@ public function generateStreamStep( ->post('messages', $body), ); - return yield from $this->processTextStream( + $step = yield from $this->processTextStream( $invocationId, $provider, $model, $response->getBody(), $this->pausedContent($messages), ); + + // A stream that ends without completing its message has no step. + return $step === null ? null : $this->withRequestContainer($step, $body); } /** diff --git a/src/ai/src/Gateway/Anthropic/Concerns/BuildsTextRequests.php b/src/ai/src/Gateway/Anthropic/Concerns/BuildsTextRequests.php index c16146e72f..eaa2e5bcab 100644 --- a/src/ai/src/Gateway/Anthropic/Concerns/BuildsTextRequests.php +++ b/src/ai/src/Gateway/Anthropic/Concerns/BuildsTextRequests.php @@ -8,6 +8,7 @@ use Hypervel\Ai\Attributes\CacheToolDefinitions; use Hypervel\Ai\Gateway\Anthropic\AnthropicSchemaSanitizer; use Hypervel\Ai\Gateway\StepContext; +use Hypervel\Ai\Gateway\StepResponse; use Hypervel\Ai\Gateway\TextGenerationOptions; use Hypervel\Ai\ObjectSchema; use Hypervel\Ai\Providers\Provider; @@ -100,6 +101,21 @@ protected function withContainer(array $body, StepContext $stepContext): array return $body; } + /** + * Keep the container the request ran in when the response doesn't name one. + * + * Anthropic only returns the container when the step runs code, so a step that only calls local tools would + * otherwise drop it, and the next code step would start in a new container without the earlier files. + */ + protected function withRequestContainer(StepResponse $step, array $body): StepResponse + { + $container = $body['container'] ?? null; + + $step->continuationToken ??= is_array($container) ? ($container['id'] ?? null) : $container; + + return $step; + } + /** * Stamp the requested cache breakpoints onto the final request body. */ diff --git a/tests/Ai/Feature/Providers/Anthropic/StreamingTest.php b/tests/Ai/Feature/Providers/Anthropic/StreamingTest.php index 8973ad5eb4..34c292ae87 100644 --- a/tests/Ai/Feature/Providers/Anthropic/StreamingTest.php +++ b/tests/Ai/Feature/Providers/Anthropic/StreamingTest.php @@ -5,6 +5,7 @@ namespace Hypervel\Tests\Ai\Feature\Providers\Anthropic; use Closure; +use Hypervel\Ai\Attributes\MaxSteps; use Hypervel\Ai\Exceptions\StreamErrorException; use Hypervel\Ai\Gateway\StepResult; use Hypervel\Ai\Messages\UserMessage; @@ -233,6 +234,55 @@ public function testStreamingReusesTheCodeExecutionContainerOnTheFollowUpRequest $this->assertSame('container_123', $recorded[1][0]->data()['container']); } + #[TestWith([null])] + #[TestWith(['container_override'])] + public function testStreamingCarriesOverTheContainerAStepRanInWhenItsResponseOmitsIt(?string $override): void + { + $toolCallResponse = fn (string $id, array $delta): array => [ + 'body' => $this->ssePayload([ + $this->messageStart(), + $this->contentBlockStart(0, ['type' => 'tool_use', 'id' => $id, 'name' => 'FixedNumberGenerator', 'input' => '']), + $this->contentBlockDelta(0, ['type' => 'input_json_delta', 'partial_json' => '{}']), + $this->contentBlockStop(0), + ['type' => 'message_delta', 'delta' => $delta, 'usage' => ['output_tokens' => 5]], + ]), + 'status' => 200, + 'headers' => ['Content-Type' => 'text/event-stream'], + ]; + + Http::fake([ + 'api.anthropic.com/*' => Http::sequence([ + Http::response(...$toolCallResponse('toolu_1', ['stop_reason' => 'tool_use', 'container' => ['id' => 'container_123', 'expires_at' => '2026-11-08T11:00:00Z']])), + Http::response(...$toolCallResponse('toolu_2', ['stop_reason' => 'tool_use'])), + Http::response( + body: $this->ssePayload([ + $this->messageStart(), + $this->contentBlockStart(0, ['type' => 'text', 'text' => '']), + $this->contentBlockDelta(0, ['type' => 'text_delta', 'text' => 'The numbers are 72019']), + $this->contentBlockStop(0), + $this->messageDelta('end_turn', 10), + ]), + status: 200, + headers: ['Content-Type' => 'text/event-stream'], + ), + ]), + ]); + + $steps = 0; + + $this->collectStreamEvents(agent: (new #[MaxSteps(3)] class extends ProviderOptionsWithToolsAgent {})->withMiddleware([ + function (PendingStep $step, Closure $next) use (&$steps, $override): StepResult { + return $next(++$steps === 2 && $override !== null ? $step->withProviderOptions(['container' => $override]) : $step); + }, + ])); + + $recorded = Http::recorded(); + + $this->assertCount(3, $recorded); + $this->assertSame($override ?? 'container_123', $recorded[1][0]->data()['container']); + $this->assertSame($override ?? 'container_123', $recorded[2][0]->data()['container']); + } + public function testStreamingRunsToolCallsWhenTheMessageEndsWithEndTurn(): void { Http::fake([ diff --git a/tests/Ai/Feature/Providers/Anthropic/ToolCallLoopTest.php b/tests/Ai/Feature/Providers/Anthropic/ToolCallLoopTest.php index fa5bae897c..af88d37404 100644 --- a/tests/Ai/Feature/Providers/Anthropic/ToolCallLoopTest.php +++ b/tests/Ai/Feature/Providers/Anthropic/ToolCallLoopTest.php @@ -5,6 +5,7 @@ namespace Hypervel\Tests\Ai\Feature\Providers\Anthropic; use Closure; +use Hypervel\Ai\Attributes\MaxSteps; use Hypervel\Ai\Contracts\Agent; use Hypervel\Ai\Contracts\HasProviderOptions; use Hypervel\Ai\Contracts\HasTools; @@ -176,6 +177,54 @@ public function testTheCodeExecutionContainerIsReusedByTheFollowUpRequest(?strin } } + #[TestWith([null])] + #[TestWith(['container_override'])] + public function testTheContainerAStepRanInCarriesOverWhenItsResponseOmitsIt(?string $override): void + { + Http::fake([ + 'api.anthropic.com/*' => Http::sequence([ + Http::response([ + 'id' => 'msg_code', + 'type' => 'message', + 'role' => 'assistant', + 'model' => 'claude-sonnet-4-6', + 'content' => [ + ['type' => 'tool_use', 'id' => 'toolu_1', 'name' => 'FixedNumberGenerator', 'input' => new stdClass], + ], + 'container' => ['id' => 'container_123', 'expires_at' => '2026-11-08T11:00:00Z'], + 'stop_reason' => 'tool_use', + 'usage' => ['input_tokens' => 10, 'output_tokens' => 5], + ]), + Http::response([ + 'id' => 'msg_local_tool', + 'type' => 'message', + 'role' => 'assistant', + 'model' => 'claude-sonnet-4-6', + 'content' => [ + ['type' => 'tool_use', 'id' => 'toolu_2', 'name' => 'FixedNumberGenerator', 'input' => new stdClass], + ], + 'stop_reason' => 'tool_use', + 'usage' => ['input_tokens' => 10, 'output_tokens' => 5], + ]), + $this->fakeTextResponse('The number is 72019'), + ]), + ]); + + $steps = 0; + + (new #[MaxSteps(3)] class(fixed: true) extends ToolUsingAgent {})->withMiddleware([ + function (PendingStep $step, Closure $next) use (&$steps, $override): StepResult { + return $next(++$steps === 2 && $override !== null ? $step->withProviderOptions(['container' => $override]) : $step); + }, + ])->prompt('Generate two random numbers', provider: 'anthropic'); + + $recorded = Http::recorded(); + + $this->assertCount(3, $recorded); + $this->assertSame($override ?? 'container_123', $recorded[1][0]->data()['container']); + $this->assertSame($override ?? 'container_123', $recorded[2][0]->data()['container']); + } + public function testTheCodeExecutionContainerIsReusedWhenAPausedTurnResumes(): void { Http::fake([ From 88cf484bd4182657c0babcc96a9b6d7b77aeb584 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:35:34 +0000 Subject: [PATCH 24/30] Move CREATE foreign keys after their targets instead When creating a table, primary, unique and index commands written after a foreign key were moved ahead of it so the referenced index exists first. Moving the targets also moved them ahead of commands they depend on, so Scout's trigram index ran before the command that creates the pg_trgm extension and failed. Foreign keys now move after the last target command instead, and every other command keeps its written order. Covered in DatabaseSchemaBlueprintTest. --- src/database/src/Schema/Blueprint.php | 34 +++++++++---------- .../Database/DatabaseSchemaBlueprintTest.php | 4 +-- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/database/src/Schema/Blueprint.php b/src/database/src/Schema/Blueprint.php index d28942d3f4..c575cde55f 100755 --- a/src/database/src/Schema/Blueprint.php +++ b/src/database/src/Schema/Blueprint.php @@ -176,7 +176,7 @@ protected function addImpliedCommands(): void $this->addFluentCommands(); if ($this->creating()) { - $this->promoteCreateIndexesBeforeForeignKeys(); + $this->placeForeignKeysAfterTheirTargets(); } else { $this->commands = array_map( fn ($command) => $command instanceof ColumnDefinition @@ -303,37 +303,35 @@ private function placeGeneratedIndexesAfterColumns(array $generatedIndexes): voi } /** - * Promote CREATE indexes before foreign keys. + * Move CREATE foreign keys after the indexes they may reference. * * Target indexes must exist before referencing foreign keys on supported server databases. - * CREATE is declarative, so moving only targets preserves later non-target commands. + * Moving the foreign keys, rather than the targets, keeps every other command in its written + * order, so a command a target index depends on, such as creating an extension, still runs first. */ - private function promoteCreateIndexesBeforeForeignKeys(): void + private function placeForeignKeysAfterTheirTargets(): void { - $foreignKeyOffset = array_find_key( - $this->commands, - static fn (Fluent $command): bool => $command->name === 'foreign', + $lastTargetOffset = array_find_key( + array_reverse($this->commands, true), + static fn (Fluent $command): bool => in_array($command->name, self::FOREIGN_KEY_TARGET_COMMANDS, true), ); - if (is_null($foreignKeyOffset)) { + if (is_null($lastTargetOffset)) { return; } - $prefix = array_slice($this->commands, 0, $foreignKeyOffset); - $targets = []; - $remainder = []; - - for ($offset = $foreignKeyOffset, $count = count($this->commands); $offset < $count; ++$offset) { - $command = $this->commands[$offset]; + $foreignKeys = []; + $commands = []; - if (in_array($command->name, self::FOREIGN_KEY_TARGET_COMMANDS, true)) { - $targets[] = $command; + foreach (array_slice($this->commands, 0, $lastTargetOffset + 1) as $command) { + if ($command->name === 'foreign') { + $foreignKeys[] = $command; } else { - $remainder[] = $command; + $commands[] = $command; } } - $this->commands = [...$prefix, ...$targets, ...$remainder]; + $this->commands = [...$commands, ...$foreignKeys, ...array_slice($this->commands, $lastTargetOffset + 1)]; } /** diff --git a/tests/Database/DatabaseSchemaBlueprintTest.php b/tests/Database/DatabaseSchemaBlueprintTest.php index 61014ed171..a2a6c45760 100755 --- a/tests/Database/DatabaseSchemaBlueprintTest.php +++ b/tests/Database/DatabaseSchemaBlueprintTest.php @@ -384,7 +384,7 @@ public function testRemoveColumnMatchesNumericLookingNamesStrictly(): void $this->assertSame(['1e2', '0100'], array_column($blueprint->getCommands(), 'name')); } - public function testCreatePromotesForeignKeyTargetsWithoutMovingOtherCommands(): void + public function testCreateMovesForeignKeysAfterTheirTargetsWithoutReorderingOtherCommands(): void { $blueprint = new DatabaseSchemaBlueprintOrderingTestBlueprint( $this->getConnection('Postgres'), @@ -406,7 +406,7 @@ function (DatabaseSchemaBlueprintOrderingTestBlueprint $table) { $this->assertSame( [ - 'create', 'unique', 'index', 'primary', 'foreign', 'foreign', 'custom', + 'create', 'unique', 'index', 'custom', 'primary', 'foreign', 'foreign', 'AutoIncrementStartingValues', 'Comment', 'AutoIncrementStartingValues', 'Comment', 'AutoIncrementStartingValues', 'Comment', ], From 74036376e75419ca95d09b386cf3b18490adcad8 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:35:34 +0000 Subject: [PATCH 25/30] Fix Scout searchable index names on prefixed and long tables dropSearchable() qualified the trigram index name with the table's schema. The PostgreSQL grammar already qualifies dropped indexes, and the qualified name picked up the table prefix on its schema, so the drop targeted an index that doesn't exist. PostgreSQL truncates identifiers to 63 bytes, so two generated searchable index names sharing their first 63 bytes collided when the table was created. Overlong generated names are now shortened with a hash of the full name, without splitting a multibyte character, and dropSearchable() derives the same names. Covered in SearchableSchemaTest and DatabasePgsqlSearchTest, which runs the create, the ordering with a foreign key, and the drop against PostgreSQL. --- src/scout/src/SearchableSchema.php | 29 ++++++--- .../Postgres/DatabasePgsqlSearchTest.php | 26 +++++++- tests/Scout/Unit/SearchableSchemaTest.php | 61 +++++++++++++++++++ 3 files changed, 106 insertions(+), 10 deletions(-) diff --git a/src/scout/src/SearchableSchema.php b/src/scout/src/SearchableSchema.php index 63b72ce55a..5a0e2fdde6 100644 --- a/src/scout/src/SearchableSchema.php +++ b/src/scout/src/SearchableSchema.php @@ -6,7 +6,6 @@ use Hypervel\Database\Schema\Blueprint; use Hypervel\Database\Schema\Grammars\Grammar; -use Hypervel\Support\Str; use InvalidArgumentException; class SearchableSchema @@ -25,7 +24,7 @@ public static function register(): void SearchableSchema::vectorExpression($columns, $language) ); - $this->index($tsvector, null, 'gin'); + $this->index($tsvector, SearchableSchema::indexName($this->createIndexName('index', [$tsvector])), 'gin'); if ($trigram === []) { return; @@ -36,22 +35,38 @@ public static function register(): void foreach ($trigram as $column) { SearchableSchema::ensureValidIdentifier($column); - $this->rawIndex("\"{$column}\" gin_trgm_ops", $this->createIndexName('trigram', [$column]))->algorithm('gin'); + $this->rawIndex("\"{$column}\" gin_trgm_ops", SearchableSchema::indexName($this->createIndexName('trigram', [$column])))->algorithm('gin'); } }); Blueprint::macro('dropSearchable', function (array $trigram = [], string $tsvector = 'search_vector'): void { - $table = $this->getTable(); - $schema = str_contains($table, '.') ? Str::beforeLast($table, '.') . '.' : ''; - + // Unlike upstream, the index name isn't qualified with the table's schema here: the PostgreSQL grammar + // already qualifies dropped indexes, and a qualified name would get the table prefix on its schema. foreach ($trigram as $column) { - $this->dropIndex($schema . $this->createIndexName('trigram', [$column])); + $this->dropIndex(SearchableSchema::indexName($this->createIndexName('trigram', [$column]))); } $this->dropColumn($tsvector); }); } + /** + * Fit the given generated index name within PostgreSQL's 63-byte identifier limit. + * + * Unlike upstream, an overlong name is shortened deterministically rather than left for PostgreSQL to truncate, + * since two generated names sharing their first 63 bytes would collide. The cut never splits a character. + * + * @internal + */ + public static function indexName(string $name): string + { + if (strlen($name) <= 63) { + return $name; + } + + return mb_strcut($name, 0, 46, 'UTF-8') . '_' . substr(hash('xxh128', $name), 0, 16); + } + /** * Build the weighted tsvector expression for the given columns. * diff --git a/tests/Integration/Scout/Database/Postgres/DatabasePgsqlSearchTest.php b/tests/Integration/Scout/Database/Postgres/DatabasePgsqlSearchTest.php index 5250f0fa8b..680fe8c612 100644 --- a/tests/Integration/Scout/Database/Postgres/DatabasePgsqlSearchTest.php +++ b/tests/Integration/Scout/Database/Postgres/DatabasePgsqlSearchTest.php @@ -85,6 +85,26 @@ public function testSearchableCreatesGeneratedVectorAndGinIndexes(): void $this->assertFalse(Schema::hasColumn('scout_posts', 'search_vector')); } + public function testOverlongSearchableIndexNamesDoNotCollideAndCanBeDropped(): void + { + // PostgreSQL would truncate both trigram index names to the same identifier, and a byte cut + // at the shortened length would split one of the table name's multibyte characters. + $table = 'scout_p' . str_repeat('é', 22); + + Schema::create($table, function (Blueprint $table): void { + $table->id(); + $table->text('description_primary'); + $table->text('description_secondary'); + $table->searchable(['description_primary'], trigram: ['description_primary', 'description_secondary']); + }); + + $this->assertCount(3, $this->indexes($table)); + + Schema::table($table, fn (Blueprint $table) => $table->dropSearchable(['description_primary', 'description_secondary'])); + + $this->assertSame([], $this->indexes($table)); + } + public function testSearchableCanBeAddedToAnExistingTable(): void { Schema::table('scout_posts', fn (Blueprint $table) => $table->dropSearchable(['title'])); @@ -333,13 +353,13 @@ protected function assertSearchableTableWorks(SchemaBuilder $schema, string $tab } /** - * Get the definitions of the searchable table's indexes, keyed by name. + * Get the definitions of the given table's indexes other than its primary key, keyed by name. * * @return array */ - protected function indexes(): array + protected function indexes(string $table = 'scout_posts'): array { - return collect(DB::select("select indexname, indexdef from pg_indexes where tablename = 'scout_posts' and indexname <> 'scout_posts_pkey' order by indexname")) + return collect(DB::select('select indexname, indexdef from pg_indexes where tablename = ? and indexname <> ? order by indexname', [$table, "{$table}_pkey"])) ->pluck('indexdef', 'indexname') ->all(); } diff --git a/tests/Scout/Unit/SearchableSchemaTest.php b/tests/Scout/Unit/SearchableSchemaTest.php index 982d17d362..5fe188825e 100644 --- a/tests/Scout/Unit/SearchableSchemaTest.php +++ b/tests/Scout/Unit/SearchableSchemaTest.php @@ -4,12 +4,27 @@ namespace Hypervel\Tests\Scout\Unit; +use Hypervel\Database\Connection; +use Hypervel\Database\Schema\Blueprint; +use Hypervel\Database\Schema\Grammars\PostgresGrammar; +use Hypervel\Database\Schema\PostgresBuilder; use Hypervel\Scout\SearchableSchema; use Hypervel\Tests\TestCase; use InvalidArgumentException; +use JMac\Testing\Double; +use JMac\Testing\Integrations\PHPUnit\VerifiesDoubles; class SearchableSchemaTest extends TestCase { + use VerifiesDoubles; + + protected function setUp(): void + { + parent::setUp(); + + SearchableSchema::register(); + } + public function testVectorExpressionWeightsColumnsAndCoalescesNulls(): void { $this->assertSame( @@ -23,6 +38,38 @@ public function testVectorExpressionAcceptsSchemaQualifiedLanguages(): void $this->assertStringContainsString("to_tsvector('public.english_unaccent'", SearchableSchema::vectorExpression(['title'], 'public.english_unaccent')); } + public function testTheTrigramExtensionIsCreatedBeforeItsIndexWhenTheTableHasAForeignKey(): void + { + $blueprint = new Blueprint($this->getConnection(), 'posts', function (Blueprint $table): void { + $table->create(); + $table->foreignId('user_id')->constrained(); + $table->text('title'); + $table->searchable(['title'], trigram: ['title']); + }); + + $statements = $blueprint->toSql(); + $offsetOf = fn (string $needle): int => array_find_key($statements, fn (string $statement): bool => str_contains($statement, $needle)); + + $this->assertLessThan($offsetOf('gin_trgm_ops'), $offsetOf('create extension if not exists pg_trgm')); + $this->assertLessThan($offsetOf('foreign key'), $offsetOf('gin_trgm_ops')); + } + + public function testDropSearchableDropsTheTrigramIndexItCreatedOnASchemaQualifiedPrefixedTable(): void + { + $create = new Blueprint($this->getConnection('pre_'), 'tenant.posts', function (Blueprint $table): void { + $table->searchable(['title'], trigram: ['title']); + }); + $drop = new Blueprint($this->getConnection('pre_'), 'tenant.posts', function (Blueprint $table): void { + $table->dropSearchable(['title']); + }); + + $this->assertContains('create index "tenant_posts_title_trigram" on "tenant"."pre_posts" using gin ("title" gin_trgm_ops)', $create->toSql()); + $this->assertSame([ + 'drop index "tenant"."tenant_posts_title_trigram"', + 'alter table "tenant"."pre_posts" drop column "search_vector"', + ], $drop->toSql()); + } + public function testVectorExpressionRejectsInvalidInput(): void { foreach ([[[], 'english'], [['title'], "english'); drop table users; --"], [['title' => 'E'], 'english'], [['ti"tle'], 'english']] as [$columns, $language]) { @@ -34,4 +81,18 @@ public function testVectorExpressionRejectsInvalidInput(): void } } } + + /** + * Get a PostgreSQL connection for compiling blueprints. + */ + protected function getConnection(string $prefix = ''): Connection + { + $connection = Double::for(Connection::class)->strict(); + $connection->allows('getTablePrefix')->returns($prefix); + $connection->allows('getConfig')->with('prefix_indexes')->returns(null); + $connection->allows('getSchemaGrammar')->returns(new PostgresGrammar($connection)); + $connection->allows('getSchemaBuilder')->returns(Double::for(PostgresBuilder::class)->passthru()); + + return $connection; + } } From a974167e7dfb374b82f69e11b12abef285a0367d Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:35:34 +0000 Subject: [PATCH 26/30] Encode ampersands and quotes in secured Markdown mail withSecuredEncoding() escaped Markdown and HTML syntax in interpolated values, but not ampersands or quotes. A value echoed into a component attribute, such as a button URL, could close the attribute and add new ones, and an ampersand could form an HTML entity like ©. Ampersands and quotes are now encoded with the other characters and restored as escaped HTML after rendering, so interpolated values can't introduce tags, attributes or entities. Covered in MailableWithSecuredEncodingTest, including a button URL with quotes and an injected attribute. --- src/mail/src/Markdown.php | 18 +++++++++++++++--- .../message-with-punctuation.blade.php | 4 ++++ .../Mail/MailableWithSecuredEncodingTest.php | 7 ++++++- .../MailableWithoutSecuredEncodingTest.php | 1 + 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/mail/src/Markdown.php b/src/mail/src/Markdown.php index 87850025ec..4fc46d5257 100644 --- a/src/mail/src/Markdown.php +++ b/src/mail/src/Markdown.php @@ -20,7 +20,10 @@ class Markdown { /** - * References that keep interpolated punctuation out of Markdown syntax. + * References that keep interpolated punctuation out of Markdown and HTML syntax. + * + * Unlike Laravel, ampersands and quotes are encoded too, as the default encoder does, so secured + * encoding never lets a value break out of an HTML attribute or turn literal entity text into a character. */ protected const array ENCODED_CHARACTERS = [ '\\' => '\', @@ -28,13 +31,16 @@ class Markdown ']' => ']', '<' => '<', '>' => '>', + '&' => '&', + '"' => '"', + "'" => ''', ]; /** * Restore references left escaped in code or unchanged in raw HTML. * - * Zero padding keeps ordinary entity examples and literal backslashes intact. - * Angle brackets remain escaped so restoration cannot introduce HTML tags. + * Zero padding keeps ordinary entity examples and literal backslashes intact. Angle brackets, + * ampersands and quotes remain escaped so restoration cannot introduce HTML tags, attributes or entities. */ protected const array DECODED_CHARACTERS = [ '&#0000092;' => '\\', @@ -42,11 +48,17 @@ class Markdown '&#0000093;' => ']', '&#0000060;' => '<', '&#0000062;' => '>', + '&#0000038;' => '&', + '&#0000034;' => '"', + '&#0000039;' => ''', '\' => '\\', '[' => '[', ']' => ']', '<' => '<', '>' => '>', + '&' => '&', + '"' => '"', + ''' => ''', ]; /** diff --git a/tests/Integration/Mail/Fixtures/message-with-punctuation.blade.php b/tests/Integration/Mail/Fixtures/message-with-punctuation.blade.php index ea295817bc..2331db0920 100644 --- a/tests/Integration/Mail/Fixtures/message-with-punctuation.blade.php +++ b/tests/Integration/Mail/Fixtures/message-with-punctuation.blade.php @@ -9,6 +9,10 @@ {{ $value }} + +Open + + {{ $value }} diff --git a/tests/Integration/Mail/MailableWithSecuredEncodingTest.php b/tests/Integration/Mail/MailableWithSecuredEncodingTest.php index 289e386b19..c591ced4e7 100644 --- a/tests/Integration/Mail/MailableWithSecuredEncodingTest.php +++ b/tests/Integration/Mail/MailableWithSecuredEncodingTest.php @@ -175,10 +175,12 @@ public static function markdownEncodedLinkTextTemplateDataProvider(): iterable public function testSecuredRenderingPreservesLiteralContentAcrossMailComponents(): void { - $value = '\\\server\share [x] '; + $value = '\\\server\share [x] "quoted" it\'s © & more'; + $url = 'https://example.com/search?q="mail" data-injected="yes"&tag=it\'s'; $html = $this->app->make(Markdown::class)->render('message-with-punctuation', [ 'value' => $value, 'punctuation' => '[x]', + 'url' => $url, 'host' => 'evil.test\\', 'number' => 42, ])->toHtml(); @@ -188,6 +190,9 @@ public function testSecuredRenderingPreservesLiteralContentAcrossMailComponents( $this->assertSame($value, $document->filter('code')->eq(1)->text()); $this->assertSame($value, $document->filter('.panel-item')->text()); $this->assertSame($value, $document->filter('.subcopy')->text()); + // The CSS inliner's HTML serializer percent-encodes link URLs. + $this->assertSame($url, rawurldecode($document->selectLink('Open')->attr('href'))); + $this->assertNull($document->selectLink('Open')->attr('data-injected')); $this->assertStringContainsString('content: "[x]"', $document->filter('style')->last()->text()); $this->assertSame('https://evil.test%5C.good.test', $document->selectLink('Destination')->attr('href')); $this->assertContains('42', $document->filter('p')->each(fn (Crawler $paragraph): string => $paragraph->text())); diff --git a/tests/Integration/Mail/MailableWithoutSecuredEncodingTest.php b/tests/Integration/Mail/MailableWithoutSecuredEncodingTest.php index 6dcfd9b674..8398f60d5a 100644 --- a/tests/Integration/Mail/MailableWithoutSecuredEncodingTest.php +++ b/tests/Integration/Mail/MailableWithoutSecuredEncodingTest.php @@ -127,6 +127,7 @@ public function testLiteralCodeEscapesArePreservedWithoutSecuredEncoding(): void $html = $this->app->make(Markdown::class)->render('message-with-punctuation', [ 'value' => 'ordinary text', 'punctuation' => '[x]', + 'url' => 'https://hypervel.org', 'host' => 'safe', 'number' => 42, ])->toHtml(); From f1dc13326a0d37347eddf1b81da98fef3f8aa747 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:35:34 +0000 Subject: [PATCH 27/30] Send quoted Typesense equality filters as one-member lists Typesense decodes backtick quoting in list members but keeps the backticks in a scalar value. On a field that indexes backticks, a quoted equality filter such as band:=`Rock || Roll` therefore matched nothing, and the matching inequality excluded nothing. Equality and inequality values that need quoting are now sent as a one-member list, which Typesense decodes. Values containing their own backticks keep the scalar form: Typesense can't quote a literal backtick, and the list form would match the value without its backticks, which is a different record on such a field. The array-value comment now also explains why explicit operators are kept. Covered in TypesenseEngineTest and against Typesense in TypesenseFilteringIntegrationTest, on a field that indexes backticks. --- src/scout/src/Engines/TypesenseEngine.php | 14 ++++++++++++-- .../Fixtures/Models/TypesenseSearchableModel.php | 3 ++- .../TypesenseFilteringIntegrationTest.php | 16 ++++++++++++++-- tests/Scout/Unit/Engines/TypesenseEngineTest.php | 2 +- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/scout/src/Engines/TypesenseEngine.php b/src/scout/src/Engines/TypesenseEngine.php index 71ddb93fc7..1aca367758 100644 --- a/src/scout/src/Engines/TypesenseEngine.php +++ b/src/scout/src/Engines/TypesenseEngine.php @@ -754,7 +754,7 @@ protected function parseWhereFilter( string $operator = '=' ): string { $operator = match ($operator) { - // Array values are raw filter expressions, such as geo filters, so equality leaves the operator out. Unlike + // Array values are filter expressions, such as geo filters, so equality leaves the operator out. Unlike // upstream, explicit operators are kept for array values instead of being dropped. '=' => is_array($value) ? ':' : ':=', '!=' => ':!=', @@ -771,7 +771,17 @@ protected function parseWhereFilter( ->implode('')); } - return sprintf('%s%s%s', $key, $operator, $this->quoteFilterValue($value, ['&&', '||', '`', '(', ')'])); + $quotedValue = $this->quoteFilterValue($value, ['&&', '||', '`', '(', ')']); + + // Unlike upstream, a quoted equality value is sent as a one-member list. Typesense decodes a list member's + // backticks as delimiters but keeps them in a scalar value, so a quoted scalar misses its record on a field + // that indexes backticks. A value with its own backticks stays scalar: Typesense can't quote them, and the + // list would match the value without them, which is a different record on such a field. + if (in_array($operator, [':=', ':!='], true) && is_string($value) && $quotedValue !== $value && ! str_contains($value, '`')) { + return sprintf('%s%s[%s]', $key, $operator, $quotedValue); + } + + return sprintf('%s%s%s', $key, $operator, $quotedValue); } /** diff --git a/tests/Integration/Scout/Typesense/Fixtures/Models/TypesenseSearchableModel.php b/tests/Integration/Scout/Typesense/Fixtures/Models/TypesenseSearchableModel.php index b25b98d423..d4a7fdf233 100644 --- a/tests/Integration/Scout/Typesense/Fixtures/Models/TypesenseSearchableModel.php +++ b/tests/Integration/Scout/Typesense/Fixtures/Models/TypesenseSearchableModel.php @@ -46,7 +46,8 @@ public function typesenseCollectionSchema(): array ['name' => 'id', 'type' => 'string', 'facet' => true], ['name' => 'ranking', 'type' => 'int32', 'facet' => true], ['name' => 'title', 'type' => 'string', 'facet' => true, 'sort' => true], - ['name' => 'body', 'type' => 'string', 'facet' => true], + // Indexes backticks, so the filter tests cover a field where Typesense keeps them in a quoted scalar value. + ['name' => 'body', 'type' => 'string', 'facet' => true, 'symbols_to_index' => ['`']], ], ]; } diff --git a/tests/Integration/Scout/Typesense/TypesenseFilteringIntegrationTest.php b/tests/Integration/Scout/Typesense/TypesenseFilteringIntegrationTest.php index c80d0ed301..f4c4770120 100644 --- a/tests/Integration/Scout/Typesense/TypesenseFilteringIntegrationTest.php +++ b/tests/Integration/Scout/Typesense/TypesenseFilteringIntegrationTest.php @@ -81,8 +81,8 @@ public function testWhereNotInExcludesSpecifiedValues(): void public function testFiltersMatchValuesContainingSpecialCharacters(): void { - foreach (['Laravel Framework', 'Rock || Roll', 'Acme (UK)', 'Acme, Inc', 'Acme'] as $title) { - TypesenseSearchableModel::create(['title' => $title, 'body' => 'Body']); + foreach (['Laravel Framework', 'Rock || Roll', 'Acme (UK)', 'Acme, Inc', 'Acme', 'A`B`C', 'ABC'] as $title) { + TypesenseSearchableModel::create(['title' => $title, 'body' => $title]); } TypesenseSearchableModel::query()->get()->each(fn ($m) => $this->engine->update(new EloquentCollection([$m]))); @@ -99,6 +99,18 @@ public function testFiltersMatchValuesContainingSpecialCharacters(): void $this->assertNotContains('Rock || Roll', $titles); $this->assertNotContains('Acme, Inc', $titles); $this->assertContains('Acme', $titles); + + // The body field indexes backticks, so quoted values must still match exactly and exclude exactly there. + $this->assertSame(['Rock || Roll'], TypesenseSearchableModel::search('')->where('body', 'Rock || Roll')->get()->pluck('title')->all()); + + $titles = TypesenseSearchableModel::search('')->where('body', '!=', 'Acme (UK)')->get()->pluck('title')->all(); + + $this->assertNotContains('Acme (UK)', $titles); + $this->assertCount(6, $titles); + + // Typesense can't quote a literal backtick, but the value without its backticks is a different record there. + $this->assertNotContains('ABC', TypesenseSearchableModel::search('')->where('body', 'A`B`C')->get()->pluck('title')->all()); + $this->assertContains('ABC', TypesenseSearchableModel::search('')->where('body', '!=', 'A`B`C')->get()->pluck('title')->all()); } public function testMultipleWhereClausesAreCombinedWithAnd(): void diff --git a/tests/Scout/Unit/Engines/TypesenseEngineTest.php b/tests/Scout/Unit/Engines/TypesenseEngineTest.php index 3e38a5611a..15484644fc 100644 --- a/tests/Scout/Unit/Engines/TypesenseEngineTest.php +++ b/tests/Scout/Unit/Engines/TypesenseEngineTest.php @@ -134,7 +134,7 @@ public function testFilterValuesWithSpecialCharactersAreQuoted(): void $result = $this->invokeMethod($engine, 'filters', [$builder]); $this->assertSame( - 'band:=`Rock || Roll` && company:!=`Acme (UK)` && title:=`Its R&&D` && name:=New* && price:=[10..100] && location:(48.85, 2.34, 5 km)' + 'band:=[`Rock || Roll`] && company:!=[`Acme (UK)`] && title:=`Its R&&D` && name:=New* && price:=[10..100] && location:(48.85, 2.34, 5 km)' . ' && status:`Rock || Roll` && company:=[Laravel, `Acme (UK)`, `Acme, Inc`] && band:!=[`Rock || Roll`]', $result ); From 27d709db0749016c5936a8855fcb324c819ecc84 Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:35:34 +0000 Subject: [PATCH 28/30] Correct MCP, middleware and request docs wording - assertStructuredContentFragment() checks that each key/value pair appears somewhere in the structured content. - insertBefore() and insertAfter() insert nothing when the middleware being positioned against is in neither the default global stack nor the stack passed to use(). - bearerToken() returns null when the header has no bearer scheme. --- src/docs/mcp.md | 2 +- src/docs/middleware.md | 2 +- src/docs/requests.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/docs/mcp.md b/src/docs/mcp.md index 7ff8022d56..4c21f7d7fd 100644 --- a/src/docs/mcp.md +++ b/src/docs/mcp.md @@ -2527,7 +2527,7 @@ The following structured content assertions are available: | `assertStructuredContentPaths($paths)` | Assert multiple path / value pairs. | | `assertStructuredContentPathCanonicalizing($path, $expect)` | Assert the values at a given path, ignoring order. | | `assertStructuredContentPathsCanonicalizing($paths)` | Assert multiple path / value pairs, ignoring order. | -| `assertStructuredContentFragment($data)` | Assert that a fragment appears anywhere in the structured content. | +| `assertStructuredContentFragment($data)` | Assert that each key/value pair appears somewhere in the structured content. | | `assertStructuredContentFragments($data)` | Assert that each given fragment appears in the structured content. | | `assertStructuredContentMissing($data)` | Assert that a fragment does not appear in the structured content. | | `assertStructuredContentMissingExact($data)` | Assert that an exact fragment does not appear in the structured content. | diff --git a/src/docs/middleware.md b/src/docs/middleware.md index 3995fbc584..0543555572 100644 --- a/src/docs/middleware.md +++ b/src/docs/middleware.md @@ -149,7 +149,7 @@ use Hypervel\Http\Middleware\TrustProxies; You may also pass an array of middleware as the second argument to either method. > [!WARNING] -> If the middleware you are positioning against is not part of the global middleware stack, nothing will be inserted. +> If the middleware you are positioning against is not part of the default global middleware stack, or the stack you pass to the `use` method, nothing will be inserted. #### Preferring JSON Responses diff --git a/src/docs/requests.md b/src/docs/requests.md index 9fc77c257d..9770675a4b 100644 --- a/src/docs/requests.md +++ b/src/docs/requests.md @@ -228,7 +228,7 @@ if ($request->hasHeader('X-Header-Name')) { } ``` -For convenience, the `bearerToken` method may be used to retrieve a bearer token from the `Authorization` header. If no token is present, `null` will be returned: +For convenience, the `bearerToken` method may be used to retrieve a bearer token from the `Authorization` header. If the header contains no bearer scheme, the method returns `null`: ```php $token = $request->bearerToken(); From 00a637a10f0e7735b67c3450da42d9b2fd23d5fa Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:35:34 +0000 Subject: [PATCH 29/30] Assert the Wayfinder redirect route registers query The redirect route test checked every verb the route registered, but not that query was among them, so it still passed if the redirect route stopped registering query. --- src/wayfinder/tests/QueryMethod.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/wayfinder/tests/QueryMethod.test.ts b/src/wayfinder/tests/QueryMethod.test.ts index 114cd7a9f6..824086b5a8 100644 --- a/src/wayfinder/tests/QueryMethod.test.ts +++ b/src/wayfinder/tests/QueryMethod.test.ts @@ -109,6 +109,8 @@ describe("get and query route", () => { describe("redirect route", () => { test("every registered verb", () => { + expect(photos.redirect.definition.methods).toContain("query"); + for (const method of photos.redirect.definition.methods) { expect(photos.redirect[method]()).toEqual({ url: "/old-photos", From 2c6b9239366c01a820f0c5e2f9ff78ac20a39e4b Mon Sep 17 00:00:00 2001 From: Raj Siva-Rajah <5361908+binaryfire@users.noreply.github.com> Date: Fri, 20 Nov 2026 06:57:55 +0000 Subject: [PATCH 30/30] Run Scout's Typesense tests against Typesense 30.1 CI ran Typesense 27.1, which only supports symbols_to_index for a whole collection. The Typesense test fixture sets it on one field, so on 27.1 that field indexed values like any other field, and the filter tests for fields that index backticks couldn't tell a fixed filter from a broken one. Typesense 29.0 added field-level symbols_to_index. CI now uses 30.1, the version upstream Scout tests against, so those filter tests check the behavior they describe. --- .github/workflows/scout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scout.yml b/.github/workflows/scout.yml index 46ad74a5a8..a76285d64f 100644 --- a/.github/workflows/scout.yml +++ b/.github/workflows/scout.yml @@ -61,7 +61,7 @@ jobs: services: typesense: - image: typesense/typesense:27.1 + image: typesense/typesense:30.1 env: TYPESENSE_API_KEY: secret TYPESENSE_DATA_DIR: /tmp