diff --git a/python/freetoken/server/anthropic_api.py b/python/freetoken/server/anthropic_api.py index d9a6972f09..86ac89e793 100644 --- a/python/freetoken/server/anthropic_api.py +++ b/python/freetoken/server/anthropic_api.py @@ -16,7 +16,6 @@ from typing import Any from fastapi import FastAPI, Request -from fastapi.exception_handlers import request_validation_exception_handler from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, StreamingResponse @@ -80,6 +79,8 @@ def register_anthropic_routes( get_state: Callable[[], Any], get_model_sampling: Callable[[], dict[str, Any]], ) -> None: + validation_handler = app.exception_handlers[RequestValidationError] + @app.post("/v1/messages") async def v1_messages(req: AnthropicMessagesRequest, request: Request): log_request("/v1/messages", req, request) @@ -104,7 +105,7 @@ async def _validation_error(request: Request, exc: RequestValidationError): return _anthropic_error_response( 400, "invalid_request_error", _validation_error_message(exc) ) - return await request_validation_exception_handler(request, exc) + return await validation_handler(request, exc) async def handle_anthropic_messages( diff --git a/python/freetoken/server/api_models.py b/python/freetoken/server/api_models.py index 6c3ade0924..96e36a85c4 100644 --- a/python/freetoken/server/api_models.py +++ b/python/freetoken/server/api_models.py @@ -67,9 +67,9 @@ class ChatCompletionRequest(BaseModel): messages: list[Message] max_tokens: int | None = None max_completion_tokens: int | None = None - temperature: float | None = None + temperature: float | None = Field(default=None, allow_inf_nan=False) top_k: int | None = None - top_p: float | None = None + top_p: float | None = Field(default=None, allow_inf_nan=False) n: int = 1 stream: bool = False stream_options: StreamOptions | None = None @@ -105,9 +105,9 @@ class CompletionRequest(BaseModel): prompt: str | list[str] | list[int] | list[list[int]] max_tokens: int | None = None max_completion_tokens: int | None = None - temperature: float | None = None + temperature: float | None = Field(default=None, allow_inf_nan=False) top_k: int | None = None - top_p: float | None = None + top_p: float | None = Field(default=None, allow_inf_nan=False) n: int = 1 stream: bool = False stream_options: StreamOptions | None = None diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.py index 4442a84908..e1630a942c 100644 --- a/python/freetoken/server/openai_api.py +++ b/python/freetoken/server/openai_api.py @@ -2,12 +2,15 @@ import asyncio import json +import math import time import uuid from collections.abc import AsyncIterator, Callable from typing import Any from fastapi import FastAPI, Request +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, StreamingResponse from freetoken.core import SamplingParams from freetoken.message import TokenizeMsg @@ -114,6 +117,15 @@ def register_openai_routes( get_state: Callable[[], Any], get_model_sampling: Callable[[], dict[str, Any]], ) -> None: + @app.exception_handler(RequestValidationError) + async def _validation_error(request: Request, exc: RequestValidationError): + # Rejected non-finite inputs must be printable in a strict JSON error response. + errors = jsonable_encoder( + exc.errors(), + custom_encoder={float: lambda value: value if math.isfinite(value) else str(value)}, + ) + return JSONResponse(status_code=422, content={"detail": errors}) + @app.api_route("/v1", methods=["GET", "POST", "HEAD", "OPTIONS"]) async def v1_root(): return {"status": "ok"} diff --git a/tests/server/test_openai_api.py b/tests/server/test_openai_api.py index 899f9e878f..14404c75fc 100644 --- a/tests/server/test_openai_api.py +++ b/tests/server/test_openai_api.py @@ -8,6 +8,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from freetoken.message import TokenizeMsg, UserReply +from freetoken.server.anthropic_api import register_anthropic_routes from freetoken.server.openai_api import ( ChatCompletionRequest, CompletionRequest, @@ -521,6 +522,51 @@ def test_omitted_max_tokens_honors_server_default(): assert fallback_state.sent.sampling_params.max_tokens == DEFAULT_MAX_OUTPUT_TOKENS +@pytest.mark.parametrize("path,prompt", [ + ("/v1/chat/completions", '"messages": [{"role": "user", "content": "hi"}]'), + ("/v1/completions", '"prompt": "hi"'), +]) +@pytest.mark.parametrize("field", ["temperature", "top_p"]) +@pytest.mark.parametrize("value", ["NaN", "Infinity", "-Infinity", "1e9999"]) +@pytest.mark.parametrize("with_anthropic", [False, True]) +def test_sampling_routes_refuse_non_finite_values(path, prompt, field, value, with_anthropic): + state = FakeState([UserReply(uid=42, incremental_output="hi", finished=True)]) + app = FastAPI() + register_openai_routes(app, lambda: state, lambda: {}) + if with_anthropic: + register_anthropic_routes(app, lambda: state, lambda: {}) + + response = TestClient(app, raise_server_exceptions=False).post( + path, + content=f'{{"model": "client-model", {prompt}, "{field}": {value}}}', + headers={"Content-Type": "application/json"}, + ) + + assert response.status_code == 422 + error = response.json()["detail"][0] + assert error["loc"] == ["body", field] + assert error["type"] == "finite_number" + assert state.sent is None + + +@pytest.mark.parametrize("path,prompt", [ + ("/v1/chat/completions", {"messages": [{"role": "user", "content": "hi"}]}), + ("/v1/completions", {"prompt": "hi"}), +]) +def test_sampling_routes_accept_finite_values(path, prompt): + state = FakeState([UserReply(uid=42, incremental_output="hi", finished=True)]) + app = FastAPI() + register_openai_routes(app, lambda: state, lambda: {}) + + response = TestClient(app).post( + path, json={"model": "client-model", **prompt, "temperature": 0.7, "top_p": 0.9}, + ) + + assert response.status_code == 200 + assert state.sent.sampling_params.temperature == 0.7 + assert state.sent.sampling_params.top_p == 0.9 + + def test_models_route_returns_served_model_name(): state = FakeState([]) app = FastAPI()