diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index a9373df4d70417d..fa9e1925e17afd7 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -148,6 +148,251 @@ def test_deregister_collation(self): self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') +class CollationErrorTests(MemoryDatabaseMixin, unittest.TestCase): + # gh-161837: A collation callback cannot report an error to SQLite, so + # the exception stays set while SQLite continues the step. The module + # must propagate that exception, and it must not call other Python code + # while the exception is set. + + # When the collation raises, it returns 0 ("equal"). Thus the comparison + # is true, and abs() of the smallest 64-bit integer makes the step fail. + step_fails = """ + select case when y = 'q' collate mycoll then abs(x) else 0 end + from t order by rowid + """ + step_succeeds = "select x, y = 'q' collate mycoll from t order by rowid" + + def setUp(self): + super().setUp() + self.events = [] + self.setUp_collation() + self.con.execute("create table t(x, y)") + self.con.executemany("insert into t values (?, ?)", + [(1, "a"), (-2**63, "boom")]) + + def setUp_collation(self): + def mycoll(x, y): + if "boom" in (x, y): + self.events.append("collation raised") + raise KeyError("collation raised") + return (x > y) - (x < y) + self.con.create_collation("mycoll", mycoll) + + def assertCollationRaises(self): + return self.assertRaisesRegex(KeyError, "collation raised") + + def events_after_exception(self): + return self.events[self.events.index("collation raised") + 1:] + + def test_execute(self): + for sql in self.step_fails, self.step_succeeds: + with self.subTest(sql=sql), self.assertCollationRaises(): + self.con.execute(sql + " desc") + + def test_fetchone(self): + # The cursor reads one row ahead, so the first fetch steps to the + # row that makes the collation raise. + for sql in self.step_fails, self.step_succeeds: + with self.subTest(sql=sql): + cur = self.con.execute(sql) + with self.assertCollationRaises(): + cur.fetchone() + self.assertIsNone(cur.fetchone()) + self.assertEqual(cur.execute("select 1").fetchone(), (1,)) + + def test_iteration(self): + for sql in self.step_fails, self.step_succeeds: + with self.subTest(sql=sql): + rows = [] + with self.assertCollationRaises(): + for row in self.con.execute(sql): + rows.append(row) + self.assertEqual(rows, []) + + def test_executemany(self): + def params(): + yield ("boom",) + self.fail("executemany() continued after the exception") + with self.assertCollationRaises(): + self.con.executemany( + "insert into t(y) values (? = 'q' collate mycoll)", params()) + + def test_executescript(self): + for sql in self.step_fails, self.step_succeeds: + with self.subTest(sql=sql): + with self.assertCollationRaises(): + self.con.executescript(sql + "; create table u(x)") + # The script stops after the statement with the exception. + with self.assertRaisesRegex(sqlite.OperationalError, + "no such table"): + self.con.execute("select * from u") + + def test_converter_not_called(self): + calls = [] + sqlite.register_converter("COLLTEST", calls.append) + self.addCleanup(sqlite.converters.pop, "COLLTEST", None) + with memory_database(detect_types=sqlite.PARSE_COLNAMES) as con: + con.create_collation("mycoll", lambda x, y: 1 / 0) + with self.assertRaises(ZeroDivisionError): + con.execute(""" + select 'a' = 'b' collate mycoll as "x [colltest]" + """) + self.assertEqual(calls, []) + + def test_function_not_called(self): + def f(value): + self.events.append("function") + self.con.create_function("f", 1, f) + with self.assertCollationRaises(): + self.con.execute("select f(y = 'q' collate mycoll) from t " + "order by rowid desc") + self.assertEqual(self.events, ["collation raised"]) + + def test_aggregate_step_not_called(self): + events = self.events + class Agg: + def step(self, value): + events.append("step") + def finalize(self): + events.append("finalize") + self.con.create_aggregate("agg", 1, Agg) + with self.assertCollationRaises(): + self.con.execute("select agg(y = 'q' collate mycoll) from t") + # SQLite calls finalize() to release the aggregate. + self.assertEqual(self.events_after_exception(), ["finalize"]) + + @with_tracebacks(ZeroDivisionError) + def test_aggregate_finalize_error(self): + class Agg: + def step(self, value): + pass + def finalize(self): + 1 / 0 + self.con.create_aggregate("agg", 1, Agg) + with self.assertCollationRaises(): + self.con.execute("select agg(y = 'q' collate mycoll) from t") + + def test_window_function_value_not_called(self): + events = self.events + class Window: + def step(self, value): + events.append("step") + def inverse(self, value): + events.append("inverse") + def value(self): + events.append("value") + return 0 + def finalize(self): + return 0 + self.con.create_window_function("win", 1, Window) + # For this query, SQLite evaluates the filter for all rows before it + # calls the window function. The filter is false for the first row, + # so the first call after the exception is value(), before step(). + with self.assertCollationRaises(): + self.con.execute(""" + select win(x) filter (where y = 'q' collate mycoll) + over (order by rowid) from t + """) + self.assertEqual(self.events_after_exception(), []) + + def test_window_function_inverse_not_called(self): + events = self.events + class Window: + def step(self, value): + events.append("step") + def inverse(self, value): + events.append("inverse") + def value(self): + events.append("value") + return 0 + def finalize(self): + return 0 + self.con.create_window_function("win", 1, Window) + self.con.executemany("insert into t values (?, ?)", + [(3, "c"), (4, "d")]) + self.con.execute("create table u(w, y)") + # SQLite uses the collation when it adds a row to the index. For the + # next row, the first call is inverse(). + self.con.execute("create index u_y on u(y collate mycoll)") + with self.assertCollationRaises(): + self.con.execute(""" + insert into u + select win(x) over (order by rowid + rows between 1 preceding and current row), + y + from t order by rowid + """) + self.assertEqual(self.events_after_exception(), []) + + def test_authorizer_not_called(self): + def authorizer(action, *args): + self.events.append(action) + return sqlite.SQLITE_OK + sql = """ + select t.x, p.name from t join pragma_table_info('t') p + where t.y = 'q' collate mycoll + """ + self.con.executemany("insert into t values (?, ?)", [(3, "c")]) + self.con.set_authorizer(authorizer) + # Control: if the comparison is true, SQLite prepares a statement for + # pragma_table_info() during the step, and it calls the authorizer. + self.con.create_collation("mycoll", lambda x, y: 0) + self.con.execute(sql).fetchall() + self.assertIn(sqlite.SQLITE_PRAGMA, self.events) + self.events.clear() + self.setUp_collation() + with self.assertCollationRaises(): + self.con.execute(sql) + self.con.set_authorizer(None) + self.assertEqual(self.events_after_exception(), []) + + def test_prepare(self): + # With STAT4, the query planner calls the collation when SQLite + # prepares the statement. + options = [o for (o,) in self.con.execute("pragma compile_options")] + if "ENABLE_STAT4" not in options: + self.skipTest("requires SQLite with SQLITE_ENABLE_STAT4") + fail = False + def statcoll(x, y): + if fail: + raise KeyError("collation raised") + return (x > y) - (x < y) + self.con.create_collation("statcoll", statcoll) + self.con.execute("create table s(v text collate statcoll)") + self.con.execute("create index s_v on s(v)") + self.con.executemany("insert into s values (?)", + [(str(i % 7),) for i in range(200)]) + self.con.execute("analyze") + fail = True + with self.assertCollationRaises(): + self.con.execute("select * from s where v = '3'") + + def test_progress_handler_not_called(self): + def progress(): + self.events.append("progress") + return 0 + self.con.set_progress_handler(progress, 1) + with self.assertCollationRaises(): + self.con.execute(self.step_succeeds + " desc") + self.con.set_progress_handler(None, 1) + self.assertIn("progress", self.events) + self.assertEqual(self.events_after_exception(), []) + + def test_trace_callback_not_called(self): + self.con.execute("create table u(x)") + # SQLite calls the trace callback again for each trigger program. + self.con.execute("create trigger u_tr after insert on u " + "begin select 1; end") + traced = [] + sql = ("insert into u select y = 'q' collate mycoll from t " + "order by rowid desc") + self.con.set_trace_callback(traced.append) + with self.assertCollationRaises(): + self.con.execute(sql) + self.con.set_trace_callback(None) + self.assertEqual(traced, [sql]) + + class AuthorizerTests(MemoryDatabaseMixin, unittest.TestCase): def assert_not_authorized(self, func, /, *args, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2026-11-11-21-00-00.gh-issue-161837.Kq3sXv.rst b/Misc/NEWS.d/next/Library/2026-11-11-21-00-00.gh-issue-161837.Kq3sXv.rst new file mode 100644 index 000000000000000..db6177611598fc6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-11-21-00-00.gh-issue-161837.Kq3sXv.rst @@ -0,0 +1,7 @@ +Fix a :exc:`SystemError` and crashes in :mod:`sqlite3` when a collation +callback raises an exception. Now the exception from the collation +propagates. Before, :meth:`~sqlite3.Cursor.execute` raised +:exc:`~sqlite3.OperationalError` instead if the same step also failed. While +the exception is set, the module no longer calls user-defined functions, the +authorizer, the progress handler or the trace callback. It calls only the +``finalize()`` method of an aggregate, to release the aggregate. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index f8bd3a8df35b668..74c6e6aa88567a7 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -960,10 +960,33 @@ set_sqlite_error(sqlite3_context *context, const char *msg) print_or_clear_traceback(ctx); } +/* A callback can leave an exception set until sqlite3_step() returns. For + * example, a collation callback cannot report an error to SQLite (see + * collation_callback()). Other callbacks in the same step must not call + * Python code while the exception is set. Return 1 if an exception is set. + * If context is not NULL, also make the SQL function fail, so that SQLite + * stops the step. */ +static int +exception_is_pending(sqlite3_context *context) +{ + if (!PyErr_Occurred()) { + return 0; + } + if (context != NULL) { + sqlite3_result_error(context, + "an earlier callback raised an exception", -1); + } + return 1; +} + static void func_callback(sqlite3_context *context, int argc, sqlite3_value **argv) { PyGILState_STATE threadstate = PyGILState_Ensure(); + if (exception_is_pending(context)) { + PyGILState_Release(threadstate); + return; + } PyObject* args; PyObject* py_retval = NULL; @@ -994,6 +1017,10 @@ static void step_callback(sqlite3_context *context, int argc, sqlite3_value **params) { PyGILState_STATE threadstate = PyGILState_Ensure(); + if (exception_is_pending(context)) { + PyGILState_Release(threadstate); + return; + } PyObject* args; PyObject* function_result = NULL; @@ -1084,7 +1111,6 @@ final_callback(sqlite3_context *context) } if (!ok) { int attr_err = PyErr_ExceptionMatches(PyExc_AttributeError); - _PyErr_ChainExceptions1(exc); /* Note: contrary to the step, value, and inverse callbacks, SQLite * does _not_, as of SQLite 3.38.0, propagate errors to sqlite3_step() @@ -1094,9 +1120,8 @@ final_callback(sqlite3_context *context) ? "user-defined aggregate's 'finalize' method not defined" : "user-defined aggregate's 'finalize' method raised error"); } - else { - PyErr_SetRaisedException(exc); - } + // Set the exception of the earlier callback again, so that it propagates. + PyErr_SetRaisedException(exc); error: PyGILState_Release(threadstate); @@ -1257,6 +1282,10 @@ static void inverse_callback(sqlite3_context *context, int argc, sqlite3_value **params) { PyGILState_STATE gilstate = PyGILState_Ensure(); + if (exception_is_pending(context)) { + PyGILState_Release(gilstate); + return; + } callback_context *ctx = (callback_context *)sqlite3_user_data(context); incref_callback_context(ctx); @@ -1306,6 +1335,10 @@ static void value_callback(sqlite3_context *context) { PyGILState_STATE gilstate = PyGILState_Ensure(); + if (exception_is_pending(context)) { + PyGILState_Release(gilstate); + return; + } callback_context *ctx = (callback_context *)sqlite3_user_data(context); incref_callback_context(ctx); @@ -1454,6 +1487,10 @@ authorizer_callback(void *ctx_vp, int action, const char *arg1, const char *access_attempt_source) { PyGILState_STATE gilstate = PyGILState_Ensure(); + if (exception_is_pending(NULL)) { + PyGILState_Release(gilstate); + return SQLITE_DENY; + } PyObject *ret; int rc = SQLITE_DENY; @@ -1491,6 +1528,11 @@ static int progress_callback(void *ctx_vp) { PyGILState_STATE gilstate = PyGILState_Ensure(); + if (exception_is_pending(NULL)) { + /* Stop the step. */ + PyGILState_Release(gilstate); + return 1; + } int rc; PyObject *ret; @@ -1530,6 +1572,10 @@ trace_callback(unsigned int type, void *ctx_vp, void *stmt, void *sql) } PyGILState_STATE gilstate = PyGILState_Ensure(); + if (exception_is_pending(NULL)) { + PyGILState_Release(gilstate); + return 0; + } callback_context *ctx = (callback_context *)ctx_vp; incref_callback_context(ctx); diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index ad8c434c1b2fb8f..23395bc9ca31d39 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -541,6 +541,10 @@ get_statement_from_cache(pysqlite_Cursor *self, PyObject *operation) return PyObject_Vectorcall(cache, args + 1, nargsf, NULL); } +/* A callback can leave an exception set until sqlite3_step() returns. For + * example, a collation callback cannot report an error to SQLite, so SQLite + * continues the step with a wrong result of the comparison. Thus, after a + * step, check PyErr_Occurred() first, and propagate that exception. */ static inline int stmt_step(sqlite3_stmt *statement) { @@ -910,15 +914,11 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation } rc = stmt_step(self->statement->st); + if (PyErr_Occurred()) { + /* A callback left an exception set. */ + goto error; + } if (rc != SQLITE_DONE && rc != SQLITE_ROW) { - if (PyErr_Occurred()) { - /* there was an error that occurred in a user-defined callback */ - if (state->enable_callback_tracebacks) { - PyErr_Print(); - } else { - PyErr_Clear(); - } - } set_error_from_db(state, self->connection->db); goto error; } @@ -1125,6 +1125,10 @@ pysqlite_cursor_executescript_impl(pysqlite_Cursor *self, } Py_END_ALLOW_THREADS + if (PyErr_Occurred()) { + /* A callback left an exception set. */ + return NULL; + } if (rc != SQLITE_OK) { goto error; } @@ -1166,6 +1170,13 @@ pysqlite_cursor_iternext(PyObject *op) return NULL; } int rc = stmt_step(stmt); + if (PyErr_Occurred()) { + /* A callback left an exception set. */ + (void)stmt_reset(self->statement); + Py_CLEAR(self->statement); + Py_DECREF(row); + return NULL; + } if (rc == SQLITE_DONE) { if (self->statement->is_dml) { self->rowcount = (long)sqlite3_changes(self->connection->db); diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index f31c699482f6014..16aaf0b085a1c48 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -68,6 +68,12 @@ pysqlite_statement_create(pysqlite_Connection *connection, PyObject *sql) rc = sqlite3_prepare_v2(db, sql_cstr, (int)size + 1, &stmt, &tail); Py_END_ALLOW_THREADS + if (PyErr_Occurred()) { + /* A callback left an exception set. For example, the query planner + * can call a collation. */ + (void)sqlite3_finalize(stmt); + return NULL; + } if (rc != SQLITE_OK) { set_error_from_db(state, db); return NULL;