MDEV-39834: Add TidesDB 9(v9.3.3) storage engine

debian/control

@@ -985,6 +985,18 @@ Description: Amazon S3 archival storage engine for MariaDB
  third-party public or private cloud that implements S3 API), but still have
  them accessible in MariaDB in read-only mode.
 
+Package: mariadb-plugin-tidesdb
+Architecture: any
+Depends: mariadb-server (= ${server:Version}),
+         ${misc:Depends},
+         ${shlibs:Depends}
+Description: TidesDB storage engine for MariaDB server
+ TidesDB is an LSM B+-tree storage engine with ACID transactions, MVCC,
+ configurable compression (zstd/lz4/snappy), per-row and table TTL,
+ full-text, spatial and vector indexes, and an optional S3-compatible
+ object store backend.
+ This package contains the TidesDB plugin for MariaDB server.
+
 Package: mariadb-plugin-rocksdb
 Architecture: amd64 arm64 mips64el ppc64el riscv64
 Depends: mariadb-server (= ${server:Version}),

debian/mariadb-plugin-tidesdb.install

@@ -0,0 +1,2 @@
+etc/mysql/mariadb.conf.d/tidesdb.cnf
+usr/lib/mysql/plugin/ha_tidesdb.so

mysql-test/suite/tidesdb/include/cleanup_tidesdb.inc

@@ -0,0 +1,3 @@
+disable_query_log;
+ALTER DATABASE test DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_uca1400_ai_ci;
+enable_query_log;

mysql-test/suite/tidesdb/include/have_tidesdb.inc

@@ -0,0 +1,7 @@
+disable_query_log;
+--error 0,1286
+eval SET @@default_storage_engine = TidesDB;
+ALTER DATABASE test DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
+call mtr.add_suppression("Plugin 'TIDESDB' is of maturity level gamma");
+call mtr.add_suppression("TIDESDB: hton_commit: tidesdb_txn_commit returned");
+enable_query_log;

mysql-test/suite/tidesdb/include/have_tidesdb_vector.inc

@@ -0,0 +1,15 @@
+# Skip the test unless the server supports the VECTOR data type (MariaDB 11.7+).
+# TidesDB itself implements vector indexes, but the SQL-layer VECTOR type does
+# not exist on older servers, so CREATE TABLE ... VECTOR fails there.
+--disable_query_log
+--disable_warnings
+--error 0,ER_UNKNOWN_DATA_TYPE,ER_PARSE_ERROR,ER_NOT_SUPPORTED_YET
+CREATE TEMPORARY TABLE tdb_vector_probe (v VECTOR(4) NOT NULL);
+let $tdb_no_vector= $mysql_errno;
+DROP TEMPORARY TABLE IF EXISTS tdb_vector_probe;
+--enable_warnings
+--enable_query_log
+if ($tdb_no_vector)
+{
+  --skip TidesDB: VECTOR data type not supported (requires MariaDB 11.7+)
+}

mysql-test/suite/tidesdb/r/tidesdb_alter_large_table.result

@@ -0,0 +1,54 @@
+#
+# Large-table ALTER under REPEATABLE_READ.
+#
+# Copy-phase ALTER scans every row of the source table into the
+# rebuilt table while a single REPEATABLE_READ transaction is open
+# (autocommit=0 forces this), so the engine must keep the read-set
+# bookkeeping bounded as the scan grows.  Unbounded growth here
+# used to crash the server inside tidesdb_txn_add_to_read_set.
+# The test asserts that the scan completes, the rebuild commits,
+# and the row count is preserved.
+#
+CREATE TABLE t_alter_big (
+a INT AUTO_INCREMENT PRIMARY KEY,
+b INT
+) ENGINE=TidesDB;
+INSERT INTO t_alter_big (a, b) VALUES (DEFAULT, 10), (DEFAULT, 20), (DEFAULT, 30);
+# Double the rows repeatedly to get ~100K rows
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+INSERT INTO t_alter_big (b) SELECT b FROM t_alter_big;
+SELECT COUNT(*) FROM t_alter_big;
+COUNT(*)
+98304
+# autocommit=0 makes the surrounding session use REPEATABLE_READ,
+# which is the isolation that loaded the read-set during ALTER.
+SET autocommit=0;
+# Sanity-check ALTER's error reporting on contradictory key DDL.
+ALTER TABLE t_alter_big ADD PRIMARY KEY (a);
+ERROR 42000: Multiple primary key defined
+ALTER TABLE t_alter_big DROP PRIMARY KEY;
+ERROR 42000: Incorrect table definition; there can be only one auto column and it must be defined as a key
+# Copy-based ALTER over ~100K rows under REPEATABLE_READ.  Must
+# complete cleanly without exhausting memory or crashing the
+# server in the read-set machinery.
+ALTER TABLE t_alter_big DROP PRIMARY KEY, CHANGE a a INT;
+SELECT COUNT(*) FROM t_alter_big;
+COUNT(*)
+98304
+SET autocommit=1;
+DROP TABLE t_alter_big;
+#
+# Done.

mysql-test/suite/tidesdb/r/tidesdb_analyze.result

@@ -0,0 +1,47 @@
+#
+# ANALYZE TABLE for TidesDB -- verifies CF stats output
+#
+CREATE TABLE t1 (
+id INT PRIMARY KEY,
+val VARCHAR(40),
+KEY idx_val (val)
+) ENGINE=TidesDB;
+INSERT INTO t1 VALUES (1, 'alpha'), (2, 'bravo'), (3, 'charlie'),
+(4, 'delta'), (5, 'echo'),  (6, 'foxtrot');
+# ANALYZE TABLE should return status OK and emit CF stats as notes.
+# Mask volatile numeric values (memtable size, avg sizes, etc.)
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	Note	[TIDESDB] CF 'test__t1'  total_keys=N  data_size=N bytes  memtable=N bytes  levels=1  read_amp=N  cache_hit=N%
+test.t1	analyze	Note	[TIDESDB] avg_key=N bytes  avg_value=N bytes
+test.t1	analyze	Note	[TIDESDB] level 1  sstables=N  size=N bytes  keys=N
+test.t1	analyze	Note	[TIDESDB] idx CF 'test__t1__idx_idx_val'  keys=N  data_size=N bytes  levels=1
+test.t1	analyze	Note	[TIDESDB] idx 'idx_val' sampled=6 distinct=6 rec_per_key=1
+test.t1	analyze	status	OK
+# ANALYZE a table without secondary indexes
+CREATE TABLE t2 (
+id INT PRIMARY KEY,
+data VARCHAR(200)
+) ENGINE=TidesDB;
+INSERT INTO t2 VALUES (1, REPEAT('x', 100)), (2, REPEAT('y', 100));
+ANALYZE TABLE t2;
+Table	Op	Msg_type	Msg_text
+test.t2	analyze	status	Engine-independent statistics collected
+test.t2	analyze	Note	[TIDESDB] CF 'test__t2'  total_keys=N  data_size=N bytes  memtable=N bytes  levels=1  read_amp=N  cache_hit=N%
+test.t2	analyze	Note	[TIDESDB] avg_key=N bytes  avg_value=N bytes
+test.t2	analyze	Note	[TIDESDB] level 1  sstables=N  size=N bytes  keys=N
+test.t2	analyze	status	OK
+# ANALYZE an empty table
+CREATE TABLE t3 (
+id INT PRIMARY KEY
+) ENGINE=TidesDB;
+ANALYZE TABLE t3;
+Table	Op	Msg_type	Msg_text
+test.t3	analyze	status	Engine-independent statistics collected
+test.t3	analyze	Note	[TIDESDB] CF 'test__t3'  total_keys=N  data_size=N bytes  memtable=N bytes  levels=1  read_amp=N  cache_hit=N%
+test.t3	analyze	Note	[TIDESDB] avg_key=N bytes  avg_value=N bytes
+test.t3	analyze	Note	[TIDESDB] level 1  sstables=N  size=N bytes  keys=N
+test.t3	analyze	status	OK
+# Cleanup
+DROP TABLE t1, t2, t3;

mysql-test/suite/tidesdb/r/tidesdb_auto_increment.result

@@ -0,0 +1,112 @@
+#
+# TEST 1: Basic auto-increment
+#
+CREATE TABLE t_ai (id INT AUTO_INCREMENT PRIMARY KEY, v VARCHAR(50)) ENGINE=TidesDB;
+INSERT INTO t_ai (v) VALUES ('a'), ('b'), ('c');
+SELECT * FROM t_ai ORDER BY id;
+id	v
+1	a
+2	b
+3	c
+#
+# TEST 2: Explicit value larger than counter
+#
+INSERT INTO t_ai VALUES (100, 'explicit');
+INSERT INTO t_ai (v) VALUES ('after_explicit');
+SELECT * FROM t_ai ORDER BY id;
+id	v
+1	a
+2	b
+3	c
+100	explicit
+101	after_explicit
+#
+# TEST 3: Gap after rollback
+#
+BEGIN;
+INSERT INTO t_ai (v) VALUES ('will_rollback');
+SELECT MAX(id) FROM t_ai;
+MAX(id)
+102
+ROLLBACK;
+INSERT INTO t_ai (v) VALUES ('after_rollback');
+SELECT id, v FROM t_ai WHERE v IN ('after_rollback', 'after_explicit') ORDER BY id;
+id	v
+101	after_explicit
+103	after_rollback
+#
+# TEST 4: LAST_INSERT_ID
+#
+INSERT INTO t_ai (v) VALUES ('last_id_test');
+SELECT LAST_INSERT_ID() > 0 AS has_last_id;
+has_last_id
+1
+#
+# TEST 5: Auto-increment with REPLACE INTO
+#
+CREATE TABLE t_ai_replace (
+id INT AUTO_INCREMENT PRIMARY KEY,
+name VARCHAR(50) UNIQUE
+) ENGINE=TidesDB;
+INSERT INTO t_ai_replace (name) VALUES ('x'), ('y'), ('z');
+REPLACE INTO t_ai_replace (name) VALUES ('y');
+SELECT * FROM t_ai_replace ORDER BY name;
+id	name
+1	x
+4	y
+3	z
+#
+# TEST 5b: an auto-increment PK must not bypass the UNIQUE secondary check
+#
+INSERT INTO t_ai_replace (name) VALUES ('z');
+ERROR 23000: Duplicate entry 'z' for key 'name'
+INSERT INTO t_ai_replace (name) VALUES ('x')
+ON DUPLICATE KEY UPDATE name = 'x2';
+SELECT * FROM t_ai_replace ORDER BY name;
+id	name
+1	x2
+4	y
+3	z
+# no value may appear twice in the UNIQUE column
+SELECT name, COUNT(*) AS c FROM t_ai_replace GROUP BY name HAVING c > 1;
+name	c
+#
+# TEST 6: BIGINT auto-increment
+#
+CREATE TABLE t_ai_big (id BIGINT AUTO_INCREMENT PRIMARY KEY, v INT) ENGINE=TidesDB;
+INSERT INTO t_ai_big (v) VALUES (1), (2), (3);
+INSERT INTO t_ai_big VALUES (9999999999, 4);
+INSERT INTO t_ai_big (v) VALUES (5);
+SELECT * FROM t_ai_big ORDER BY id;
+id	v
+1	1
+2	2
+3	3
+9999999999	4
+10000000000	5
+#
+# TEST 7: Auto-increment after TRUNCATE resets counter
+#
+TRUNCATE TABLE t_ai;
+INSERT INTO t_ai (v) VALUES ('fresh_start');
+SELECT * FROM t_ai;
+id	v
+1	fresh_start
+#
+# TEST 8: ALTER TABLE ... AUTO_INCREMENT=N takes effect
+#
+CREATE TABLE t_ai_alter (id INT AUTO_INCREMENT PRIMARY KEY, v VARCHAR(10)) ENGINE=TidesDB;
+INSERT INTO t_ai_alter (v) VALUES ('a'), ('b');
+ALTER TABLE t_ai_alter AUTO_INCREMENT=1000;
+INSERT INTO t_ai_alter (v) VALUES ('jumped');
+SELECT * FROM t_ai_alter ORDER BY id;
+id	v
+1	a
+2	b
+3	jumped
+DROP TABLE t_ai_alter;
+#
+# Cleanup
+#
+DROP TABLE t_ai, t_ai_replace, t_ai_big;
+# Done.

mysql-test/suite/tidesdb/r/tidesdb_backup.result

@@ -0,0 +1,73 @@
+CALL mtr.add_suppression("\\[TIDESDB\\] Backup to .* failed");
+#
+# ============================================
+# TEST 1: Online backup creates a valid copy
+# ============================================
+#
+CREATE TABLE t_backup (
+id INT PRIMARY KEY,
+val VARCHAR(100)
+) ENGINE=TIDESDB;
+INSERT INTO t_backup VALUES (1, 'alpha'), (2, 'beta'), (3, 'gamma');
+SELECT * FROM t_backup ORDER BY id;
+id	val
+1	alpha
+2	beta
+3	gamma
+# Triggering online backup
+# Backup should have created the directory
+Backup directory exists: YES
+# Check that SHOW VARIABLES reflects the backup path
+SELECT @@GLOBAL.tidesdb_backup_dir IS NOT NULL AS backup_dir_set;
+backup_dir_set
+1
+# Insert more data after backup (should NOT appear in backup)
+INSERT INTO t_backup VALUES (4, 'delta'), (5, 'epsilon');
+SELECT COUNT(*) AS rows_after FROM t_backup;
+rows_after
+5
+DROP TABLE t_backup;
+#
+# ============================================
+# TEST 2: Backup to existing non-empty dir fails
+# ============================================
+#
+# Re-running backup to same directory should fail (not empty)
+SET GLOBAL tidesdb_backup_dir = 'MYSQLTEST_VARDIR/tmp/tidesdb_backup_test';
+ERROR HY000: [TIDESDB] Backup to 'MYSQLTEST_VARDIR/tmp/tidesdb_backup_test' failed (err=-6)
+#
+# ============================================
+# TEST 3: Clear backup_dir variable
+# ============================================
+#
+SET GLOBAL tidesdb_backup_dir = '';
+SELECT @@GLOBAL.tidesdb_backup_dir IS NULL AS backup_dir_cleared;
+backup_dir_cleared
+1
+#
+# ============================================
+# TEST 4: Concurrent reads/writes during backup
+# ============================================
+#
+CREATE TABLE t_concurrent (
+id INT PRIMARY KEY,
+data VARCHAR(200)
+) ENGINE=TIDESDB;
+# Inserted 100 rows
+SELECT COUNT(*) AS before_backup FROM t_concurrent;
+before_backup
+100
+# Backup completed while table was loaded
+SELECT COUNT(*) AS after_backup FROM t_concurrent;
+after_backup
+100
+INSERT INTO t_concurrent VALUES (101, 'post-backup');
+SELECT COUNT(*) AS with_post_backup FROM t_concurrent;
+with_post_backup
+101
+DROP TABLE t_concurrent;
+#
+# === Cleanup ===
+#
+SET GLOBAL tidesdb_backup_dir = '';
+# Done.

mysql-test/suite/tidesdb/r/tidesdb_bulk_commit_durability.result

@@ -0,0 +1,36 @@
+DROP TABLE IF EXISTS bulk_src;
+DROP TABLE IF EXISTS bulk_dst;
+CREATE TABLE bulk_src (
+id      INT PRIMARY KEY,
+payload VARCHAR(200)
+) ENGINE=TIDESDB;
+CREATE TABLE bulk_dst (
+id      INT PRIMARY KEY,
+payload VARCHAR(200)
+) ENGINE=TIDESDB;
+SELECT COUNT(*) AS src_rows FROM bulk_src;
+src_rows
+1000
+#
+# Run 50 bulk INSERT ... SELECT statements (50,000 rows total).
+# Each statement crosses the bulk-commit threshold, exercising
+# the maybe_bulk_commit() path that previously swallowed errors.
+#
+#
+# Assertion: every row from every batch must be present.  If
+# maybe_bulk_commit() ever swallows an inner commit failure again,
+# this verdict line will read "LOST <N> rows" instead of "OK".
+#
+SELECT IF(COUNT(*) = 50000,
+'OK',
+CONCAT('LOST ', 50000 - COUNT(*), ' rows of 50000'))
+AS verdict
+FROM bulk_dst;
+verdict
+OK
+SELECT COUNT(*) AS dst_rows, MIN(id) AS min_id, MAX(id) AS max_id FROM bulk_dst;
+dst_rows	min_id	max_id
+50000	1	50000
+DROP TABLE bulk_src;
+DROP TABLE bulk_dst;
+# Done.