Implement synchronous fast path

inc/ocf_mngt.h

@@ -285,9 +285,10 @@ struct ocf_mngt_cache_config {
 	bool pt_unaligned_io;
 
 	/**
-	 * @brief If set, try to submit all I/O in fast path.
+	 * @brief If set, try to handle I/O synchronously in fast path
+	 *	before falling back to the regular asynchronous path.
 	 */
-	bool use_submit_io_fast;
+	bool use_sync_fastpath;
 
 	/**
 	 * @brief Backfill configuration
@@ -317,7 +318,7 @@ static inline void ocf_mngt_cache_config_set_default(
 	cfg->backfill.queue_unblock_size = 60000;
 	cfg->locked = false;
 	cfg->pt_unaligned_io = false;
-	cfg->use_submit_io_fast = false;
+	cfg->use_sync_fastpath = true;
 }
 
 /**

src/concurrency/ocf_cache_line_concurrency.c

@@ -191,6 +191,16 @@ int ocf_req_async_lock_wr(struct ocf_alock *alock,
 	return ocf_alock_lock_wr(alock, req, cmpl);
 }
 
+bool ocf_req_trylock_rd(struct ocf_alock *alock, struct ocf_request *req)
+{
+	return ocf_alock_trylock_rd(alock, req);
+}
+
+bool ocf_req_trylock_wr(struct ocf_alock *alock, struct ocf_request *req)
+{
+	return ocf_alock_trylock_wr(alock, req);
+}
+
 void ocf_req_unlock_rd(struct ocf_alock *alock, struct ocf_request *req)
 {
 	int32_t i;

src/concurrency/ocf_cache_line_concurrency.h

@@ -1,6 +1,7 @@
 /*
  * Copyright(c) 2012-2021 Intel Corporation
  * Copyright(c) 2024 Huawei Technologies
+ * Copyright(c) 2026 Unvertical
  * SPDX-License-Identifier: BSD-3-Clause
  */
 
@@ -85,6 +86,30 @@ int ocf_req_async_lock_wr(struct ocf_alock *c,
 int ocf_req_async_lock_rd(struct ocf_alock *c,
 		struct ocf_request *req, ocf_req_async_lock_cb cmpl);
 
+/**
+ * @brief Try to lock OCF request for read access without waiting
+ *
+ * @param c - cacheline concurrency private data
+ * @param req - OCF request
+ *
+ * @retval true - all the request's cache lines have been locked
+ * @retval false - lock not acquired, nothing is locked and the request was
+ *		not added to any waiting list
+ */
+bool ocf_req_trylock_rd(struct ocf_alock *c, struct ocf_request *req);
+
+/**
+ * @brief Try to lock OCF request for write access without waiting
+ *
+ * @param c - cacheline concurrency private data
+ * @param req - OCF request
+ *
+ * @retval true - all the request's cache lines have been locked
+ * @retval false - lock not acquired, nothing is locked and the request was
+ *		not added to any waiting list
+ */
+bool ocf_req_trylock_wr(struct ocf_alock *c, struct ocf_request *req);
+
 /**
  * @brief Unlock OCF request from write access
  *

src/engine/cache_engine.c

@@ -19,7 +19,6 @@
 #include "engine_wa.h"
 #include "engine_wb.h"
 #include "engine_wo.h"
-#include "engine_fast.h"
 #include "engine_flush.h"
 #include "engine_discard.h"
 #include "../utils/utils_user_part.h"
@@ -38,7 +37,6 @@ enum ocf_io_if_type {
 	OCF_IO_MAX_IF,
 
 	/* Private OCF interfaces */
-	OCF_IO_FAST_IF,
 	OCF_IO_FLUSH_IF,
 	OCF_IO_DISCARD_IF,
 	OCF_IO_PRIV_MAX_IF,
@@ -50,49 +48,66 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = {
 			[OCF_READ] = ocf_read_generic,
 			[OCF_WRITE] = ocf_write_wt,
 		},
+		.fast = {
+			[OCF_READ] = ocf_read_generic_fast,
+			[OCF_WRITE] = ocf_write_wt_fast,
+		},
 		.name = "Write Through"
 	},
 	[OCF_IO_WB_IF] = {
 		.cbs = {
 			[OCF_READ] = ocf_read_generic,
 			[OCF_WRITE] = ocf_write_wb,
 		},
+		.fast = {
+			[OCF_READ] = ocf_read_generic_fast,
+			[OCF_WRITE] = ocf_write_wb_fast,
+		},
 		.name = "Write Back"
 	},
 	[OCF_IO_WA_IF] = {
 		.cbs = {
 			[OCF_READ] = ocf_read_generic,
 			[OCF_WRITE] = ocf_write_wa,
 		},
+		.fast = {
+			[OCF_READ] = ocf_read_generic_fast,
+			[OCF_WRITE] = ocf_write_wa_fast,
+		},
 		.name = "Write Around"
 	},
 	[OCF_IO_WI_IF] = {
 		.cbs = {
 			[OCF_READ] = ocf_read_generic,
 			[OCF_WRITE] = ocf_write_wi,
 		},
+		.fast = {
+			[OCF_READ] = ocf_read_generic_fast,
+			[OCF_WRITE] = ocf_write_wi_fast,
+		},
 		.name = "Write Invalidate"
 	},
 	[OCF_IO_PT_IF] = {
 		.cbs = {
 			[OCF_READ] = ocf_read_pt,
 			[OCF_WRITE] = ocf_write_wi,
 		},
+		.fast = {
+			[OCF_READ] = ocf_read_pt_fast,
+			[OCF_WRITE] = ocf_write_wi_fast,
+		},
 		.name = "Pass Through",
 	},
 	[OCF_IO_WO_IF] = {
 		.cbs = {
 			[OCF_READ] = ocf_read_wo,
 			[OCF_WRITE] = ocf_write_wb,
 		},
-		.name = "Write Only",
-	},
-	[OCF_IO_FAST_IF] = {
-		.cbs = {
-			[OCF_READ] = ocf_read_fast,
-			[OCF_WRITE] = ocf_write_fast,
+		.fast = {
+			[OCF_READ] = ocf_read_wo_fast,
+			[OCF_WRITE] = ocf_write_wb_fast,
 		},
-		.name = "Fast",
+		.name = "Write Only",
 	},
 	[OCF_IO_FLUSH_IF] = {
 		.cbs = {
@@ -117,7 +132,6 @@ static const struct ocf_io_if *cache_mode_io_if_map[ocf_req_cache_mode_max] = {
 	[ocf_req_cache_mode_wi] = &IO_IFS[OCF_IO_WI_IF],
 	[ocf_req_cache_mode_wo] = &IO_IFS[OCF_IO_WO_IF],
 	[ocf_req_cache_mode_pt] = &IO_IFS[OCF_IO_PT_IF],
-	[ocf_req_cache_mode_fast] = &IO_IFS[OCF_IO_FAST_IF],
 };
 
 const char *ocf_get_io_iface_name(ocf_req_cache_mode_t cache_mode)
@@ -221,23 +235,21 @@ int ocf_engine_hndl_req(struct ocf_request *req)
 	return 0;
 }
 
-int ocf_engine_hndl_fast_req(struct ocf_request *req)
-{
-	ocf_req_cb engine_cb;
-	int ret;
-
-	engine_cb = ocf_cache_mode_to_engine_cb(req->cache_mode, req->rw);
-	if (!engine_cb)
-		return -OCF_ERR_INVAL;
-
-	ocf_req_get(req);
+/*
+ * Larger requests hold the hash bucket lock for long and are likely to fail
+ * the fast path requirements anyway.
+ */
+#define OCF_ENGINE_FAST_MAX_LINES 64
 
-	ret = engine_cb(req);
+bool ocf_engine_hndl_fast_req(struct ocf_request *req)
+{
+	if (req->core_line_count > OCF_ENGINE_FAST_MAX_LINES)
+		return false;
 
-	if (ret == OCF_FAST_PATH_NO)
-		ocf_req_put(req);
+	if (req->cache_mode == ocf_req_cache_mode_max)
+		return false;
 
-	return ret;
+	return cache_mode_io_if_map[req->cache_mode]->fast[req->rw](req);
 }
 
 void ocf_engine_hndl_discard_req(struct ocf_request *req)

src/engine/cache_engine.h

@@ -23,9 +23,13 @@ static inline ocf_req_cache_mode_t ocf_cache_mode_to_req_cache_mode(
 	return (ocf_req_cache_mode_t)mode;
 }
 
+typedef bool (*ocf_req_fast_cb)(struct ocf_request *req);
+
 struct ocf_io_if {
 	ocf_req_cb cbs[2]; /* READ and WRITE */
 
+	ocf_req_fast_cb fast[2]; /* READ and WRITE */
+
 	const char *name;
 };
 
@@ -40,10 +44,7 @@ bool ocf_fallback_pt_is_on(ocf_cache_t cache);
 
 int ocf_engine_hndl_req(struct ocf_request *req);
 
-#define OCF_FAST_PATH_YES	7
-#define OCF_FAST_PATH_NO	13
-
-int ocf_engine_hndl_fast_req(struct ocf_request *req);
+bool ocf_engine_hndl_fast_req(struct ocf_request *req);
 
 void ocf_engine_hndl_flush_req(struct ocf_request *req);