diff --git a/inc/ocf_mngt.h b/inc/ocf_mngt.h index 2674cdda..2b3bbe96 100644 --- a/inc/ocf_mngt.h +++ b/inc/ocf_mngt.h @@ -285,9 +285,10 @@ struct ocf_mngt_cache_config { bool pt_unaligned_io; /** - * @brief If set, try to submit all I/O in fast path. + * @brief If set, try to handle I/O synchronously in fast path + * before falling back to the regular asynchronous path. */ - bool use_submit_io_fast; + bool use_sync_fastpath; /** * @brief Backfill configuration @@ -317,7 +318,7 @@ static inline void ocf_mngt_cache_config_set_default( cfg->backfill.queue_unblock_size = 60000; cfg->locked = false; cfg->pt_unaligned_io = false; - cfg->use_submit_io_fast = false; + cfg->use_sync_fastpath = true; } /** diff --git a/src/concurrency/ocf_cache_line_concurrency.c b/src/concurrency/ocf_cache_line_concurrency.c index 3729f13c..23fef428 100644 --- a/src/concurrency/ocf_cache_line_concurrency.c +++ b/src/concurrency/ocf_cache_line_concurrency.c @@ -191,6 +191,16 @@ int ocf_req_async_lock_wr(struct ocf_alock *alock, return ocf_alock_lock_wr(alock, req, cmpl); } +bool ocf_req_trylock_rd(struct ocf_alock *alock, struct ocf_request *req) +{ + return ocf_alock_trylock_rd(alock, req); +} + +bool ocf_req_trylock_wr(struct ocf_alock *alock, struct ocf_request *req) +{ + return ocf_alock_trylock_wr(alock, req); +} + void ocf_req_unlock_rd(struct ocf_alock *alock, struct ocf_request *req) { int32_t i; diff --git a/src/concurrency/ocf_cache_line_concurrency.h b/src/concurrency/ocf_cache_line_concurrency.h index 6ceb3b9a..74f7b665 100644 --- a/src/concurrency/ocf_cache_line_concurrency.h +++ b/src/concurrency/ocf_cache_line_concurrency.h @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2021 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -85,6 +86,30 @@ int ocf_req_async_lock_wr(struct ocf_alock *c, int ocf_req_async_lock_rd(struct ocf_alock *c, struct ocf_request *req, ocf_req_async_lock_cb cmpl); +/** + * @brief Try to lock OCF request for read access without waiting + * + * @param c - cacheline concurrency private data + * @param req - OCF request + * + * @retval true - all the request's cache lines have been locked + * @retval false - lock not acquired, nothing is locked and the request was + * not added to any waiting list + */ +bool ocf_req_trylock_rd(struct ocf_alock *c, struct ocf_request *req); + +/** + * @brief Try to lock OCF request for write access without waiting + * + * @param c - cacheline concurrency private data + * @param req - OCF request + * + * @retval true - all the request's cache lines have been locked + * @retval false - lock not acquired, nothing is locked and the request was + * not added to any waiting list + */ +bool ocf_req_trylock_wr(struct ocf_alock *c, struct ocf_request *req); + /** * @brief Unlock OCF request from write access * diff --git a/src/engine/cache_engine.c b/src/engine/cache_engine.c index 70081e99..c9ca719e 100644 --- a/src/engine/cache_engine.c +++ b/src/engine/cache_engine.c @@ -19,7 +19,6 @@ #include "engine_wa.h" #include "engine_wb.h" #include "engine_wo.h" -#include "engine_fast.h" #include "engine_flush.h" #include "engine_discard.h" #include "../utils/utils_user_part.h" @@ -38,7 +37,6 @@ enum ocf_io_if_type { OCF_IO_MAX_IF, /* Private OCF interfaces */ - OCF_IO_FAST_IF, OCF_IO_FLUSH_IF, OCF_IO_DISCARD_IF, OCF_IO_PRIV_MAX_IF, @@ -50,6 +48,10 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = { [OCF_READ] = ocf_read_generic, [OCF_WRITE] = ocf_write_wt, }, + .fast = { + [OCF_READ] = ocf_read_generic_fast, + [OCF_WRITE] = ocf_write_wt_fast, + }, .name = "Write Through" }, [OCF_IO_WB_IF] = { @@ -57,6 +59,10 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = { [OCF_READ] = ocf_read_generic, [OCF_WRITE] = ocf_write_wb, }, + .fast = { + [OCF_READ] = ocf_read_generic_fast, + [OCF_WRITE] = ocf_write_wb_fast, + }, .name = "Write Back" }, [OCF_IO_WA_IF] = { @@ -64,6 +70,10 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = { [OCF_READ] = ocf_read_generic, [OCF_WRITE] = ocf_write_wa, }, + .fast = { + [OCF_READ] = ocf_read_generic_fast, + [OCF_WRITE] = ocf_write_wa_fast, + }, .name = "Write Around" }, [OCF_IO_WI_IF] = { @@ -71,6 +81,10 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = { [OCF_READ] = ocf_read_generic, [OCF_WRITE] = ocf_write_wi, }, + .fast = { + [OCF_READ] = ocf_read_generic_fast, + [OCF_WRITE] = ocf_write_wi_fast, + }, .name = "Write Invalidate" }, [OCF_IO_PT_IF] = { @@ -78,6 +92,10 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = { [OCF_READ] = ocf_read_pt, [OCF_WRITE] = ocf_write_wi, }, + .fast = { + [OCF_READ] = ocf_read_pt_fast, + [OCF_WRITE] = ocf_write_wi_fast, + }, .name = "Pass Through", }, [OCF_IO_WO_IF] = { @@ -85,14 +103,11 @@ static const struct ocf_io_if IO_IFS[OCF_IO_PRIV_MAX_IF] = { [OCF_READ] = ocf_read_wo, [OCF_WRITE] = ocf_write_wb, }, - .name = "Write Only", - }, - [OCF_IO_FAST_IF] = { - .cbs = { - [OCF_READ] = ocf_read_fast, - [OCF_WRITE] = ocf_write_fast, + .fast = { + [OCF_READ] = ocf_read_wo_fast, + [OCF_WRITE] = ocf_write_wb_fast, }, - .name = "Fast", + .name = "Write Only", }, [OCF_IO_FLUSH_IF] = { .cbs = { @@ -117,7 +132,6 @@ static const struct ocf_io_if *cache_mode_io_if_map[ocf_req_cache_mode_max] = { [ocf_req_cache_mode_wi] = &IO_IFS[OCF_IO_WI_IF], [ocf_req_cache_mode_wo] = &IO_IFS[OCF_IO_WO_IF], [ocf_req_cache_mode_pt] = &IO_IFS[OCF_IO_PT_IF], - [ocf_req_cache_mode_fast] = &IO_IFS[OCF_IO_FAST_IF], }; const char *ocf_get_io_iface_name(ocf_req_cache_mode_t cache_mode) @@ -221,23 +235,21 @@ int ocf_engine_hndl_req(struct ocf_request *req) return 0; } -int ocf_engine_hndl_fast_req(struct ocf_request *req) -{ - ocf_req_cb engine_cb; - int ret; - - engine_cb = ocf_cache_mode_to_engine_cb(req->cache_mode, req->rw); - if (!engine_cb) - return -OCF_ERR_INVAL; - - ocf_req_get(req); +/* + * Larger requests hold the hash bucket lock for long and are likely to fail + * the fast path requirements anyway. + */ +#define OCF_ENGINE_FAST_MAX_LINES 64 - ret = engine_cb(req); +bool ocf_engine_hndl_fast_req(struct ocf_request *req) +{ + if (req->core_line_count > OCF_ENGINE_FAST_MAX_LINES) + return false; - if (ret == OCF_FAST_PATH_NO) - ocf_req_put(req); + if (req->cache_mode == ocf_req_cache_mode_max) + return false; - return ret; + return cache_mode_io_if_map[req->cache_mode]->fast[req->rw](req); } void ocf_engine_hndl_discard_req(struct ocf_request *req) diff --git a/src/engine/cache_engine.h b/src/engine/cache_engine.h index 0bcb0941..cdda2937 100644 --- a/src/engine/cache_engine.h +++ b/src/engine/cache_engine.h @@ -23,9 +23,13 @@ static inline ocf_req_cache_mode_t ocf_cache_mode_to_req_cache_mode( return (ocf_req_cache_mode_t)mode; } +typedef bool (*ocf_req_fast_cb)(struct ocf_request *req); + struct ocf_io_if { ocf_req_cb cbs[2]; /* READ and WRITE */ + ocf_req_fast_cb fast[2]; /* READ and WRITE */ + const char *name; }; @@ -40,10 +44,7 @@ bool ocf_fallback_pt_is_on(ocf_cache_t cache); int ocf_engine_hndl_req(struct ocf_request *req); -#define OCF_FAST_PATH_YES 7 -#define OCF_FAST_PATH_NO 13 - -int ocf_engine_hndl_fast_req(struct ocf_request *req); +bool ocf_engine_hndl_fast_req(struct ocf_request *req); void ocf_engine_hndl_flush_req(struct ocf_request *req); diff --git a/src/engine/engine_fast.c b/src/engine/engine_fast.c deleted file mode 100644 index 5a687ed0..00000000 --- a/src/engine/engine_fast.c +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright(c) 2012-2022 Intel Corporation - * Copyright(c) 2024 Huawei Technologies - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "ocf/ocf.h" -#include "../ocf_cache_priv.h" -#include "engine_fast.h" -#include "engine_common.h" -#include "engine_io.h" -#include "engine_pt.h" -#include "engine_wb.h" -#include "../ocf_request.h" -#include "../utils/utils_user_part.h" -#include "../concurrency/ocf_concurrency.h" -#include "../concurrency/ocf_metadata_concurrency.h" -#include "../metadata/metadata.h" - -#define OCF_ENGINE_DEBUG 0 - -#define OCF_ENGINE_DEBUG_IO_NAME "fast" -#include "engine_debug.h" - -/* _____ _ ______ _ _____ _ _ - * | __ \ | | | ____| | | | __ \ | | | | - * | |__) |___ __ _ __| | | |__ __ _ ___| |_ | |__) |_ _| |_| |__ - * | _ // _ \/ _` |/ _` | | __/ _` / __| __| | ___/ _` | __| '_ \ - * | | \ \ __/ (_| | (_| | | | | (_| \__ \ |_ | | | (_| | |_| | | | - * |_| \_\___|\__,_|\__,_| |_| \__,_|___/\__| |_| \__,_|\__|_| |_| - */ - -static void _ocf_read_fast_complete(struct ocf_request *req, int error) -{ - OCF_DEBUG_RQ(req, "HIT completion"); - - if (error) { - ocf_core_stats_cache_error_update(req->core, OCF_READ); - - OCF_DEBUG_RQ(req, "ERROR"); - - ocf_queue_push_req_pt(req); - } else { - ocf_req_unlock(ocf_cache_line_concurrency(req->cache), req); - - /* Complete request */ - req->complete(req, error); - - /* Free the request at the last point of the completion path */ - ocf_req_put(req); - } -} - -static int _ocf_read_fast_do(struct ocf_request *req) -{ - if (ocf_engine_is_miss(req)) { - /* It seams that after resume, now request is MISS, do PT */ - OCF_DEBUG_RQ(req, "Switching to read PT"); - ocf_read_pt_do(req); - return 0; - - } - - /* Get OCF request - increase reference counter */ - ocf_req_get(req); - - if (ocf_engine_needs_repart(req)) { - OCF_DEBUG_RQ(req, "Re-Part"); - - ocf_hb_req_prot_lock_wr(req); - - /* Probably some cache lines are assigned into wrong - * partition. Need to move it to new one - */ - ocf_user_part_move(req); - - ocf_hb_req_prot_unlock_wr(req); - } - - /* Submit IO */ - OCF_DEBUG_RQ(req, "Submit"); - ocf_engine_forward_cache_io_req(req, OCF_READ, _ocf_read_fast_complete); - - /* Update statistics */ - ocf_engine_update_request_stats(req); - ocf_engine_update_block_stats(req); - - /* Put OCF request - decrease reference counter */ - ocf_req_put(req); - - return 0; -} - -int ocf_read_fast(struct ocf_request *req) -{ - bool hit; - int lock = OCF_LOCK_NOT_ACQUIRED; - bool part_has_space; - - /* Get OCF request - increase reference counter */ - ocf_req_get(req); - - /* Set resume handler */ - req->engine_handler = _ocf_read_fast_do; - - /*- Metadata RD access -----------------------------------------------*/ - - ocf_req_hash(req); - ocf_hb_req_prot_lock_rd(req); - - /* Traverse request to cache if there is hit */ - ocf_engine_traverse(req); - - hit = ocf_engine_is_hit(req); - - part_has_space = ocf_user_part_has_space(req); - - if (hit && part_has_space) { - lock = ocf_req_async_lock_rd( - ocf_cache_line_concurrency(req->cache), - req, ocf_engine_on_resume); - } - - ocf_hb_req_prot_unlock_rd(req); - - if (hit && part_has_space) { - OCF_DEBUG_RQ(req, "Fast path success"); - - if (lock >= 0) { - if (lock != OCF_LOCK_ACQUIRED) { - /* Lock was not acquired, need to wait for resume */ - OCF_DEBUG_RQ(req, "NO LOCK"); - } else { - /* Lock was acquired can perform IO */ - _ocf_read_fast_do(req); - } - } else { - OCF_DEBUG_RQ(req, "LOCK ERROR"); - req->complete(req, lock); - ocf_req_put(req); - } - } else { - OCF_DEBUG_RQ(req, "Fast path failure"); - } - - /* Put OCF request - decrease reference counter */ - ocf_req_put(req); - - return (hit && part_has_space) ? OCF_FAST_PATH_YES : OCF_FAST_PATH_NO; -} - -/* __ __ _ _ ______ _ _____ _ _ - * \ \ / / (_) | | ____| | | | __ \ | | | | - * \ \ /\ / / __ _| |_ ___ | |__ __ _ ___| |_ | |__) |_ _| |_| |__ - * \ \/ \/ / '__| | __/ _ \ | __/ _` / __| __| | ___/ _` | __| '_ \ - * \ /\ /| | | | || __/ | | | (_| \__ \ |_ | | | (_| | |_| | | | - * \/ \/ |_| |_|\__\___| |_| \__,_|___/\__| |_| \__,_|\__|_| |_| - */ - -int ocf_write_fast(struct ocf_request *req) -{ - bool mapped; - int lock = OCF_LOCK_NOT_ACQUIRED; - int part_has_space = false; - - /* Get OCF request - increase reference counter */ - ocf_req_get(req); - - /* Set resume handler */ - req->engine_handler = ocf_write_wb_do; - - /*- Metadata RD access -----------------------------------------------*/ - - ocf_req_hash(req); - ocf_hb_req_prot_lock_rd(req); - - /* Traverse request to cache if there is hit */ - ocf_engine_traverse(req); - - mapped = ocf_engine_is_mapped(req); - - part_has_space = ocf_user_part_has_space(req); - - if (mapped && part_has_space) { - lock = ocf_req_async_lock_wr( - ocf_cache_line_concurrency(req->cache), - req, ocf_engine_on_resume); - } - - ocf_hb_req_prot_unlock_rd(req); - - if (mapped && part_has_space) { - if (lock >= 0) { - OCF_DEBUG_RQ(req, "Fast path success"); - - if (lock != OCF_LOCK_ACQUIRED) { - /* Lock was not acquired, need to wait for resume */ - OCF_DEBUG_RQ(req, "NO LOCK"); - } else { - /* Lock was acquired can perform IO */ - ocf_write_wb_do(req); - } - } else { - OCF_DEBUG_RQ(req, "Fast path lock failure"); - req->complete(req, lock); - ocf_req_put(req); - } - } else { - OCF_DEBUG_RQ(req, "Fast path failure"); - } - - /* Put OCF request - decrease reference counter */ - ocf_req_put(req); - - return (mapped && part_has_space) ? OCF_FAST_PATH_YES : OCF_FAST_PATH_NO; -} diff --git a/src/engine/engine_fast.h b/src/engine/engine_fast.h deleted file mode 100644 index 674d7f3e..00000000 --- a/src/engine/engine_fast.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright(c) 2012-2021 Intel Corporation - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef __ENGINE_FAST_H__ -#define __ENGINE_FAST_H__ - -int ocf_read_fast(struct ocf_request *req); -int ocf_write_fast(struct ocf_request *req); - -#endif /* __ENGINE_FAST_H__ */ diff --git a/src/engine/engine_pt.c b/src/engine/engine_pt.c index 0e62fecc..8d2b11f3 100644 --- a/src/engine/engine_pt.c +++ b/src/engine/engine_pt.c @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2023-2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" @@ -43,23 +44,8 @@ static inline void _ocf_read_pt_submit(struct ocf_request *req) ocf_engine_forward_core_io_req(req, _ocf_read_pt_complete); } -int ocf_read_pt_do(struct ocf_request *req) +static inline void _ocf_read_pt_process(struct ocf_request *req) { - /* Get OCF request - increase reference counter */ - ocf_req_get(req); - - if (req->info.dirty_any) { - ocf_hb_req_prot_lock_rd(req); - /* Need to clean, start it */ - ocf_engine_clean(req); - ocf_hb_req_prot_unlock_rd(req); - - /* Do not processing, because first we need to clean request */ - ocf_req_put(req); - - return 0; - } - if (ocf_engine_needs_repart(req)) { OCF_DEBUG_RQ(req, "Re-Part"); @@ -84,6 +70,26 @@ int ocf_read_pt_do(struct ocf_request *req) ocf_core_stats_request_pt_update(req->core, req->part_id, req->rw, req->info.hit_no, req->core_line_count); +} + +int ocf_read_pt_do(struct ocf_request *req) +{ + /* Get OCF request - increase reference counter */ + ocf_req_get(req); + + if (req->info.dirty_any) { + ocf_hb_req_prot_lock_rd(req); + /* Need to clean, start it */ + ocf_engine_clean(req); + ocf_hb_req_prot_unlock_rd(req); + + /* Do not processing, because first we need to clean request */ + ocf_req_put(req); + + return 0; + } + + _ocf_read_pt_process(req); /* Put OCF request - decrease reference counter */ ocf_req_put(req); @@ -160,6 +166,37 @@ int ocf_read_pt(struct ocf_request *req) return 0; } +bool ocf_read_pt_fast(struct ocf_request *req) +{ + ocf_req_hash(req); + ocf_hb_req_prot_lock_rd(req); + + ocf_engine_lookup(req); + + if (req->info.dirty_any) + goto fallback; + + if (ocf_engine_mapped_count(req) && !ocf_req_trylock_rd( + ocf_cache_line_concurrency(req->cache), req)) { + goto fallback; + } + + ocf_engine_set_hot(req); + + ocf_hb_req_prot_unlock_rd(req); + + /* Released on completion */ + ocf_req_get(req); + + _ocf_read_pt_process(req); + + return true; + +fallback: + ocf_hb_req_prot_unlock_rd(req); + return false; +} + void ocf_queue_push_req_pt(struct ocf_request *req) { ocf_queue_push_req_cb(req, ocf_read_pt_do, diff --git a/src/engine/engine_pt.h b/src/engine/engine_pt.h index 4742d6c4..14a3bf29 100644 --- a/src/engine/engine_pt.h +++ b/src/engine/engine_pt.h @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2021 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -11,6 +12,8 @@ int ocf_read_pt(struct ocf_request *req); int ocf_read_pt_do(struct ocf_request *req); +bool ocf_read_pt_fast(struct ocf_request *req); + void ocf_queue_push_req_pt(struct ocf_request *req); #endif /* __ENGINE_PT_H__ */ diff --git a/src/engine/engine_rd.c b/src/engine/engine_rd.c index 95729f48..371a4b42 100644 --- a/src/engine/engine_rd.c +++ b/src/engine/engine_rd.c @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2024-2025 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -220,3 +221,46 @@ int ocf_read_generic(struct ocf_request *req) return 0; } + +bool ocf_read_generic_fast(struct ocf_request *req) +{ + /* Requests to disabled partitions go in pass-through */ + if (!ocf_user_part_is_enabled(&req->cache->user_parts[req->part_id])) + return ocf_read_pt_fast(req); + + ocf_req_hash(req); + ocf_hb_req_prot_lock_rd(req); + + ocf_engine_lookup(req); + + /* Miss requires mapping, which can't be done synchronously */ + if (!ocf_engine_is_hit(req)) + goto fallback; + + if (!ocf_req_trylock_rd(ocf_cache_line_concurrency(req->cache), req)) + goto fallback; + + ocf_engine_set_hot(req); + + ocf_hb_req_prot_unlock_rd(req); + + /* Released on completion */ + ocf_req_get(req); + + if (ocf_engine_needs_repart(req)) { + ocf_hb_req_prot_lock_wr(req); + ocf_user_part_move(req); + ocf_hb_req_prot_unlock_wr(req); + } + + ocf_read_generic_submit_hit(req); + + ocf_engine_update_request_stats(req); + ocf_engine_update_block_stats(req); + + return true; + +fallback: + ocf_hb_req_prot_unlock_rd(req); + return false; +} diff --git a/src/engine/engine_rd.h b/src/engine/engine_rd.h index ae5c025d..7fc24955 100644 --- a/src/engine/engine_rd.h +++ b/src/engine/engine_rd.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -8,6 +9,8 @@ int ocf_read_generic(struct ocf_request *req); +bool ocf_read_generic_fast(struct ocf_request *req); + void ocf_read_generic_submit_hit(struct ocf_request *req); #endif /* __ENGINE_RD_H__ */ diff --git a/src/engine/engine_wa.c b/src/engine/engine_wa.c index ee1ea533..db7fafc4 100644 --- a/src/engine/engine_wa.c +++ b/src/engine/engine_wa.c @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ #include "ocf/ocf.h" @@ -52,4 +53,18 @@ int ocf_write_wa(struct ocf_request *req) return 0; } +bool ocf_write_wa_fast(struct ocf_request *req) +{ + bool hit; + + ocf_req_hash(req); + ocf_hb_req_prot_lock_rd(req); + + ocf_engine_lookup(req); + hit = ocf_engine_is_hit(req); + + ocf_hb_req_prot_unlock_rd(req); + + return hit ? ocf_write_wt_fast(req) : ocf_write_wi_fast(req); +} diff --git a/src/engine/engine_wa.h b/src/engine/engine_wa.h index 902d816e..6d7c03e1 100644 --- a/src/engine/engine_wa.h +++ b/src/engine/engine_wa.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -8,4 +9,6 @@ int ocf_write_wa(struct ocf_request *req); +bool ocf_write_wa_fast(struct ocf_request *req); + #endif /* __ENGINE_WA_H__ */ diff --git a/src/engine/engine_wb.c b/src/engine/engine_wb.c index 025e8cfe..cb226aca 100644 --- a/src/engine/engine_wb.c +++ b/src/engine/engine_wb.c @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -185,3 +186,39 @@ int ocf_write_wb(struct ocf_request *req) return 0; } + +bool ocf_write_wb_fast(struct ocf_request *req) +{ + /* Requests to disabled partitions go in pass-through */ + if (!ocf_user_part_is_enabled(&req->cache->user_parts[req->part_id])) + return ocf_write_wi_fast(req); + + ocf_req_hash(req); + ocf_hb_req_prot_lock_rd(req); + + ocf_engine_lookup(req); + + if (!ocf_engine_is_mapped(req)) + goto fallback; + + if (!ocf_req_trylock_wr(ocf_cache_line_concurrency(req->cache), req)) + goto fallback; + + ocf_engine_set_hot(req); + + ocf_hb_req_prot_unlock_rd(req); + + /* Released on completion */ + ocf_req_get(req); + + _ocf_write_wb_submit(req); + + ocf_engine_update_request_stats(req); + ocf_engine_update_block_stats(req); + + return true; + +fallback: + ocf_hb_req_prot_unlock_rd(req); + return false; +} diff --git a/src/engine/engine_wb.h b/src/engine/engine_wb.h index b7ffa610..98bab7d8 100644 --- a/src/engine/engine_wb.h +++ b/src/engine/engine_wb.h @@ -10,4 +10,6 @@ int ocf_write_wb(struct ocf_request *req); int ocf_write_wb_do(struct ocf_request *req); +bool ocf_write_wb_fast(struct ocf_request *req); + #endif /* __ENGINE_WB_H__ */ diff --git a/src/engine/engine_wi.c b/src/engine/engine_wi.c index 6ff298f5..a63f4972 100644 --- a/src/engine/engine_wi.c +++ b/src/engine/engine_wi.c @@ -118,6 +118,23 @@ static void _ocf_write_wi_core_complete(struct ocf_request *req, int error) } } +static inline void _ocf_write_wi_submit(struct ocf_request *req) +{ + OCF_DEBUG_RQ(req, "Submit"); + + /* Submit write IO to the core */ + ocf_engine_forward_core_io_req(req, _ocf_write_wi_core_complete); + + /* Update statistics */ + ocf_engine_update_block_stats(req); + + ocf_core_stats_pt_block_update(req->core, req->part_id, req->rw, + req->bytes); + + ocf_core_stats_request_pt_update(req->core, req->part_id, req->rw, + req->info.hit_no, req->core_line_count); +} + static int _ocf_write_wi_core_write(struct ocf_request *req) { /* Get OCF request - increase reference counter */ @@ -136,19 +153,7 @@ static int _ocf_write_wi_core_write(struct ocf_request *req) return 0; } - OCF_DEBUG_RQ(req, "Submit"); - - /* Submit write IO to the core */ - ocf_engine_forward_core_io_req(req, _ocf_write_wi_core_complete); - - /* Update statistics */ - ocf_engine_update_block_stats(req); - - ocf_core_stats_pt_block_update(req->core, req->part_id, req->rw, - req->bytes); - - ocf_core_stats_request_pt_update(req->core, req->part_id, req->rw, - req->info.hit_no, req->core_line_count); + _ocf_write_wi_submit(req); /* Put OCF request - decrease reference counter */ ocf_req_put(req); @@ -212,3 +217,34 @@ int ocf_write_wi(struct ocf_request *req) return 0; } + +bool ocf_write_wi_fast(struct ocf_request *req) +{ + ocf_req_hash(req); + ocf_hb_req_prot_lock_rd(req); + + ocf_engine_lookup(req); + + if (req->info.dirty_any) + goto fallback; + + if (ocf_engine_mapped_count(req) && !ocf_req_trylock_wr( + ocf_cache_line_concurrency(req->cache), req)) { + goto fallback; + } + + ocf_engine_set_hot(req); + + ocf_hb_req_prot_unlock_rd(req); + + /* Released on completion */ + ocf_req_get(req); + + _ocf_write_wi_submit(req); + + return true; + +fallback: + ocf_hb_req_prot_unlock_rd(req); + return false; +} diff --git a/src/engine/engine_wi.h b/src/engine/engine_wi.h index 5bdbe7f1..1c011070 100644 --- a/src/engine/engine_wi.h +++ b/src/engine/engine_wi.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -8,4 +9,6 @@ int ocf_write_wi(struct ocf_request *req); +bool ocf_write_wi_fast(struct ocf_request *req); + #endif /* __ENGINE_WI_H__ */ diff --git a/src/engine/engine_wo.c b/src/engine/engine_wo.c index a73f9d9c..c961286a 100644 --- a/src/engine/engine_wo.c +++ b/src/engine/engine_wo.c @@ -171,10 +171,8 @@ static void _ocf_read_wo_core_complete(struct ocf_request *req, int error) ocf_queue_push_req(req, OCF_QUEUE_ALLOW_SYNC | OCF_QUEUE_PRIO_HIGH); } -static int ocf_read_wo_do(struct ocf_request *req) +static inline void _ocf_read_wo_process(struct ocf_request *req) { - ocf_req_get(req); - /* Lack of cacheline repartitioning here is deliberate. WO cache mode * reads should not affect cacheline status as reading data from the * cache is just an internal optimization. Also WO cache mode is @@ -194,6 +192,13 @@ static int ocf_read_wo_do(struct ocf_request *req) ocf_engine_update_request_stats(req); ocf_engine_update_block_stats(req); +} + +static int ocf_read_wo_do(struct ocf_request *req) +{ + ocf_req_get(req); + + _ocf_read_wo_process(req); ocf_req_put(req); return 0; @@ -248,3 +253,31 @@ int ocf_read_wo(struct ocf_request *req) return 0; } + +bool ocf_read_wo_fast(struct ocf_request *req) +{ + ocf_req_hash(req); + ocf_hb_req_prot_lock_rd(req); + + ocf_engine_lookup(req); + + if (ocf_engine_mapped_count(req) && !ocf_req_trylock_rd( + ocf_cache_line_concurrency(req->cache), req)) { + goto fallback; + } + + ocf_engine_set_hot(req); + + ocf_hb_req_prot_unlock_rd(req); + + /* Released on completion */ + ocf_req_get(req); + + _ocf_read_wo_process(req); + + return true; + +fallback: + ocf_hb_req_prot_unlock_rd(req); + return false; +} diff --git a/src/engine/engine_wo.h b/src/engine/engine_wo.h index 9f3c365f..e2e0d033 100644 --- a/src/engine/engine_wo.h +++ b/src/engine/engine_wo.h @@ -1,6 +1,7 @@ /* * Copyright(c) 2019-2021 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -11,4 +12,6 @@ int ocf_read_wo(struct ocf_request *req); +bool ocf_read_wo_fast(struct ocf_request *req); + #endif /* __ENGINE_WO_H__ */ diff --git a/src/engine/engine_wt.c b/src/engine/engine_wt.c index 8bd51f3c..1b78916b 100644 --- a/src/engine/engine_wt.c +++ b/src/engine/engine_wt.c @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -152,11 +153,8 @@ static inline void _ocf_write_wt_submit(struct ocf_request *req) ocf_engine_forward_core_io_req(req, _ocf_write_wt_core_complete); } -static int _ocf_write_wt_do(struct ocf_request *req) +static inline void _ocf_write_wt_process(struct ocf_request *req) { - /* Get OCF request - increase reference counter */ - ocf_req_get(req); - if (!req->info.dirty_any) { /* Set metadata bits before the request submission only if the dirty status for any of the request's cachelines won't change */ @@ -170,6 +168,14 @@ static int _ocf_write_wt_do(struct ocf_request *req) /* Update statistics */ ocf_engine_update_request_stats(req); ocf_engine_update_block_stats(req); +} + +static int _ocf_write_wt_do(struct ocf_request *req) +{ + /* Get OCF request - increase reference counter */ + ocf_req_get(req); + + _ocf_write_wt_process(req); /* Put OCF request - decrease reference counter */ ocf_req_put(req); @@ -219,3 +225,36 @@ int ocf_write_wt(struct ocf_request *req) return 0; } + +bool ocf_write_wt_fast(struct ocf_request *req) +{ + /* Requests to disabled partitions go in pass-through */ + if (!ocf_user_part_is_enabled(&req->cache->user_parts[req->part_id])) + return ocf_write_wi_fast(req); + + ocf_req_hash(req); + ocf_hb_req_prot_lock_rd(req); + + ocf_engine_lookup(req); + + if (!ocf_engine_is_mapped(req)) + goto fallback; + + if (!ocf_req_trylock_wr(ocf_cache_line_concurrency(req->cache), req)) + goto fallback; + + ocf_engine_set_hot(req); + + ocf_hb_req_prot_unlock_rd(req); + + /* Released on completion */ + ocf_req_get(req); + + _ocf_write_wt_process(req); + + return true; + +fallback: + ocf_hb_req_prot_unlock_rd(req); + return false; +} diff --git a/src/engine/engine_wt.h b/src/engine/engine_wt.h index 2135f993..ad4141bc 100644 --- a/src/engine/engine_wt.h +++ b/src/engine/engine_wt.h @@ -1,5 +1,6 @@ /* * Copyright(c) 2012-2021 Intel Corporation + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ @@ -8,4 +9,6 @@ int ocf_write_wt(struct ocf_request *req); +bool ocf_write_wt_fast(struct ocf_request *req); + #endif /* __ENGINE_WT_H__ */ diff --git a/src/mngt/ocf_mngt_cache.c b/src/mngt/ocf_mngt_cache.c index 7505e680..7e60e5b4 100644 --- a/src/mngt/ocf_mngt_cache.c +++ b/src/mngt/ocf_mngt_cache.c @@ -1115,7 +1115,7 @@ static int _ocf_mngt_init_prepare_cache(struct ocf_cache_mngt_init_params *param cache->pt_unaligned_io = cfg->pt_unaligned_io; #endif - cache->use_submit_io_fast = cfg->use_submit_io_fast; + cache->use_sync_fastpath = cfg->use_sync_fastpath; cache->metadata.is_volatile = cfg->metadata_volatile; diff --git a/src/ocf_cache_priv.h b/src/ocf_cache_priv.h index 570da8c6..70fa40a8 100644 --- a/src/ocf_cache_priv.h +++ b/src/ocf_cache_priv.h @@ -128,7 +128,7 @@ struct ocf_cache { bool pt_unaligned_io; - bool use_submit_io_fast; + bool use_sync_fastpath; struct { struct ocf_async_lock lock; diff --git a/src/ocf_core.c b/src/ocf_core.c index 86d307ee..52c21165 100644 --- a/src/ocf_core.c +++ b/src/ocf_core.c @@ -221,51 +221,12 @@ static void ocf_req_complete(struct ocf_request *req, int error) ocf_req_put(req); } -static inline ocf_req_cache_mode_t _ocf_core_req_resolve_fast_mode( - ocf_cache_t cache, struct ocf_request *req) -{ - switch (req->cache_mode) { - case ocf_req_cache_mode_wb: - case ocf_req_cache_mode_wo: - return ocf_req_cache_mode_fast; - default: - break; - } - - if (!cache->use_submit_io_fast) - return ocf_req_cache_mode_max; - - return ocf_req_cache_mode_fast; -} - -static int ocf_core_submit_io_fast(struct ocf_request *req, ocf_cache_t cache) -{ - ocf_req_cache_mode_t original_mode, resolved_mode; - int ret; - - if (req->cache_mode == ocf_req_cache_mode_pt) - return OCF_FAST_PATH_NO; - - resolved_mode = _ocf_core_req_resolve_fast_mode(cache, req); - if (resolved_mode == ocf_req_cache_mode_max) - return OCF_FAST_PATH_NO; - - original_mode = req->cache_mode; - req->cache_mode = resolved_mode; - - ret = ocf_engine_hndl_fast_req(req); - if (ret == OCF_FAST_PATH_NO) - req->cache_mode = original_mode; - - return ret; -} - static void ocf_core_volume_submit_io(ocf_io_t io) { struct ocf_request *req = ocf_io_to_req(io); ocf_core_t core; ocf_cache_t cache; - int ret, fastpath; + int ret; OCF_CHECK_NULL(io); @@ -306,9 +267,7 @@ static void ocf_core_volume_submit_io(ocf_io_t io) /* Prevent race condition with prefetch */ ocf_req_get(req); - fastpath = ocf_core_submit_io_fast(req, cache); - - if (fastpath == OCF_FAST_PATH_YES) + if (cache->use_sync_fastpath && ocf_engine_hndl_fast_req(req)) goto prefetch; ocf_req_clear_map(req); diff --git a/src/ocf_request.h b/src/ocf_request.h index 37807676..f9da4e50 100644 --- a/src/ocf_request.h +++ b/src/ocf_request.h @@ -24,8 +24,6 @@ typedef enum { ocf_req_cache_mode_wo = ocf_cache_mode_wo, /* internal modes */ - ocf_req_cache_mode_fast, - /*!< Fast path */ ocf_req_cache_mode_d2c, /*!< Direct to Core - pass through to core without touching cacheline metadata */ diff --git a/src/utils/utils_alock.c b/src/utils/utils_alock.c index 97075e1a..00df47e8 100644 --- a/src/utils/utils_alock.c +++ b/src/utils/utils_alock.c @@ -804,6 +804,28 @@ int ocf_alock_lock_wr(struct ocf_alock *alock, return lock; } +static inline bool ocf_alock_trylock(struct ocf_alock *alock, + struct ocf_request *req, int rw) +{ + ENV_BUG_ON(env_atomic_read(&req->lock_remaining)); + req->alock_rw = rw; + /* trylock never puts the request on a waitlist */ + req->is_deferred = false; + + return alock->cbs->lock_entries_fast(alock, req, rw) == + OCF_LOCK_ACQUIRED; +} + +bool ocf_alock_trylock_rd(struct ocf_alock *alock, struct ocf_request *req) +{ + return ocf_alock_trylock(alock, req, OCF_READ); +} + +bool ocf_alock_trylock_wr(struct ocf_alock *alock, struct ocf_request *req) +{ + return ocf_alock_trylock(alock, req, OCF_WRITE); +} + bool ocf_cache_line_is_used(struct ocf_alock *alock, ocf_cache_line_t entry) { diff --git a/src/utils/utils_alock.h b/src/utils/utils_alock.h index 44fe8ffa..7434ef83 100644 --- a/src/utils/utils_alock.h +++ b/src/utils/utils_alock.h @@ -1,6 +1,7 @@ /* * Copyright(c) 2012-2022 Intel Corporation * Copyright(c) 2024 Huawei Technologies + * Copyright(c) 2026 Unvertical * SPDX-License-Identifier: BSD-3-Clause */ #ifndef __UTILS_ALOCK_H__ @@ -55,6 +56,10 @@ int ocf_alock_lock_rd(struct ocf_alock *alock, int ocf_alock_lock_wr(struct ocf_alock *alock, struct ocf_request *req, ocf_req_async_lock_cb cmpl); +bool ocf_alock_trylock_rd(struct ocf_alock *alock, struct ocf_request *req); + +bool ocf_alock_trylock_wr(struct ocf_alock *alock, struct ocf_request *req); + /** * @Check if cache line is used. * diff --git a/tests/functional/pyocf/types/cache.py b/tests/functional/pyocf/types/cache.py index 56eef742..6c81e513 100644 --- a/tests/functional/pyocf/types/cache.py +++ b/tests/functional/pyocf/types/cache.py @@ -102,7 +102,7 @@ class CacheConfig(Structure): ("_metadata_volatile", c_bool), ("_locked", c_bool), ("_pt_unaligned_io", c_bool), - ("_use_submit_io_fast", c_bool), + ("_use_sync_fastpath", c_bool), ("_backfill", Backfill), ] @@ -262,7 +262,7 @@ class Cache: DEFAULT_BACKFILL_QUEUE_SIZE = 65536 DEFAULT_BACKFILL_UNBLOCK = 60000 DEFAULT_PT_UNALIGNED_IO = False - DEFAULT_USE_SUBMIT_FAST = False + DEFAULT_USE_SYNC_FASTPATH = True def __init__( self, @@ -275,7 +275,7 @@ def __init__( max_queue_size: int = DEFAULT_BACKFILL_QUEUE_SIZE, queue_unblock_size: int = DEFAULT_BACKFILL_UNBLOCK, pt_unaligned_io: bool = DEFAULT_PT_UNALIGNED_IO, - use_submit_fast: bool = DEFAULT_USE_SUBMIT_FAST, + use_sync_fastpath: bool = DEFAULT_USE_SYNC_FASTPATH, ): self.device = None self.started = False @@ -289,7 +289,7 @@ def __init__( self.max_queue_size = max_queue_size self.queue_unblock_size = queue_unblock_size self.pt_unaligned_io = pt_unaligned_io - self.use_submit_fast = use_submit_fast + self.use_sync_fastpath = use_sync_fastpath self.cache_handle = c_void_p() self._as_parameter_ = self.cache_handle @@ -314,7 +314,7 @@ def start_cache( ), _locked=locked, _pt_unaligned_io=self.pt_unaligned_io, - _use_submit_fast=self.use_submit_fast, + _use_sync_fastpath=self.use_sync_fastpath, ) status = self.owner.lib.ocf_mngt_cache_start( diff --git a/tests/functional/tests/management/test_start_stop.py b/tests/functional/tests/management/test_start_stop.py index baee1baf..6d784623 100644 --- a/tests/functional/tests/management/test_start_stop.py +++ b/tests/functional/tests/management/test_start_stop.py @@ -180,7 +180,7 @@ def test_start_params(pyocf_ctx, mode: CacheMode, cls: CacheLineSize, layout: Me unblock_size = randrange(1, queue_size) volatile_metadata = randrange(2) == 1 unaligned_io = randrange(2) == 1 - submit_fast = randrange(2) == 1 + sync_fastpath = randrange(2) == 1 name = "test" logger.info("[STAGE] Start cache") @@ -193,7 +193,7 @@ def test_start_params(pyocf_ctx, mode: CacheMode, cls: CacheLineSize, layout: Me max_queue_size=queue_size, queue_unblock_size=unblock_size, pt_unaligned_io=unaligned_io, - use_submit_fast=submit_fast, + use_sync_fastpath=sync_fastpath, ) stats = cache.get_stats() @@ -201,7 +201,7 @@ def test_start_params(pyocf_ctx, mode: CacheMode, cls: CacheLineSize, layout: Me assert stats["conf"]["cache_line_size"] == cls, "Cache line size" assert cache.get_name() == name, "Cache name" # TODO: metadata_volatile, max_queue_size, - # queue_unblock_size, pt_unaligned_io, use_submit_fast + # queue_unblock_size, pt_unaligned_io, use_sync_fastpath # TODO: test in functional tests