From 7b59eda9332b6a4665694d2d8b93eec10400e4f0 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 29 Jul 2026 10:46:04 -0500 Subject: [PATCH 01/49] Add binary cache --- docs/dev/triage-migraphx.rst | 35 +++ src/include/migraphx/program.hpp | 5 + src/targets/gpu/CMakeLists.txt | 36 +++ src/targets/gpu/binary_cache.cpp | 263 ++++++++++++++++ src/targets/gpu/compile_hip.cpp | 41 +++ src/targets/gpu/compile_hip_code_object.cpp | 60 +++- src/targets/gpu/compile_ops.cpp | 259 ++++++++++++++-- src/targets/gpu/compile_pointwise.cpp | 14 +- src/targets/gpu/compiled_code.cpp | 56 ++++ src/targets/gpu/compiler.cpp | 13 +- .../gpu/include/migraphx/gpu/binary_cache.hpp | 133 ++++++++ .../gpu/include/migraphx/gpu/compile_hip.hpp | 9 + .../migraphx/gpu/compile_hip_code_object.hpp | 22 +- .../gpu/include/migraphx/gpu/compile_ops.hpp | 2 + .../migraphx/gpu/compile_pointwise.hpp | 4 + .../gpu/include/migraphx/gpu/compiler.hpp | 47 ++- .../gpu/include/migraphx/gpu/context.hpp | 5 + src/targets/gpu/include/migraphx/gpu/mlir.hpp | 12 + src/targets/gpu/jit/gather.cpp | 16 +- src/targets/gpu/jit/mlir.cpp | 15 +- src/targets/gpu/jit/pad.cpp | 16 +- src/targets/gpu/jit/pointwise.cpp | 47 +-- src/targets/gpu/jit/reduce.cpp | 76 +++-- src/targets/gpu/mlir.cpp | 28 ++ src/targets/gpu/target.cpp | 12 +- test/gpu/binary_cache.cpp | 289 ++++++++++++++++++ 26 files changed, 1404 insertions(+), 111 deletions(-) create mode 100644 src/targets/gpu/binary_cache.cpp create mode 100644 src/targets/gpu/compiled_code.cpp create mode 100644 src/targets/gpu/include/migraphx/gpu/binary_cache.hpp create mode 100644 test/gpu/binary_cache.cpp diff --git a/docs/dev/triage-migraphx.rst b/docs/dev/triage-migraphx.rst index c17849ca4c7..78be28bdb67 100644 --- a/docs/dev/triage-migraphx.rst +++ b/docs/dev/triage-migraphx.rst @@ -151,4 +151,39 @@ Debugging and Tracing - ``MIGRAPHX_TRACE_BENCHMARKING=3`` # Kernel benchmarking process +Binary Cache +============ + +Compiled kernels are always shared within a process, so a kernel that appears many times in a +model is only compiled once. Setting a directory makes that reuse survive across runs: + +.. code-block:: bash + + export MIGRAPHX_BINARY_CACHE=$HOME/.cache/migraphx + +Entries are grouped by a directory naming the entry format, the HIP toolchain, a digest of the +embedded kernel headers, and the rocMLIR build, so entries a build cannot use are never +consulted. The toolchain is identified by compiling a fixed kernel and hashing the result, +because it is loaded at runtime and what it reports about itself is coarser than what it +generates. Reclaim space by deleting the directories for builds you no longer use: + +.. code-block:: bash + + ls $HOME/.cache/migraphx # each directory has a cache.info describing the build + rm -r $HOME/.cache/migraphx/v1-hipe0f11f454e40-* + +The same settings are available as backend options, which take precedence over the environment +and are how tests configure the cache: + +.. code-block:: python + + model.compile(migraphx.get_target("gpu"), + binary_cache="/tmp/cache", binary_cache_verify=True) + +Entries are keyed on the exact source handed to the backend compiler. If a kernel is suspected +of being reused when it should not be, set ``binary_cache_verify``. That compiles even when a +result could be reused and fails loudly if the two disagree, which is the only way an +incomplete key shows up other than as wrong results. It is slower than compiling without a +cache at all, so use it to diagnose rather than routinely. + This systematic approach helps maintainers quickly understand and fix root causes. \ No newline at end of file diff --git a/src/include/migraphx/program.hpp b/src/include/migraphx/program.hpp index cc274b7a9f1..a3e522b8570 100644 --- a/src/include/migraphx/program.hpp +++ b/src/include/migraphx/program.hpp @@ -179,6 +179,11 @@ struct MIGRAPHX_EXPORT program // bumped if any changes occur to the format of the MXR file. static constexpr int program_file_version = 8; }; + +/// Lets a program be a member of a reflected type, using the same conversion as saving one. +inline void migraphx_to_value(value& v, const program& p) { v = p.to_value(); } +inline void migraphx_from_value(const value& v, program& p) { p.from_value(v); } + } // namespace MIGRAPHX_INLINE_NS } // namespace migraphx diff --git a/src/targets/gpu/CMakeLists.txt b/src/targets/gpu/CMakeLists.txt index ee20237831e..4965ca86127 100644 --- a/src/targets/gpu/CMakeLists.txt +++ b/src/targets/gpu/CMakeLists.txt @@ -24,6 +24,25 @@ find_package(hip REQUIRED) +# Produce a short identity for a toolchain file, used to separate binary cache entries that +# different toolchains cannot share. The content hash is exact but slow on the larger libraries, +# so fall back to size and timestamp, which still changes whenever the file is replaced. +function(migraphx_hash_file FILE_PATH OUT_VAR) + if(NOT FILE_PATH OR NOT EXISTS "${FILE_PATH}") + set(${OUT_VAR} "unknown" PARENT_SCOPE) + return() + endif() + file(SIZE "${FILE_PATH}" HASH_FILE_SIZE) + if(HASH_FILE_SIZE LESS 268435456) + file(MD5 "${FILE_PATH}" HASH_RESULT) + else() + file(TIMESTAMP "${FILE_PATH}" HASH_FILE_TIME "%s" UTC) + string(MD5 HASH_RESULT "${FILE_PATH}-${HASH_FILE_SIZE}-${HASH_FILE_TIME}") + endif() + string(SUBSTRING "${HASH_RESULT}" 0 12 HASH_RESULT) + set(${OUT_VAR} "${HASH_RESULT}" PARENT_SCOPE) +endfunction() + # MIGRAPHX_USE_AMDMLSS is resolved to a plain boolean at the top-level CMakeLists. if(MIGRAPHX_USE_AMDMLSS) message(STATUS "MIGraphX is using AMDMLSS") @@ -202,7 +221,9 @@ endif() add_library(migraphx_gpu analyze_streams.cpp allocation_model.cpp + binary_cache.cpp code_object_op.cpp + compiled_code.cpp compile_ops.cpp compile_gen.cpp compile_hip.cpp @@ -313,6 +334,21 @@ if(MIGRAPHX_ENABLE_MLIR) find_package(rocMLIR 1.0.0 CONFIG REQUIRED) message(STATUS "Build with rocMLIR::rockCompiler ${rocMLIR_VERSION}") target_compile_definitions(migraphx_gpu PRIVATE "-DMIGRAPHX_MLIR") + + # Identify the rocMLIR build for the binary cache by hashing the library that gets linked. + # That covers a locally built or substituted rocMLIR, which a recorded version or commit + # would not. + get_target_property(MIGRAPHX_ROCKCOMPILER_LIB rocMLIR::rockCompiler IMPORTED_LOCATION) + if(NOT MIGRAPHX_ROCKCOMPILER_LIB) + # rockCompiler is an interface target, so the archive is named as a link dependency. + get_target_property(MIGRAPHX_ROCKCOMPILER_LIB rocMLIR::rockCompiler INTERFACE_LINK_LIBRARIES) + list(FILTER MIGRAPHX_ROCKCOMPILER_LIB INCLUDE REGEX "rockCompiler") + list(GET MIGRAPHX_ROCKCOMPILER_LIB 0 MIGRAPHX_ROCKCOMPILER_LIB) + endif() + migraphx_hash_file("${MIGRAPHX_ROCKCOMPILER_LIB}" MIGRAPHX_ROCMLIR_ID) + message(STATUS "rocMLIR binary cache id: ${MIGRAPHX_ROCMLIR_ID}") + target_compile_definitions(migraphx_gpu + PRIVATE "-DMIGRAPHX_ROCMLIR_ID=\"${MIGRAPHX_ROCMLIR_ID}\"") # Make this private to avoid multiple inclusions of LLVM symbols. target_link_libraries(migraphx_gpu PRIVATE rocMLIR::rockCompiler) # Hide LLVM internals that come from rocMLIR. diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp new file mode 100644 index 00000000000..7c0bf4b21ee --- /dev/null +++ b/src/targets/gpu/binary_cache.cpp @@ -0,0 +1,263 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +// cppcheck-suppress definePrefix +#define WIN32_LEAN_AND_MEAN +#include +#undef getpid +// cppcheck-suppress [definePrefix, defineUpperCase] +#define getpid _getpid +#else +#include +#include +#endif + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_BINARY_CACHE) + +// Bump when the shape of a stored fragment changes, which happens when a compiler changes the +// instructions it replaces with or when a serialized operator gains or loses a field. Such a +// change is invisible to the key, since the source handed to the backend is unaffected. +static constexpr const char* binary_cache_format = "v1"; + +#ifdef MIGRAPHX_ROCMLIR_ID +static constexpr const char* rocmlir_id = MIGRAPHX_ROCMLIR_ID; +#else +static constexpr const char* rocmlir_id = "nomlir"; +#endif + +binary_cache_settings binary_cache_settings::defaults() +{ + static const std::string path = string_value_of(MIGRAPHX_BINARY_CACHE{}); + return {path, false}; +} + +void binary_cache::configure(const binary_cache_settings& s) +{ + const std::lock_guard lock(mutex); + settings = s; +} + +bool binary_cache::enabled() +{ + const std::lock_guard lock(mutex); + return not settings.path.empty(); +} + +bool binary_cache::verify() +{ + const std::lock_guard lock(mutex); + return settings.verify; +} + +/// A digest of the kernel headers compiled into this build. Taken from the embedded sources +/// rather than the files on disk, so it tracks what is actually compiled even when the build +/// system has not reconfigured. +static const std::string& kernels_digest() +{ + static const std::string digest = [] { + std::stringstream ss; + for(const auto& [path, content] : ::migraphx_kernels()) + { + ss << path << "\n" << content << "\n"; + } + return md5(ss.str()).substr(0, 12); + }(); + return digest; +} + +std::string binary_cache::version_dir(const context& ctx) +{ + auto toolchain = hip_compiler_version(ctx.get_current_device().get_device_name()); + if(toolchain.empty()) + return {}; + return std::string{binary_cache_format} + "-hip" + toolchain + "-kernels" + kernels_digest() + + "-rocmlir" + rocmlir_id; +} + +/// Entries are grouped by the device they were compiled for. The architecture is part of this +/// because it is passed to the compiler rather than appearing in the source; the core count and +/// wavefront size are here to keep the directory self-describing, since they already reach the +/// key through the launch bounds baked into the source. +static std::string device_dir(const context& ctx) +{ + const auto& device = ctx.get_current_device(); + return to_c_id(device.get_device_name()) + "_cu" + std::to_string(device.get_cu_count()) + + "_wf" + std::to_string(device.get_wavefront_size()); +} + +/// Where an entry lives, or an empty path when the toolchain cannot be identified and entries +/// from different toolchains would be indistinguishable. +static fs::path entry_path(const fs::path& root, const context& ctx, const std::string& key) +{ + auto version = binary_cache::version_dir(ctx); + if(version.empty()) + return {}; + return root / version / device_dir(ctx) / (md5(key) + ".mxr"); +} + +/// Record what this build is, so a directory full of hashes can be identified later. +static void write_stamp(const context& ctx, const fs::path& dir) +{ + auto stamp = dir / "cache.info"; + if(fs::exists(stamp)) + return; + std::stringstream ss; + ss << "format: " << binary_cache_format << "\n"; + ss << "hip: " << hip_compiler_version(ctx.get_current_device().get_device_name()) << "\n"; + ss << "kernels: " << kernels_digest() << "\n"; + ss << "rocmlir: " << rocmlir_id << "\n"; + write_string(stamp, ss.str()); +} + +optional binary_cache::get(const context& ctx, const std::string& key) +{ + if(key.empty()) + return nullopt; + fs::path root; + { + const std::lock_guard lock(mutex); + auto it = memo.find(key); + if(it != memo.end()) + { + stats.hits++; + return it->second; + } + root = settings.path; + } + if(root.empty()) + return nullopt; + + auto path = entry_path(root, ctx, key); + if(path.empty() or not fs::exists(path)) + { + const std::lock_guard lock(mutex); + stats.misses++; + return nullopt; + } + + binary_cache_entry entry; + try + { + migraphx::from_value(from_msgpack(read_buffer(path)), entry); + } + catch(const std::exception& e) + { + // A damaged or stale entry is only worth a recompile, so treat every failure as a miss + // and let the result be written over the top. + log::warn() << "Ignoring unreadable binary cache entry " << path << ": " << e.what(); + const std::lock_guard lock(mutex); + stats.misses++; + return nullopt; + } + if(entry.key != key) + { + log::warn() << "Ignoring binary cache entry with mismatched key: " << path; + const std::lock_guard lock(mutex); + stats.misses++; + return nullopt; + } + + const std::lock_guard lock(mutex); + memo[key] = entry.code; + stats.hits++; + return entry.code; +} + +void binary_cache::insert(const context& ctx, const binary_cache_entry& entry) +{ + if(entry.key.empty()) + return; + fs::path root; + { + const std::lock_guard lock(mutex); + memo[entry.key] = entry.code; + root = settings.path; + } + if(root.empty()) + return; + + auto path = entry_path(root, ctx, entry.key); + if(path.empty()) + return; + try + { + fs::create_directories(path.parent_path()); + write_stamp(ctx, path.parent_path().parent_path()); + // Publish by rename so a reader never sees a half-written entry. The content is decided + // entirely by the key, so a writer that loses the race replaces the file with the same + // bytes and no locking is needed. + std::stringstream suffix; + suffix << "." << getpid() << "." << std::this_thread::get_id() << ".tmp"; + auto tmp = path; + tmp += suffix.str(); + write_buffer(tmp, to_msgpack(migraphx::to_value(entry))); + fs::rename(tmp, path); + } + catch(const std::exception& e) + { + log::warn() << "Failed to write binary cache entry " << path << ": " << e.what(); + } +} + +void binary_cache::record_compiled(std::size_t n) +{ + const std::lock_guard lock(mutex); + stats.compiled += n; +} + +void binary_cache::record_reused(std::size_t n) +{ + const std::lock_guard lock(mutex); + stats.reused += n; +} + +binary_cache_stats binary_cache::get_stats() +{ + const std::lock_guard lock(mutex); + return stats; +} + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx diff --git a/src/targets/gpu/compile_hip.cpp b/src/targets/gpu/compile_hip.cpp index 1c39115fedc..7ceaba9597f 100644 --- a/src/targets/gpu/compile_hip.cpp +++ b/src/targets/gpu/compile_hip.cpp @@ -26,11 +26,14 @@ #include #include #include +#include #include #include #include #include #include +#include +#include #ifdef MIGRAPHX_USE_HIPRTC #include @@ -370,6 +373,44 @@ std::vector> compile_hip_src(const std::vector& srcs #endif // MIGRAPHX_USE_HIPRTC +// A kernel with no dependencies, compiled only so the result can be identified. It has to stay +// fixed, since changing it changes the identity of every toolchain. +static const char* const toolchain_probe = R"__migraphx__( +extern "C" __global__ void migraphx_toolchain_probe(float* p) +{ + p[0] = p[0] * 2.0f + 1.0f; +} +)__migraphx__"; + +std::string hip_compiler_version(const std::string& arch) +{ + static std::mutex mutex; + static std::unordered_map ids; + const std::lock_guard lock(mutex); + auto it = ids.find(arch); + if(it != ids.end()) + return it->second; + + std::string id; + try + { + auto cos = compile_hip_src({src_file{"main.cpp", toolchain_probe}}, + {"-std=c++17"}, + arch, + /* quiet */ true); + if(not cos.empty()) + id = md5(std::string(cos.front().begin(), cos.front().end())).substr(0, 12); + } + catch(const std::exception& e) + { + // Callers treat an empty id as "cannot tell toolchains apart" and stop sharing results + // between runs, which is slower but never wrong. + log::warn() << "Cannot identify the hip toolchain: " << e.what(); + } + ids[arch] = id; + return id; +} + bool hip_can_compile(const std::string& src, const std::vector& flags) { src_file input{"main.cpp", src}; diff --git a/src/targets/gpu/compile_hip_code_object.cpp b/src/targets/gpu/compile_hip_code_object.cpp index 868153c2c9e..a7c2c973d7d 100644 --- a/src/targets/gpu/compile_hip_code_object.cpp +++ b/src/targets/gpu/compile_hip_code_object.cpp @@ -29,6 +29,7 @@ #include #include #include +#include namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { @@ -201,20 +202,11 @@ std::size_t compute_block_size(const context& ctx, std::size_t n, std::size_t ma return std::min(std::max(min_block_size, block_size), max_block_size); } -std::vector -compile_hip_raw(context& ctx, const std::string& content, hip_compile_options options) +// Append the parameters that are derived from the options and the device rather than passed in +// by the compiler. Shared with hip_compile_key so the key sees the same final parameter list +// that the compiler does. +static void add_derived_params(const context& ctx, hip_compile_options& options) { - assert(options.global > 0); - assert(options.local > 0); - std::vector srcs = options.additional_src_files; - static auto kernels{::migraphx_kernels()}; - std::transform( - kernels.begin(), - kernels.end(), - std::back_inserter(srcs), - [](const std::pair& elem) { return src_file{elem}; }); - srcs.emplace_back("main.cpp", content); - if(options.global % options.local != 0 and hip_accept_non_uniform_wg()) options.emplace_param("-fno-offload-uniform-block"); else @@ -230,12 +222,54 @@ compile_hip_raw(context& ctx, const std::string& content, hip_compile_options op options.params.insert(options.params.end(), warnings.begin(), warnings.end()); options.emplace_param("-ftemplate-backtrace-limit=0"); options.emplace_param("-Werror"); +} + +std::vector +compile_hip_raw(context& ctx, const std::string& content, hip_compile_options options) +{ + assert(options.global > 0); + assert(options.local > 0); + std::vector srcs = options.additional_src_files; + static auto kernels{::migraphx_kernels()}; + std::transform( + kernels.begin(), + kernels.end(), + std::back_inserter(srcs), + [](const std::pair& elem) { return src_file{elem}; }); + srcs.emplace_back("main.cpp", content); + + add_derived_params(ctx, options); auto cos = compile_hip_src(srcs, options.params, ctx.get_current_device().get_device_name()); if(cos.size() != 1) MIGRAPHX_THROW("No code object"); return cos.front(); } +std::string hip_compile_key(const context& ctx, const hip_src& src) +{ + auto options = src.options; + add_derived_params(ctx, options); + + std::stringstream ss; + ss << "arch=" << ctx.get_current_device().get_device_name() << "\n"; + ss << "kernel=" << options.kernel_name << "\n"; + ss << "global=" << options.global << "\n"; + ss << "local=" << options.local << "\n"; + ss << "output_arg=" << options.output_arg << "\n"; + ss << "params=" << join_strings(options.params, " ") << "\n"; + // The shapes are recorded because they become fields of the code object, not because they + // are part of the source; the source sees them through the generated tensor views below. + for(const auto& s : options.inputs) + ss << "input=" << s << "\n"; + ss << "output=" << options.output << "\n"; + for(const auto& f : options.additional_src_files) + ss << "src=" << f.path << "\n" << f.content << "\n"; + ss << generate_args_hpp(options.virtual_inputs.empty() ? options.inputs + : options.virtual_inputs); + ss << "main.cpp\n" << src.content; + return ss.str(); +} + operation compile_hip_code_object(context& ctx, const std::string& content, hip_compile_options options) { diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 431666fe54c..8a20c18819e 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -105,6 +105,53 @@ struct precompile_op }; MIGRAPHX_REGISTER_OP(precompile_op); +/** + * Apply a compiler_replace to a standalone module so the replacement can be reused. + * + * The instruction is rebuilt in a fresh module whose parameters stand in for its inputs, so the + * resulting fragment refers to those inputs only by position. Any module arguments are consumed + * by the replace, leaving a flat module that no longer refers to the program it came from. + */ +static program make_fragment(const compiler_replace& cr, instruction_ref ins) +{ + program frag; + auto* fm = frag.get_main_module(); + std::vector inputs; + std::transform(ins->inputs().begin(), + ins->inputs().end(), + std::back_inserter(inputs), + [&](const auto& arg) { + return fm->add_parameter(compiled_code::input_name(inputs.size()), + arg->get_shape()); + }); + auto frag_ins = fm->add_instruction(ins->get_operator(), inputs, ins->module_inputs()); + fm->add_return({frag_ins}); + cr.replace(*fm, frag_ins); + // Only dead code is removed here. eliminate_identity would drop the identity that mlir + // inserts to order prefills before the kernel, and memory coloring is done later by the + // module this fragment is spliced into. + run_passes(*fm, {dead_code_elimination{}}); + return frag; +} + +/// Identify the code a compile would produce without producing it. A compiler that cannot +/// describe its output, or that fails trying, gives an empty key, which costs a redundant +/// compile but is always correct. +static std::string compile_key_or_empty(context& ctx, + instruction_ref ins, + const operation& preop, + const value& solution) +{ + try + { + return compile_key(ctx, ins, preop, solution); + } + catch(...) + { + return {}; + } +} + struct dynamic_op_cache { module mod; @@ -234,7 +281,29 @@ struct dynamic_code_object_op { solution = config->solutions.front(); } - auto compiled_op = compile(ctx, ins, preop, solution); + // This runs while the program is being evaluated, so reusing an earlier result matters + // more here than anywhere else. + auto key = compile_key_or_empty(ctx, ins, preop, solution); + auto& bin_cache = ctx.get_binary_cache(); + compiler_replace compiled_op; + if(auto cached = bin_cache.get(ctx, key)) + { + compiled_op.code = *cached; + } + else + { + compiled_op = compile(ctx, ins, preop, solution); + if(compiled_op.code.empty()) + compiled_op.code.fragment = make_fragment(compiled_op, ins); + bin_cache.record_compiled(1); + binary_cache_entry entry; + entry.key = key; + entry.op_name = preop.name(); + entry.problem = config ? config->problem : value{}; + entry.solution = solution; + entry.code = compiled_op.code; + bin_cache.insert(ctx, entry); + } compiled_op.replace(runtime_mod, ins); run_passes(runtime_mod, {dead_code_elimination{}}); @@ -302,6 +371,16 @@ struct compiled_result // forward declared since it requires compile_manager static void replace_inserted_device_ops(context& ctx, module& m); +/// One compilation a plan is waiting on. Candidates are collected before anything is compiled, +/// so identical ones can be found and compiled only once. +struct compile_candidate +{ + std::size_t plan_index = 0; + std::size_t result_index = 0; + value solution = {}; + std::string key = {}; +}; + struct compile_plan { context* ctx; @@ -314,30 +393,73 @@ struct compile_plan { config = get_tuning_config(*ctx, ins, preop, exhaustive); } - template - void insert_compiles(Vector& compiles, const value& solution, std::size_t i) + + std::string get_key(const value& solution) const { - compiles.emplace_back([=] { - try - { - results[i] = compiled_result{compile(*ctx, ins, preop, solution), ins}; - } - catch(const std::exception& e) - { - const auto trace_level = value_of(MIGRAPHX_TRACE_BENCHMARKING{}); - if(trace_level > 0) - std::cerr << "Exception in " + preop.name() + ": " + e.what() << std::endl; - results[i] = nullopt; - } - catch(...) - { - results[i] = nullopt; - } - }); + return compile_key_or_empty(*ctx, ins, preop, solution); + } + + /** + * Compile anyway and check the reused result matches. + * + * A key that fails to capture something the compiler depends on does not cause a miss, it + * hands back the wrong code, so there is nothing to notice at runtime beyond bad numbers. + * Running with this on turns that into a loud failure. + */ + void verify_reuse(const value& solution, const compiled_code& reused) const + { + auto fresh = compile(*ctx, ins, preop, solution); + if(fresh.code.empty()) + fresh.code.fragment = make_fragment(fresh, ins); + if(fresh.code.fragment == reused.fragment) + return; + MIGRAPHX_THROW("Binary cache reused a result that does not match a fresh compile of " + + preop.name() + ".\nReused:\n" + to_string(reused.fragment) + + "\nCompiled:\n" + to_string(fresh.code.fragment)); } - template - void add_compiles(Vector& compiles) + optional run_compile(const value& solution, const std::string& key) const + { + auto& cache = ctx->get_binary_cache(); + if(auto cached = cache.get(*ctx, key)) + { + compiler_replace cr; + cr.code = *cached; + if(cache.verify()) + verify_reuse(solution, *cached); + return cr; + } + try + { + auto cr = compile(*ctx, ins, preop, solution); + // Always replace through the fragment, so the path taken when the result is reused + // is the only path and cannot drift from the one taken when it is compiled. + if(cr.code.empty()) + cr.code.fragment = make_fragment(cr, ins); + cache.record_compiled(1); + binary_cache_entry entry; + entry.key = key; + entry.op_name = preop.name(); + entry.problem = config ? config->problem : value{}; + entry.solution = solution; + entry.code = cr.code; + cache.insert(*ctx, entry); + return cr; + } + catch(const std::exception& e) + { + const auto trace_level = value_of(MIGRAPHX_TRACE_BENCHMARKING{}); + if(trace_level > 0) + std::cerr << "Exception in " + preop.name() + ": " + e.what() << std::endl; + return nullopt; + } + catch(...) + { + return nullopt; + } + } + + void add_candidates(std::vector& candidates, std::size_t plan_index) { if(config.has_value()) { @@ -349,7 +471,7 @@ struct compile_plan if(solution.is_null()) return; results.resize(1); - insert_compiles(compiles, solution, 0); + candidates.push_back({plan_index, 0, solution}); } else { @@ -364,7 +486,7 @@ struct compile_plan { ctx->get_problem_cache().insert(preop.name(), problem, solutions.front()); results.resize(1); - insert_compiles(compiles, solutions.front(), 0); + candidates.push_back({plan_index, 0, solutions.front()}); } else { @@ -372,7 +494,10 @@ struct compile_plan results.resize(solutions.size()); for(auto i : range(solutions.size())) { - insert_compiles(compiles, solutions[i], i); + auto& candidate = candidates.emplace_back(); + candidate.plan_index = plan_index; + candidate.result_index = i; + candidate.solution = solutions[i]; } } } @@ -380,7 +505,7 @@ struct compile_plan else { results.resize(1); - insert_compiles(compiles, value{}, 0); + candidates.push_back({plan_index, 0, value{}}); } } std::string problem_string() const @@ -469,7 +594,7 @@ struct compile_plan auto bundle = compute_benchmark_bundle(*bench_prog.get_main_module()); auto t = time_program(*ctx, std::move(bench_prog), - cr->replace.fill_map, + cr->replace.code.fill_map, bundle, /* nrun */ 20); if(trace_level > 1) @@ -544,6 +669,18 @@ static void par_compile(std::size_t n, F f) par_for(n, n / d, f); } +/// A compile that one or more plans are waiting on, along with the places its result goes. +struct compile_task +{ + /// The plan whose instruction drives the compile. Any plan in targets would do, since they + /// all produce the same code. + std::size_t plan_index = 0; + value solution = {}; + std::string key = {}; + std::vector> targets = {}; + optional result = nullopt; +}; + struct compile_manager { std::vector cps; @@ -560,14 +697,71 @@ struct compile_manager par_compile(cps.size(), [&](auto i) { cps[i].update_config(exhaustive); }); } + /** + * Group the candidates that would compile to the same code. + * + * Since every key is known before any compile starts, each group can be handed to exactly + * one task and the results shared afterwards, so the compiles never have to coordinate. + */ + std::vector make_tasks(const std::vector& candidates) const + { + std::vector tasks; + std::unordered_map task_index; + for(const auto& c : candidates) + { + std::size_t i = tasks.size(); + if(c.key.empty()) + { + tasks.push_back({c.plan_index, c.solution, c.key}); + } + else + { + auto [it, inserted] = task_index.emplace(c.key, i); + if(inserted) + tasks.push_back({c.plan_index, c.solution, c.key}); + i = it->second; + } + tasks[i].targets.emplace_back(c.plan_index, c.result_index); + } + return tasks; + } + void compile(module& m, bool is_root) { - std::vector> compiles; - for(auto& cp : cps) + std::vector candidates; + for(auto i : range(cps.size())) + { + cps[i].add_candidates(candidates, i); + } + par_compile(candidates.size(), [&](auto i) { + candidates[i].key = cps[candidates[i].plan_index].get_key(candidates[i].solution); + }); + + auto tasks = make_tasks(candidates); + if(not cps.empty()) { - cp.add_compiles(compiles); + cps.front().ctx->get_binary_cache().record_reused(candidates.size() - tasks.size()); + } + par_compile(tasks.size(), [&](auto i) { + tasks[i].result = cps[tasks[i].plan_index].run_compile(tasks[i].solution, tasks[i].key); + }); + + for(const auto& task : tasks) + { + for(auto [pi, ri] : task.targets) + { + if(not task.result.has_value()) + { + cps[pi].results[ri] = nullopt; + continue; + } + auto cr = *task.result; + // The fragment is what gets used from here on. The replace function is bound to + // the instruction that produced it, so it must not run against another one. + cr.replace_fn = nullptr; + cps[pi].results[ri] = compiled_result{std::move(cr), cps[pi].ins}; + } } - par_compile(compiles.size(), [&](auto i) { compiles[i](); }); static const auto mxr_path = string_value_of(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR{}); bool dump_mxr = not mxr_path.empty(); @@ -633,6 +827,9 @@ void compile_ops::apply(module_pass_manager& mpm) const { bool is_root = &mpm.get_module() == mpm.get_root_module(); auto& m = mpm.get_module(); + // The cache outlives this pass, since compiling a dynamic shape at evaluation time uses it + // too, so the settings are recorded on it rather than passed along. + ctx->get_binary_cache().configure(cache_settings); compile_manager cm; cm.exhaustive = exhaustive_tune; // Find all precompile ops diff --git a/src/targets/gpu/compile_pointwise.cpp b/src/targets/gpu/compile_pointwise.cpp index ee682cf2c3a..b209caaae0d 100644 --- a/src/targets/gpu/compile_pointwise.cpp +++ b/src/targets/gpu/compile_pointwise.cpp @@ -33,16 +33,18 @@ namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { namespace gpu { -operation -compile_pointwise(context& ctx, const std::vector& in_shapes, const_module_ref pm) +value pointwise_options(const_module_ref pm) { auto pf = gen::generate_pointwise(*pm, "inner_pointwise", true); std::string lambda = "MIGRAPHX_LIFT(inner_pointwise)"; auto kernel_name = gen::generate_name_from_ops(*pm, "kernel"); - return gpu::compile_op("pointwise", - ctx, - in_shapes, - {{"lambda", lambda}, {"preamble", pf}, {"kernel", kernel_name}}); + return {{"lambda", lambda}, {"preamble", pf}, {"kernel", kernel_name}}; +} + +operation +compile_pointwise(context& ctx, const std::vector& in_shapes, const_module_ref pm) +{ + return gpu::compile_op("pointwise", ctx, in_shapes, pointwise_options(pm)); } } // namespace gpu diff --git a/src/targets/gpu/compiled_code.cpp b/src/targets/gpu/compiled_code.cpp new file mode 100644 index 00000000000..b5b8609c93a --- /dev/null +++ b/src/targets/gpu/compiled_code.cpp @@ -0,0 +1,56 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +std::string compiled_code::input_name(std::size_t i) { return param_name(i); } + +void compiled_code::replace(module& m, instruction_ref ins) const +{ + const auto* frag = fragment.get_main_module(); + const auto& inputs = ins->inputs(); + // Seeding the map with the parameters keeps insert_instructions from copying them, so the + // fragment is wired directly onto the inputs of ins. + std::unordered_map map_ins; + for(auto i : range(inputs.size())) + { + map_ins[frag->get_parameter(input_name(i))] = inputs[i]; + } + auto returns = m.insert_instructions(ins, frag, &map_ins); + if(returns.size() != 1) + MIGRAPHX_THROW("compiled_code: expected a single output from the fragment but got " + + std::to_string(returns.size())); + m.replace_instruction(ins, returns.front()); +} + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx diff --git a/src/targets/gpu/compiler.cpp b/src/targets/gpu/compiler.cpp index 20660d75d97..bc7ae92f849 100644 --- a/src/targets/gpu/compiler.cpp +++ b/src/targets/gpu/compiler.cpp @@ -35,6 +35,7 @@ struct compiler_handle compiler_compile compile; compiler_compile_op compile_op; compiler_tuning_config get_tuning_config; + compiler_compile_key compile_key; }; } // namespace @@ -47,9 +48,10 @@ static auto& compiler_map() void register_compiler(const std::string& name, compiler_compile c, compiler_compile_op cop, - compiler_tuning_config ctg) + compiler_tuning_config ctg, + compiler_compile_key ck) { - compiler_map()[name] = {std::move(c), std::move(cop), std::move(ctg)}; + compiler_map()[name] = {std::move(c), std::move(cop), std::move(ctg), std::move(ck)}; } bool has_compiler_for(const std::string& name) { return compiler_map().count(name) > 0; } @@ -73,6 +75,13 @@ get_tuning_config(context& ctx, instruction_ref ins, const operation& op, bool e return compiler_map().at(op.name()).get_tuning_config(ctx, ins, op, exhaustive); } +std::string +compile_key(context& ctx, instruction_ref ins, const operation& op, const value& solution) +{ + assert(contains(compiler_map(), op.name())); + return compiler_map().at(op.name()).compile_key(ctx, ins, op, solution); +} + } // namespace gpu } // namespace MIGRAPHX_INLINE_NS } // namespace migraphx diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp new file mode 100644 index 00000000000..e9e63fda7bd --- /dev/null +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -0,0 +1,133 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_HPP +#define MIGRAPHX_GUARD_GPU_BINARY_CACHE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +struct context; + +/// What gets written to disk for one compiled kernel. The problem and solution are recorded so +/// an entry can be identified without recompiling anything. +struct binary_cache_entry +{ + std::string key = {}; + std::string op_name = {}; + value problem = {}; + value solution = {}; + compiled_code code = {}; + + template + static auto reflect(Self& self, F f) + { + return pack(f(self.key, "key"), + f(self.op_name, "op_name"), + f(self.problem, "problem"), + f(self.solution, "solution"), + f(self.code, "code")); + } +}; + +/// Counts of what the cache did, for tests and for reporting. +struct binary_cache_stats +{ + std::size_t compiled = 0; + std::size_t reused = 0; + std::size_t hits = 0; + std::size_t misses = 0; +}; + +/// How the cache behaves for one compile. Supplied through the target's backend options, with +/// the MIGRAPHX_BINARY_CACHE environment variable as the default for the directory. +struct binary_cache_settings +{ + /// Where entries are kept. Nothing is written to disk when this is empty, though results + /// are still shared in memory. + std::string path = {}; + /// Compile even when a result can be reused, and fail if the two disagree. + bool verify = false; + + MIGRAPHX_GPU_EXPORT static binary_cache_settings defaults(); +}; + +/** + * Compiled kernels, keyed by a string describing what the compiler was given. + * + * Results are held in memory for the life of the context and, when a cache directory is + * configured, written to disk so later runs can reuse them. Entries are keyed on the exact + * source handed to the backend, so anything that would change the generated code changes the + * key; things outside the source, such as the toolchain and the embedded kernel headers, are + * separated by the directory the entries live in. + */ +struct MIGRAPHX_GPU_EXPORT binary_cache +{ + /// Look up a key, consulting memory first and then the cache directory. + optional get(const context& ctx, const std::string& key); + + /// Record a compiled result under its key. + void insert(const context& ctx, const binary_cache_entry& entry); + + /// Note that a result was shared rather than compiled again. + void record_compiled(std::size_t n); + void record_reused(std::size_t n); + + binary_cache_stats get_stats(); + + void configure(const binary_cache_settings& s); + + /// True when a cache directory is configured. Nothing is written to disk otherwise, but + /// results are still shared in memory. + bool enabled(); + + /// True when reused results should be checked against a fresh compile. + bool verify(); + + /// Names the directory holding entries that this build can use: the entry format, the + /// toolchain, a digest of the embedded kernel headers, and the rocMLIR build. Empty when + /// the toolchain cannot be identified, in which case nothing is read from or written to + /// disk, since entries from different toolchains could not be told apart. + static std::string version_dir(const context& ctx); + + private: + std::mutex mutex; + std::unordered_map memo; + binary_cache_stats stats; + binary_cache_settings settings = binary_cache_settings::defaults(); +}; + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx +#endif // MIGRAPHX_GUARD_GPU_BINARY_CACHE_HPP diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp index ea024d91e5e..27b44f1a482 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp @@ -54,6 +54,15 @@ struct hiprtc_src_file } }; +/** + * Identifies the toolchain that turns kernel source into code objects for an architecture. + * + * The toolchain is loaded at runtime, and what it reports about itself is coarser than what it + * generates, so this compiles a fixed kernel and identifies the result. Returns an empty string + * if the toolchain cannot be identified. + */ +MIGRAPHX_GPU_EXPORT std::string hip_compiler_version(const std::string& arch); + MIGRAPHX_GPU_EXPORT bool hip_can_compile(const std::string& src, const std::vector& flags); diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp index 130821db7b7..48951486e90 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp @@ -64,13 +64,31 @@ struct hip_compile_options void set_launch_params(const value& v, std::size_t default_global, std::size_t default_local = 1024) { - set_launch_params( - v, [=](auto) { return default_global; }, default_local); + set_launch_params(v, [=](auto) { return default_global; }, default_local); } void emplace_param(std::string_view s) { params.emplace_back(s); } }; +/// A kernel to be compiled: its source along with the options that shape the result. Compilers +/// build one of these so the same description can be used to compile the kernel or to identify +/// it without compiling. +struct hip_src +{ + std::string content; + hip_compile_options options; +}; + +/** + * A string identifying the code object that would be produced from src. + * + * This covers everything handed to the compiler: the source, the tensor views generated for it, + * the final parameter list including the launch bounds and device defines, and the fields that + * end up on the code object rather than in the binary. The embedded kernel headers are not + * included, since callers distinguish those separately. + */ +MIGRAPHX_GPU_EXPORT std::string hip_compile_key(const context& ctx, const hip_src& src); + /// Compute global for n elements, but max out on target-specific upper limit MIGRAPHX_GPU_EXPORT std::function compute_global_for(const context& ctx, std::size_t n, std::size_t over = 1); diff --git a/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp b/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp index 1898899f1b5..979b182c6ea 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp @@ -25,6 +25,7 @@ #define MIGRAPHX_GUARD_GPU_COMPILE_OPS_HPP #include +#include #include namespace migraphx { @@ -40,6 +41,7 @@ struct MIGRAPHX_GPU_EXPORT compile_ops { context* ctx = nullptr; bool exhaustive_tune = false; + binary_cache_settings cache_settings{}; std::string name() const { return "gpu::compile_ops"; } void apply(module_pass_manager& mpm) const; }; diff --git a/src/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp b/src/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp index 8e6dc229aaf..35d12795eb4 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp @@ -36,6 +36,10 @@ inline namespace MIGRAPHX_INLINE_NS { namespace gpu { +/// The options the pointwise compiler is driven with for a pointwise submodule. Exposed so the +/// same options can be used to identify the kernel without compiling it. +value pointwise_options(const_module_ref pm); + operation compile_pointwise(context& ctx, const std::vector& in_shapes, const_module_ref pm); diff --git a/src/targets/gpu/include/migraphx/gpu/compiler.hpp b/src/targets/gpu/include/migraphx/gpu/compiler.hpp index f20053db052..d6b29774d55 100644 --- a/src/targets/gpu/include/migraphx/gpu/compiler.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compiler.hpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -72,11 +73,13 @@ struct compiler_replace { } + /// The serializable result of the compile. When this is set it supersedes replace_fn: the + /// fragment already describes the replacement, so it is used directly. + compiled_code code = {}; std::vector code_objects = {}; std::function replace_fn = nullptr; std::function trace_fn = nullptr; - std::unordered_map fill_map = {}; template static auto make_replace(F f) @@ -96,7 +99,9 @@ struct compiler_replace void replace(module& m, instruction_ref ins) const { - if(replace_fn) + if(not code.empty()) + code.replace(m, ins); + else if(replace_fn) replace_fn(*this, m, ins); else { @@ -121,11 +126,14 @@ using compiler_compile_op = std::function& inputs, const value&)>; using compiler_tuning_config = std::function(context&, instruction_ref, const operation&, bool)>; +using compiler_compile_key = + std::function; MIGRAPHX_GPU_EXPORT void register_compiler(const std::string& name, compiler_compile c, compiler_compile_op cop, - compiler_tuning_config ctg); + compiler_tuning_config ctg, + compiler_compile_key ck); MIGRAPHX_GPU_EXPORT bool has_compiler_for(const std::string& name); MIGRAPHX_GPU_EXPORT compiler_replace compile(context& ctx, @@ -139,6 +147,17 @@ MIGRAPHX_GPU_EXPORT operation compile_op(const std::string& name, MIGRAPHX_GPU_EXPORT optional get_tuning_config(context& ctx, instruction_ref ins, const operation& op, bool exhaustive); +/** + * A string that uniquely identifies the code the compiler would produce, without producing it. + * + * This is the text handed to the backend compiler, so it captures the generated source along + * with everything else that shapes the result, and no separate accounting of shapes, launch + * parameters, or device properties is needed. An empty string means the compiler cannot + * describe its output this way, and its result must not be cached. + */ +MIGRAPHX_GPU_EXPORT std::string +compile_key(context& ctx, instruction_ref ins, const operation& op, const value& solution); + template void register_compiler() { @@ -151,7 +170,8 @@ void register_compiler() return c.invoke_compile(rank<1>{}, std::forward(xs)...); }, [=](auto&&... xs) { return c.compile_op(std::forward(xs)...); }, - [=](auto&&... xs) { return c.get_tuning_config(std::forward(xs)...); }); + [=](auto&&... xs) { return c.get_tuning_config(std::forward(xs)...); }, + [=](auto&&... xs) { return c.compile_key(std::forward(xs)...); }); } } @@ -177,6 +197,13 @@ struct compiler : auto_register_compiler return nullopt; } + /// Compilers opt in to caching by overriding this. Until one does, an empty key keeps its + /// results out of the cache, which is always correct if never fast. + std::string compile_key(context&, instruction_ref, const operation&, const value&) const + { + return {}; + } + // TODO: make it a compile error if this is not overridden by Derived rather than runtime error. // Could rename Derived function to something like compile_op_impl. // Or refactor to type-erased interface. @@ -186,17 +213,17 @@ struct compiler : auto_register_compiler } template - auto - invoke_compile(rank<1>, context& ctx, instruction_ref ins, operation op, const value& solution) - const -> decltype(std::declval().compile(ctx, ins, std::move(op), solution)) + auto invoke_compile( + rank<1>, context& ctx, instruction_ref ins, operation op, const value& solution) const + -> decltype(std::declval().compile(ctx, ins, std::move(op), solution)) { return derived().compile(ctx, ins, std::move(op), solution); } template - auto - invoke_compile(rank<0>, context& ctx, instruction_ref ins, operation op, const value& solution) - const -> decltype(std::declval().compile(ctx, ins, std::move(op))) + auto invoke_compile( + rank<0>, context& ctx, instruction_ref ins, operation op, const value& solution) const + -> decltype(std::declval().compile(ctx, ins, std::move(op))) { assert(solution.empty()); (void)solution; diff --git a/src/targets/gpu/include/migraphx/gpu/context.hpp b/src/targets/gpu/include/migraphx/gpu/context.hpp index 28a409932fe..1061a91b019 100644 --- a/src/targets/gpu/include/migraphx/gpu/context.hpp +++ b/src/targets/gpu/include/migraphx/gpu/context.hpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -488,6 +489,9 @@ struct context pc->auto_save = true; } + /// Compiled kernels shared across every module compiled with this context. + binary_cache& get_binary_cache() const { return *bc; } + private: // TODO: Make this a vector to support multiple devices std::shared_ptr current_device; @@ -501,6 +505,7 @@ struct context shared begin_event = nullptr; shared finish_event = nullptr; std::shared_ptr pc = nullptr; + std::shared_ptr bc = std::make_shared(); }; inline void migraphx_to_value(value& v, const context& ctx) { v = ctx.to_value(); } diff --git a/src/targets/gpu/include/migraphx/gpu/mlir.hpp b/src/targets/gpu/include/migraphx/gpu/mlir.hpp index 4c85da3322a..795f44e1ed0 100644 --- a/src/targets/gpu/include/migraphx/gpu/mlir.hpp +++ b/src/targets/gpu/include/migraphx/gpu/mlir.hpp @@ -60,6 +60,18 @@ MIGRAPHX_GPU_EXPORT mlir_code_object compile_mlir(const context& migraphx_ctx, const std::vector& in_shapes, const value& solution); +/** + * A string identifying the code object compile_mlir would produce for these arguments. + * + * This is the mlir module as parsed, before any pipeline runs, so it is far cheaper than + * compiling. Callers that already ran get_tuning_config_mlir have the same text available as + * tuning_config::detailed_problem_info and can use that instead of calling this. + */ +MIGRAPHX_GPU_EXPORT std::string mlir_compile_key(const context& migraphx_ctx, + module m, + const std::vector& in_shapes, + const value& solution); + MIGRAPHX_GPU_EXPORT instruction_ref insert_mlir(module& m, instruction_ref ins, code_object_op co, diff --git a/src/targets/gpu/jit/gather.cpp b/src/targets/gpu/jit/gather.cpp index 9dc17db0972..66028fcea3f 100644 --- a/src/targets/gpu/jit/gather.cpp +++ b/src/targets/gpu/jit/gather.cpp @@ -61,7 +61,7 @@ struct gather_compiler : compiler { std::vector names() const { return {"gather"}; } - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + hip_src make_src(const context& ctx, const std::vector& inputs, const value& v) const { hip_compile_options options; const auto& out_s = inputs.back(); @@ -75,13 +75,25 @@ struct gather_compiler : compiler auto src = interpolate_string(gather_kernel, {{"axis", axis}}); - return compile_hip_code_object(ctx, src, options); + return {src, options}; + } + + operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + { + auto src = make_src(ctx, inputs, v); + return compile_hip_code_object(ctx, src.content, src.options); } compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const { return compile_op(ctx, to_shapes(ins->inputs()), op.to_value()); } + + std::string + compile_key(const context& ctx, instruction_ref ins, const operation& op, const value&) const + { + return hip_compile_key(ctx, make_src(ctx, to_shapes(ins->inputs()), op.to_value())); + } }; } // namespace gpu diff --git a/src/targets/gpu/jit/mlir.cpp b/src/targets/gpu/jit/mlir.cpp index 7bc73a80220..8e043f7eb41 100644 --- a/src/targets/gpu/jit/mlir.cpp +++ b/src/targets/gpu/jit/mlir.cpp @@ -193,7 +193,7 @@ struct mlir_compiler : compiler { auto id = param.value()->get_shape().type_string() + migraphx::shape::to_sizes_string({param.value()->get_shape()}); - cr.fill_map[id] = static_cast(fill_val); + cr.code.fill_map[id] = static_cast(fill_val); } } } @@ -221,7 +221,7 @@ struct mlir_compiler : compiler input_args.pop_back(); auto split_ins = find_final_split(gemm_like_ins); std::array mod_splits = smod->split(input_args, {split_ins}); - auto dot_mlir_inputs = to_shapes(mod_splits[0].inputs); + auto dot_mlir_inputs = to_shapes(mod_splits[0].inputs); // add alloc for the gemm output dot_mlir_inputs.push_back(mod_splits[0].mod.get_output_shapes().front()); mlir_code_object cop1 = compile_mlir(ctx, mod_splits[0].mod, dot_mlir_inputs, solution); @@ -236,6 +236,17 @@ struct mlir_compiler : compiler return cr; } + std::string compile_key(const context& ctx, + instruction_ref ins, + const operation&, + const value& solution) const + { + // The split into a separate gemm and pointwise kernel is decided from the submodule, + // the context and the solution, so the key covers both shapes of the result. + auto* smod = ins->module_inputs().front(); + return mlir_compile_key(ctx, *smod, to_shapes(ins->inputs()), solution); + } + compiler_replace insert(const mlir_code_object& mco) const { return {std::vector{mco.cop}, diff --git a/src/targets/gpu/jit/pad.cpp b/src/targets/gpu/jit/pad.cpp index f9aa3c10c47..a4830624d83 100644 --- a/src/targets/gpu/jit/pad.cpp +++ b/src/targets/gpu/jit/pad.cpp @@ -64,7 +64,7 @@ struct pad_compiler : compiler { std::vector names() const { return {"pad"}; } - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + hip_src make_src(const context& ctx, const std::vector& inputs, const value& v) const { auto padding = v.at("pads").to_vector(); auto input_lens = inputs.front().lens(); @@ -118,13 +118,25 @@ struct pad_compiler : compiler {{"pad_val", to_string(pad_val_string)}, {"offsets", to_string_range(roffsets)}, {"pad_mode", pad_mode_string}}); - return compile_hip_code_object(ctx, src, options); + return {src, options}; + } + + operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + { + auto src = make_src(ctx, inputs, v); + return compile_hip_code_object(ctx, src.content, src.options); } compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const { return compile_op(ctx, to_shapes(ins->inputs()), op.to_value()); } + + std::string + compile_key(const context& ctx, instruction_ref ins, const operation& op, const value&) const + { + return hip_compile_key(ctx, make_src(ctx, to_shapes(ins->inputs()), op.to_value())); + } }; } // namespace gpu } // namespace MIGRAPHX_INLINE_NS diff --git a/src/targets/gpu/jit/pointwise.cpp b/src/targets/gpu/jit/pointwise.cpp index c1baf164c68..9377471b389 100644 --- a/src/targets/gpu/jit/pointwise.cpp +++ b/src/targets/gpu/jit/pointwise.cpp @@ -71,18 +71,18 @@ struct pointwise_compiler : compiler else return 1; } - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + hip_src make_src(context& ctx, const std::vector& inputs, const value& v) const { hip_compile_options options; options.inputs = flatten_tuple_shapes(inputs); options.output = inputs.back(); options.virtual_inputs = reduce_dims(normalize_permutation(options.inputs)); options.emplace_param("-Wno-float-equal"); - auto axis = find_fast_axis(options.virtual_inputs); - auto vec = vectorize::elements(ctx, axis, options.virtual_inputs); - options.kernel_name = v.get("kernel", "kernel"); - auto noutputs = options.inputs.size() - inputs.size() + 1; - auto t = tile::elements(options.virtual_inputs, noutputs); + auto axis = find_fast_axis(options.virtual_inputs); + auto vec = vectorize::elements(ctx, axis, options.virtual_inputs); + options.kernel_name = v.get("kernel", "kernel"); + auto noutputs = options.inputs.size() - inputs.size() + 1; + auto t = tile::elements(options.virtual_inputs, noutputs); // auto t = tile{}; if(t.ntiles == 0) options.set_launch_params( @@ -100,24 +100,35 @@ struct pointwise_compiler : compiler {"tiled", t.ntiles > 0 ? "true" : "false"}, {"noutputs", std::to_string(noutputs)}, {"preamble", v.get("preamble", std::string{})}}); - return compile_hip_code_object(ctx, src, options); + return {src, options}; } - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const + operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + { + auto src = make_src(ctx, inputs, v); + return compile_hip_code_object(ctx, src.content, src.options); + } + + static value make_value(instruction_ref ins, const operation& op) { if(contains({"layout", "contiguous", "hip::copy"}, op.name())) { - return compile_op(ctx, - to_shapes(ins->inputs()), - {{"lambda", "[](auto x) { return make_tuple(x); }"}, - {"kernel", to_c_id(op.name()) + "_kernel"}}); - } - else - { - assert(not ins->module_inputs().empty()); - const_module_ref pm = ins->module_inputs().front(); - return compile_pointwise(ctx, to_shapes(ins->inputs()), pm); + return {{"lambda", "[](auto x) { return make_tuple(x); }"}, + {"kernel", to_c_id(op.name()) + "_kernel"}}; } + assert(not ins->module_inputs().empty()); + return pointwise_options(ins->module_inputs().front()); + } + + compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const + { + return compile_op(ctx, to_shapes(ins->inputs()), make_value(ins, op)); + } + + std::string + compile_key(context& ctx, instruction_ref ins, const operation& op, const value&) const + { + return hip_compile_key(ctx, make_src(ctx, to_shapes(ins->inputs()), make_value(ins, op))); } }; } // namespace gpu diff --git a/src/targets/gpu/jit/reduce.cpp b/src/targets/gpu/jit/reduce.cpp index 83b33143450..50a7c75adb4 100644 --- a/src/targets/gpu/jit/reduce.cpp +++ b/src/targets/gpu/jit/reduce.cpp @@ -110,8 +110,8 @@ static shape get_output_shape(const shape& s, const std::vector& axes) template static std::string get_reduce_algo(context& ctx, const std::vector& inputs, ReduceLens rlens) { - const auto init = std::numeric_limits::max(); - auto relements = std::accumulate(rlens.begin(), rlens.end(), 1, std::multiplies<>{}); + const auto init = std::numeric_limits::max(); + auto relements = std::accumulate(rlens.begin(), rlens.end(), 1, std::multiplies<>{}); bool is_strided_reduce = std::all_of(inputs.begin(), inputs.end(), [&](const shape& input) { // The minimum stride auto min_stride = std::inner_product( @@ -221,7 +221,7 @@ struct simple_reduce_compiler : compiler return inputs.front().elements() / inputs.back().elements(); } - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + hip_src make_src(context& ctx, const std::vector& inputs, const value& v) const { hip_compile_options options; options.inputs = inputs; @@ -236,7 +236,7 @@ struct simple_reduce_compiler : compiler // Vectorize if the axis is a reduction axis if(options.virtual_inputs.back().lens()[faxis] == 1) vec = vectorize::elements(ctx, faxis, options.virtual_inputs); - auto relements = get_reduce_elements(options.virtual_inputs) / vec.size; + auto relements = get_reduce_elements(options.virtual_inputs) / vec.size; if(algo == "block") { auto block_size = compute_block_size(ctx, relements, 256); @@ -265,18 +265,24 @@ struct simple_reduce_compiler : compiler options.kernel_name = "reduce_kernel"; std::string identity = "[](auto x) { return x; }"; auto src = interpolate_string(simple_reduce_kernel, - {{"reduction", v.at("reduction").to()}, - {"init", v.get("init", std::string{"0"})}, - {"read", v.get("read", identity)}, - {"write", v.get("write", identity)}, - {"algo", algo}, - {"transformers", make_transformer_args(vec)}, - {"preamble", v.get("preamble", std::string{})}}); + {{"reduction", v.at("reduction").to()}, + {"init", v.get("init", std::string{"0"})}, + {"read", v.get("read", identity)}, + {"write", v.get("write", identity)}, + {"algo", algo}, + {"transformers", make_transformer_args(vec)}, + {"preamble", v.get("preamble", std::string{})}}); options.emplace_param("-Wno-float-equal"); - return compile_hip_code_object(ctx, src, options); + return {src, options}; } - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const + operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + { + auto src = make_src(ctx, inputs, v); + return compile_hip_code_object(ctx, src.content, src.options); + } + + static value make_value(instruction_ref ins, const operation& op) { value v = value::object{}; reduce_op r{}; @@ -285,7 +291,18 @@ struct simple_reduce_compiler : compiler v["read"] = r.read; v["write"] = r.write; v["init"] = r.init; - return compile_op(ctx, to_shapes(ins->inputs()), v); + return v; + } + + compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const + { + return compile_op(ctx, to_shapes(ins->inputs()), make_value(ins, op)); + } + + std::string + compile_key(context& ctx, instruction_ref ins, const operation& op, const value&) const + { + return hip_compile_key(ctx, make_src(ctx, to_shapes(ins->inputs()), make_value(ins, op))); } }; @@ -362,7 +379,7 @@ struct fused_reduce_compiler : compiler { std::vector names() const { return {"fused_reduce", "split_fused_reduce"}; } - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + hip_src make_src(context& ctx, const std::vector& inputs, const value& v) const { auto plan = compute_fused_reduce_plan(ctx, inputs, v); auto noutputs = plan.finputs.size() - inputs.size() + 1; @@ -417,22 +434,39 @@ struct fused_reduce_compiler : compiler {"noutputs", std::to_string(noutputs)}, {"preamble", v.get("preamble", std::string{})}}); options.emplace_param("-Wno-float-equal"); - return compile_hip_code_object(ctx, src, options); + return {src, options}; } - compiler_replace - compile(context& ctx, instruction_ref ins, const operation& op, const value& solution) const + operation compile_op(context& ctx, const std::vector& inputs, const value& v) const + { + auto src = make_src(ctx, inputs, v); + return compile_hip_code_object(ctx, src.content, src.options); + } + + static value make_value(instruction_ref ins, const operation& op, const value& solution) { assert(not ins->module_inputs().empty()); - auto v = op.to_value(); + auto v = op.to_value(); for(const auto& x : solution) v.insert(x); auto* rm = ins->module_inputs().front(); - auto shapes = to_shapes(ins->inputs()); v["preamble"] = generate_reduce(*rm, "fused_reduce_op"); v["lambda"] = "MIGRAPHX_LIFT(fused_reduce_op)"; v["kernel"] = generate_name_from_ops(*rm) + "_kernel"; - return compile_op(ctx, shapes, v); + return v; + } + + compiler_replace + compile(context& ctx, instruction_ref ins, const operation& op, const value& solution) const + { + return compile_op(ctx, to_shapes(ins->inputs()), make_value(ins, op, solution)); + } + + std::string + compile_key(context& ctx, instruction_ref ins, const operation& op, const value& solution) const + { + return hip_compile_key( + ctx, make_src(ctx, to_shapes(ins->inputs()), make_value(ins, op, solution))); } optional diff --git a/src/targets/gpu/mlir.cpp b/src/targets/gpu/mlir.cpp index ad02ecb8918..8e64aa98c8e 100644 --- a/src/targets/gpu/mlir.cpp +++ b/src/targets/gpu/mlir.cpp @@ -1330,6 +1330,29 @@ mlir_code_object compile_mlir(const context& migraphx_ctx, return mco; } +std::string mlir_compile_key(const context& migraphx_ctx, + module m, + const std::vector& in_shapes, + const value& solution) +{ + adjust_param_shapes(m, in_shapes); + prepare(m); + + mlir_program mp; + mp.set_gpu_properties(migraphx_ctx); + mp.parse(m, in_shapes); + + std::stringstream ss; + ss << "arch=" << migraphx_ctx.get_current_device().get_device_name() << "\n"; + ss << "solution=" << solution << "\n"; + // The shapes are listed even though the parsed module already reflects them, because they + // also become fields of the code object that the caller inserts. + for(const auto& s : in_shapes) + ss << "input=" << s << "\n"; + ss << mlir_print(&mlirOperationPrint, mlirModuleGetOperation(mp.mmodule.get())); + return ss.str(); +} + instruction_ref insert_mlir(module& m, instruction_ref ins, code_object_op co, @@ -1418,6 +1441,11 @@ mlir_code_object compile_mlir(const context&, module, const std::vector&, return {}; } +std::string mlir_compile_key(const context&, module, const std::vector&, const value&) +{ + return {}; +} + instruction_ref // cppcheck-suppress funcArgNamesDifferent insert_mlir(module& m, instruction_ref, code_object_op co, const std::vector&) diff --git a/src/targets/gpu/target.cpp b/src/targets/gpu/target.cpp index 33cf7a866e6..a23b81ae0b5 100644 --- a/src/targets/gpu/target.cpp +++ b/src/targets/gpu/target.cpp @@ -106,11 +106,17 @@ namespace { struct backend_options { std::vector mlss_use_specific_ops = {}; + /// Where compiled kernels are cached between runs. Defaults to the environment setting. + std::string binary_cache = binary_cache_settings::defaults().path; + /// Compile even when a kernel could be reused, and fail if the two disagree. + bool binary_cache_verify = false; template static auto reflect(Self& self, F f) { - return pack(f(self.mlss_use_specific_ops, "mlss_use_specific_ops")); + return pack(f(self.mlss_use_specific_ops, "mlss_use_specific_ops"), + f(self.binary_cache, "binary_cache"), + f(self.binary_cache_verify, "binary_cache_verify")); } }; @@ -251,7 +257,9 @@ struct pipeline_factory adjust_allocation{gpu_allocation_model{}}, dead_code_elimination{}, lower_device_ops{}, - compile_ops{get_context(), options.exhaustive_tune}, + compile_ops{get_context(), + options.exhaustive_tune, + {backend_opts.binary_cache, backend_opts.binary_cache_verify}}, dead_code_elimination{}, promote_literals{}, dead_code_elimination{}, diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp new file mode 100644 index 00000000000..2b92ee9755b --- /dev/null +++ b/test/gpu/binary_cache.cpp @@ -0,0 +1,289 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static migraphx::program pointwise_program() +{ + migraphx::program p; + auto* mm = p.get_main_module(); + migraphx::shape s{migraphx::shape::float_type, {4, 8}}; + auto x = mm->add_parameter("x", s); + auto y = mm->add_parameter("y", s); + auto add = mm->add_instruction(migraphx::make_op("add"), x, y); + auto relu = mm->add_instruction(migraphx::make_op("relu"), add); + auto mul = mm->add_instruction(migraphx::make_op("mul"), relu, x); + mm->add_return({mul}); + return p; +} + +// Independent branches of the same shape lower to separate instructions that cannot fuse into +// each other, but they all produce the same kernel, so only one of them should be compiled. +static migraphx::program repeated_reduce(std::size_t n) +{ + migraphx::program p; + auto* mm = p.get_main_module(); + migraphx::shape s{migraphx::shape::float_type, {4, 8}}; + std::vector branches; + for(std::size_t i = 0; i < n; i++) + { + auto x = mm->add_parameter("x" + std::to_string(i), s); + auto mul = mm->add_instruction(migraphx::make_op("mul"), x, x); + branches.push_back( + mm->add_instruction(migraphx::make_op("reduce_sum", {{"axes", {1}}}), mul)); + } + mm->add_return({mm->add_instruction(migraphx::make_op("concat", {{"axis", 1}}), branches)}); + return p; +} + +static migraphx::compile_options cache_options(const migraphx::fs::path& dir, bool verify = false) +{ + migraphx::compile_options options; + migraphx::set_backend_options( + options, {{"binary_cache", dir.string()}, {"binary_cache_verify", verify}}); + return options; +} + +static migraphx::gpu::binary_cache_stats +compile_and_get_stats(migraphx::program p, const migraphx::compile_options& options) +{ + p.compile(migraphx::make_target("gpu"), options); + return migraphx::any_cast(p.get_context()) + .get_binary_cache() + .get_stats(); +} + +TEST_CASE(identical_kernels_compile_once) +{ + migraphx::tmp_dir td{"binary-cache"}; + const std::size_t n = 6; + auto stats = compile_and_get_stats(repeated_reduce(n), cache_options(td.path)); + EXPECT(stats.reused >= n - 1); + EXPECT(stats.compiled < n); +} + +// The whole point of the cache: a program compiled once should not compile again. The second +// compile gets a fresh context, so an empty in-memory cache, and everything it reuses has to +// come back off disk. +TEST_CASE(second_compile_reads_from_disk) +{ + migraphx::tmp_dir td{"binary-cache"}; + auto options = cache_options(td.path); + + auto cold = compile_and_get_stats(pointwise_program(), options); + EXPECT(cold.compiled > 0); + EXPECT(cold.misses > 0); + EXPECT(cold.hits == 0); + + auto warm = compile_and_get_stats(pointwise_program(), options); + EXPECT(warm.hits == cold.misses); + EXPECT(warm.compiled == 0); + EXPECT(warm.misses == 0); +} + +// Without a directory nothing is written, but results are still shared within the compile. +TEST_CASE(no_directory_keeps_results_in_memory) +{ + migraphx::tmp_dir td{"binary-cache"}; + auto stats = compile_and_get_stats(repeated_reduce(4), cache_options("")); + EXPECT(stats.reused > 0); + EXPECT(migraphx::fs::is_empty(td.path)); +} + +// A result taken from the cache has to produce the same numbers as one that was just compiled. +TEST_CASE(cached_results_match_reference) +{ + migraphx::tmp_dir td{"binary-cache"}; + auto options = cache_options(td.path); + + auto p_ref = pointwise_program(); + p_ref.compile(migraphx::make_target("ref")); + + migraphx::shape s{migraphx::shape::float_type, {4, 8}}; + auto x = migraphx::generate_argument(s, 0); + auto y = migraphx::generate_argument(s, 1); + + auto ref_result = p_ref.eval({{"x", x}, {"y", y}}).back(); + + // Compile once to fill the cache, then again so the result comes back out of it. + auto warmup = pointwise_program(); + warmup.compile(migraphx::make_target("gpu"), options); + + auto t = migraphx::make_target("gpu"); + auto p = pointwise_program(); + p.compile(t, options); + + migraphx::parameter_map params; + for(auto&& [name, shape] : p.get_parameter_shapes()) + { + if(name == "x") + params[name] = t.copy_to(x); + else if(name == "y") + params[name] = t.copy_to(y); + else + params[name] = t.allocate(shape); + } + auto gpu_result = t.copy_from(p.eval(params).back()); + + std::vector ref_vec; + std::vector gpu_vec; + ref_result.visit([&](auto v) { ref_vec.assign(v.begin(), v.end()); }); + gpu_result.visit([&](auto v) { gpu_vec.assign(v.begin(), v.end()); }); + + EXPECT(migraphx::verify::verify_rms_range(ref_vec, gpu_vec)); +} + +// With verification on, every reused result is checked against a fresh compile. A clean run +// means the keys really do capture what the compilers depend on. +TEST_CASE(verified_reuse_matches_fresh_compiles) +{ + migraphx::tmp_dir td{"binary-cache"}; + auto options = cache_options(td.path, /* verify */ true); + + compile_and_get_stats(pointwise_program(), options); + auto stats = compile_and_get_stats(pointwise_program(), options); + EXPECT(stats.hits > 0); +} + +// A damaged entry must cost a recompile and nothing more. +TEST_CASE(corrupt_entry_is_ignored) +{ + migraphx::tmp_dir td{"binary-cache"}; + auto options = cache_options(td.path); + + auto p = pointwise_program(); + p.compile(migraphx::make_target("gpu"), options); + + std::size_t truncated = 0; + for(const auto& entry : migraphx::fs::recursive_directory_iterator(td.path)) + { + if(entry.path().extension() != ".mxr") + continue; + migraphx::write_buffer(entry.path(), std::vector(8, 0)); + truncated++; + } + EXPECT(truncated > 0); + + auto stats = compile_and_get_stats(pointwise_program(), options); + EXPECT(stats.hits == 0); + EXPECT(stats.compiled > 0); +} + +TEST_CASE(entry_round_trip) +{ + migraphx::gpu::compiled_code code; + { + auto* fm = code.fragment.get_main_module(); + auto x = fm->add_parameter(migraphx::gpu::compiled_code::input_name(0), + {migraphx::shape::float_type, {2}}); + fm->add_return({fm->add_instruction(migraphx::make_op("abs"), x)}); + } + code.fill_map["float_type{2}"] = 3.0; + + migraphx::gpu::binary_cache_entry entry; + entry.key = "some-key"; + entry.op_name = "pointwise"; + entry.solution = migraphx::value{{"algo", "block"}}; + entry.code = code; + + auto buffer = migraphx::to_msgpack(migraphx::to_value(entry)); + migraphx::gpu::binary_cache_entry loaded; + migraphx::from_value(migraphx::from_msgpack(buffer), loaded); + + EXPECT(loaded.key == entry.key); + EXPECT(loaded.op_name == entry.op_name); + EXPECT(loaded.solution == entry.solution); + EXPECT(loaded.code.fill_map == entry.code.fill_map); + EXPECT(*loaded.code.fragment.get_main_module() == *entry.code.fragment.get_main_module()); +} + +// The key has to cover everything handed to the compiler, not just the source text. Two +// kernels that differ only in their tensor views or launch bounds must not share an entry. +TEST_CASE(key_covers_more_than_the_source) +{ + migraphx::gpu::context ctx; + migraphx::shape s{migraphx::shape::float_type, {4, 8}}; + + migraphx::gpu::hip_src src; + src.content = "// kernel source"; + src.options.inputs = {s, s}; + src.options.output = s; + src.options.global = 1024; + src.options.local = 256; + src.options.kernel_name = "kernel"; + auto base = migraphx::gpu::hip_compile_key(ctx, src); + + EXPECT(base == migraphx::gpu::hip_compile_key(ctx, src)); + + auto changed_source = src; + changed_source.content = "// different source"; + EXPECT(migraphx::gpu::hip_compile_key(ctx, changed_source) != base); + + auto changed_name = src; + changed_name.options.kernel_name = "other_kernel"; + EXPECT(migraphx::gpu::hip_compile_key(ctx, changed_name) != base); + + auto changed_global = src; + changed_global.options.global = 2048; + EXPECT(migraphx::gpu::hip_compile_key(ctx, changed_global) != base); + + auto changed_local = src; + changed_local.options.local = 128; + EXPECT(migraphx::gpu::hip_compile_key(ctx, changed_local) != base); + + auto changed_params = src; + changed_params.options.emplace_param("-DEXTRA=1"); + EXPECT(migraphx::gpu::hip_compile_key(ctx, changed_params) != base); + + // Same source, but the generated tensor views differ. + auto changed_views = src; + changed_views.options.virtual_inputs = {{migraphx::shape::float_type, {32}}, + {migraphx::shape::float_type, {32}}}; + EXPECT(migraphx::gpu::hip_compile_key(ctx, changed_views) != base); +} + +TEST_CASE(version_dir_is_stable) +{ + migraphx::gpu::context ctx; + EXPECT(migraphx::gpu::binary_cache::version_dir(ctx) == + migraphx::gpu::binary_cache::version_dir(ctx)); + EXPECT(not migraphx::gpu::binary_cache::version_dir(ctx).empty()); +} + +int main(int argc, const char* argv[]) { test::run(argc, argv); } From 3242c998bf0320939e46d34faced8b1959e4a1a9 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 29 Jul 2026 12:08:20 -0500 Subject: [PATCH 02/49] Improve getting compiler version --- src/targets/gpu/binary_cache.cpp | 26 +++-- src/targets/gpu/compile_hip.cpp | 99 +++++++++++++------ .../gpu/include/migraphx/gpu/binary_cache.hpp | 8 +- .../gpu/include/migraphx/gpu/compile_hip.hpp | 22 ++++- test/gpu/binary_cache.cpp | 7 +- 5 files changed, 108 insertions(+), 54 deletions(-) diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index 7c0bf4b21ee..25f8bbfbe73 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -106,13 +106,19 @@ static const std::string& kernels_digest() return digest; } -std::string binary_cache::version_dir(const context& ctx) +const std::string& binary_cache::version_dir() { - auto toolchain = hip_compiler_version(ctx.get_current_device().get_device_name()); - if(toolchain.empty()) - return {}; - return std::string{binary_cache_format} + "-hip" + toolchain + "-kernels" + kernels_digest() + - "-rocmlir" + rocmlir_id; + static const std::string dir = [] { + const auto& compiler = hip_compiler_version(); + if(compiler.empty()) + return std::string{}; + // The version numbers make the directory readable; the hash of the full version string + // separates builds that share them, since it also covers the source revision. + return std::string{binary_cache_format} + "-hip" + compiler.major + "." + compiler.minor + + "." + md5(compiler.version).substr(0, 12) + "-kernels" + kernels_digest() + + "-rocmlir" + rocmlir_id; + }(); + return dir; } /// Entries are grouped by the device they were compiled for. The architecture is part of this @@ -130,21 +136,21 @@ static std::string device_dir(const context& ctx) /// from different toolchains would be indistinguishable. static fs::path entry_path(const fs::path& root, const context& ctx, const std::string& key) { - auto version = binary_cache::version_dir(ctx); + const auto& version = binary_cache::version_dir(); if(version.empty()) return {}; return root / version / device_dir(ctx) / (md5(key) + ".mxr"); } /// Record what this build is, so a directory full of hashes can be identified later. -static void write_stamp(const context& ctx, const fs::path& dir) +static void write_stamp(const fs::path& dir) { auto stamp = dir / "cache.info"; if(fs::exists(stamp)) return; std::stringstream ss; ss << "format: " << binary_cache_format << "\n"; - ss << "hip: " << hip_compiler_version(ctx.get_current_device().get_device_name()) << "\n"; + ss << "hip: " << hip_compiler_version().version << "\n"; ss << "kernels: " << kernels_digest() << "\n"; ss << "rocmlir: " << rocmlir_id << "\n"; write_string(stamp, ss.str()); @@ -223,7 +229,7 @@ void binary_cache::insert(const context& ctx, const binary_cache_entry& entry) try { fs::create_directories(path.parent_path()); - write_stamp(ctx, path.parent_path().parent_path()); + write_stamp(path.parent_path().parent_path()); // Publish by rename so a reader never sees a half-written entry. The content is decided // entirely by the key, so a writer that loses the race replaces the file with the same // bytes and no locking is needed. diff --git a/src/targets/gpu/compile_hip.cpp b/src/targets/gpu/compile_hip.cpp index 7ceaba9597f..7a140ed6246 100644 --- a/src/targets/gpu/compile_hip.cpp +++ b/src/targets/gpu/compile_hip.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #ifdef MIGRAPHX_USE_HIPRTC @@ -373,42 +374,78 @@ std::vector> compile_hip_src(const std::vector& srcs #endif // MIGRAPHX_USE_HIPRTC -// A kernel with no dependencies, compiled only so the result can be identified. It has to stay -// fixed, since changing it changes the identity of every toolchain. -static const char* const toolchain_probe = R"__migraphx__( -extern "C" __global__ void migraphx_toolchain_probe(float* p) +// Markers around the version string, chosen so it can be found in the compiled output without +// having to understand the object format. +static constexpr std::string_view version_prefix = "<<>>"; + +extern "C" __global__ void migraphx_version_probe(char* p) { - p[0] = p[0] * 2.0f + 1.0f; + p[0] = migraphx_compiler_version[0]; } )__migraphx__"; -std::string hip_compiler_version(const std::string& arch) +static hip_compiler_info parse_version(const std::vector& obj) { - static std::mutex mutex; - static std::unordered_map ids; - const std::lock_guard lock(mutex); - auto it = ids.find(arch); - if(it != ids.end()) - return it->second; - - std::string id; - try - { - auto cos = compile_hip_src({src_file{"main.cpp", toolchain_probe}}, - {"-std=c++17"}, - arch, - /* quiet */ true); - if(not cos.empty()) - id = md5(std::string(cos.front().begin(), cos.front().end())).substr(0, 12); - } - catch(const std::exception& e) - { - // Callers treat an empty id as "cannot tell toolchains apart" and stop sharing results - // between runs, which is slower but never wrong. - log::warn() << "Cannot identify the hip toolchain: " << e.what(); - } - ids[arch] = id; - return id; + auto begin = std::search(obj.begin(), obj.end(), version_prefix.begin(), version_prefix.end()); + if(begin == obj.end()) + return {}; + begin += version_prefix.size(); + auto end = std::search(begin, obj.end(), version_suffix.begin(), version_suffix.end()); + if(end == obj.end()) + return {}; + + auto fields = split_string(std::string{begin, end}, '|'); + if(fields.size() != 3) + return {}; + return {fields[0], fields[1], fields[2]}; +} + +const hip_compiler_info& hip_compiler_version() +{ + static const hip_compiler_info info = [] { + try + { + auto cos = compile_hip_src({src_file{"main.cpp", version_probe}}, + {"-std=c++17"}, + version_probe_arch, + /* quiet */ true); + if(not cos.empty()) + return parse_version(cos.front()); + } + catch(const std::exception& e) + { + // Callers treat an empty version as "cannot tell compilers apart" and stop sharing + // results between runs, which is slower but never wrong. + log::warn() << "Cannot identify the hip compiler: " << e.what(); + } + return hip_compiler_info{}; + }(); + return info; } bool hip_can_compile(const std::string& src, const std::vector& flags) diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index e9e63fda7bd..54317b70647 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -115,10 +115,10 @@ struct MIGRAPHX_GPU_EXPORT binary_cache bool verify(); /// Names the directory holding entries that this build can use: the entry format, the - /// toolchain, a digest of the embedded kernel headers, and the rocMLIR build. Empty when - /// the toolchain cannot be identified, in which case nothing is read from or written to - /// disk, since entries from different toolchains could not be told apart. - static std::string version_dir(const context& ctx); + /// compiler, a digest of the embedded kernel headers, and the rocMLIR build. Empty when the + /// compiler cannot be identified, in which case nothing is read from or written to disk, + /// since entries from different compilers could not be told apart. + static const std::string& version_dir(); private: std::mutex mutex; diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp index 27b44f1a482..3a3b5ce9c6e 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp @@ -54,14 +54,26 @@ struct hiprtc_src_file } }; +/// What the compiler that turns kernel source into code objects reports about itself. +struct hip_compiler_info +{ + /// Version numbers from __clang_major__ and __clang_minor__, short enough to read. + std::string major = {}; + std::string minor = {}; + /// The full __clang_version__ string, which also names the source revision. + std::string version = {}; + + bool empty() const { return version.empty(); } +}; + /** - * Identifies the toolchain that turns kernel source into code objects for an architecture. + * Ask the device compiler what version it is. * - * The toolchain is loaded at runtime, and what it reports about itself is coarser than what it - * generates, so this compiles a fixed kernel and identifies the result. Returns an empty string - * if the toolchain cannot be identified. + * It is not the compiler this library was built with, so the answer comes from compiling a + * probe and reading back what it recorded. The result is determined once and reused; it is + * empty if the compiler could not be asked. */ -MIGRAPHX_GPU_EXPORT std::string hip_compiler_version(const std::string& arch); +MIGRAPHX_GPU_EXPORT const hip_compiler_info& hip_compiler_version(); MIGRAPHX_GPU_EXPORT bool hip_can_compile(const std::string& src, const std::vector& flags); diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 2b92ee9755b..b1de64bc62b 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -280,10 +280,9 @@ TEST_CASE(key_covers_more_than_the_source) TEST_CASE(version_dir_is_stable) { - migraphx::gpu::context ctx; - EXPECT(migraphx::gpu::binary_cache::version_dir(ctx) == - migraphx::gpu::binary_cache::version_dir(ctx)); - EXPECT(not migraphx::gpu::binary_cache::version_dir(ctx).empty()); + EXPECT(migraphx::gpu::binary_cache::version_dir() == + migraphx::gpu::binary_cache::version_dir()); + EXPECT(not migraphx::gpu::binary_cache::version_dir().empty()); } int main(int argc, const char* argv[]) { test::run(argc, argv); } From 2fdaaec7397d6ce90f06db73b1d7b5ab14850395 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 29 Jul 2026 14:20:12 -0500 Subject: [PATCH 03/49] Simplify --- docs/dev/triage-migraphx.rst | 26 +- src/include/migraphx/tmp_dir.hpp | 5 +- src/targets/gpu/binary_cache.cpp | 159 ++++++------ src/targets/gpu/compile_hip.cpp | 4 +- src/targets/gpu/compile_hip_code_object.cpp | 10 +- src/targets/gpu/compile_ops.cpp | 211 ++++++++-------- src/targets/gpu/compiled_code.cpp | 7 +- .../gpu/include/migraphx/gpu/binary_cache.hpp | 128 +++++----- .../gpu/include/migraphx/gpu/compile_hip.hpp | 2 +- .../gpu/include/migraphx/gpu/compile_ops.hpp | 2 +- .../gpu/include/migraphx/gpu/compiler.hpp | 12 +- .../gpu/include/migraphx/gpu/context.hpp | 6 +- src/targets/gpu/include/migraphx/gpu/mlir.hpp | 3 +- src/targets/gpu/jit/mlir.cpp | 6 +- src/targets/gpu/target.cpp | 2 +- src/tmp_dir.cpp | 2 +- test/gpu/binary_cache.cpp | 230 ++++++++++-------- 17 files changed, 441 insertions(+), 374 deletions(-) diff --git a/docs/dev/triage-migraphx.rst b/docs/dev/triage-migraphx.rst index 78be28bdb67..d10ee0c8893 100644 --- a/docs/dev/triage-migraphx.rst +++ b/docs/dev/triage-migraphx.rst @@ -151,26 +151,29 @@ Debugging and Tracing - ``MIGRAPHX_TRACE_BENCHMARKING=3`` # Kernel benchmarking process +This systematic approach helps maintainers quickly understand and fix root causes. + Binary Cache ============ -Compiled kernels are always shared within a process, so a kernel that appears many times in a -model is only compiled once. Setting a directory makes that reuse survive across runs: +Compiled kernels are shared within a single compile, so a kernel that appears many times in a +model is compiled only once. Setting a directory makes that reuse survive across runs: .. code-block:: bash export MIGRAPHX_BINARY_CACHE=$HOME/.cache/migraphx -Entries are grouped by a directory naming the entry format, the HIP toolchain, a digest of the +Entries are grouped by a directory naming the entry format, the HIP compiler, a digest of the embedded kernel headers, and the rocMLIR build, so entries a build cannot use are never -consulted. The toolchain is identified by compiling a fixed kernel and hashing the result, -because it is loaded at runtime and what it reports about itself is coarser than what it -generates. Reclaim space by deleting the directories for builds you no longer use: +consulted. The compiler is identified by compiling a small probe that records +``__clang_version__`` into the object and reading it back, because the device compiler is loaded +at runtime and need not be the one MIGraphX was built with. Reclaim space by deleting the +directories for builds you no longer use: .. code-block:: bash ls $HOME/.cache/migraphx # each directory has a cache.info describing the build - rm -r $HOME/.cache/migraphx/v1-hipe0f11f454e40-* + rm -r $HOME/.cache/migraphx/v1-hip22.0.* The same settings are available as backend options, which take precedence over the environment and are how tests configure the cache: @@ -178,12 +181,11 @@ and are how tests configure the cache: .. code-block:: python model.compile(migraphx.get_target("gpu"), - binary_cache="/tmp/cache", binary_cache_verify=True) + advance_backend_options={"binary_cache": "/tmp/cache", + "binary_cache_verify": True}) -Entries are keyed on the exact source handed to the backend compiler. If a kernel is suspected -of being reused when it should not be, set ``binary_cache_verify``. That compiles even when a +Entries are keyed on everything handed to the backend compiler. If a kernel is suspected of +being reused when it should not be, set ``binary_cache_verify``. That compiles even when a result could be reused and fails loudly if the two disagree, which is the only way an incomplete key shows up other than as wrong results. It is slower than compiling without a cache at all, so use it to diagnose rather than routinely. - -This systematic approach helps maintainers quickly understand and fix root causes. \ No newline at end of file diff --git a/src/include/migraphx/tmp_dir.hpp b/src/include/migraphx/tmp_dir.hpp index f047a615cdd..f5c06b95f24 100644 --- a/src/include/migraphx/tmp_dir.hpp +++ b/src/include/migraphx/tmp_dir.hpp @@ -33,6 +33,9 @@ namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { +/// A name unlikely to collide with one made by another process or thread. +MIGRAPHX_EXPORT std::string unique_string(const std::string& prefix); + struct MIGRAPHX_EXPORT tmp_dir { fs::path path; @@ -45,7 +48,7 @@ struct MIGRAPHX_EXPORT tmp_dir execute(std::string_view{cmd.string()}, args); } - tmp_dir(tmp_dir const&) = delete; + tmp_dir(tmp_dir const&) = delete; tmp_dir& operator=(tmp_dir const&) = delete; ~tmp_dir(); diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index 25f8bbfbe73..1616d0de6da 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -33,21 +33,9 @@ #include #include #include +#include #include #include -#include - -#ifdef _WIN32 -// cppcheck-suppress definePrefix -#define WIN32_LEAN_AND_MEAN -#include -#undef getpid -// cppcheck-suppress [definePrefix, defineUpperCase] -#define getpid _getpid -#else -#include -#include -#endif namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { @@ -66,22 +54,73 @@ static constexpr const char* rocmlir_id = MIGRAPHX_ROCMLIR_ID; static constexpr const char* rocmlir_id = "nomlir"; #endif +std::shared_ptr make_binary_cache() { return std::make_shared(); } + binary_cache_settings binary_cache_settings::defaults() { static const std::string path = string_value_of(MIGRAPHX_BINARY_CACHE{}); return {path, false}; } -void binary_cache::configure(const binary_cache_settings& s) +binary_cache::stats::counts binary_cache::stats::get() { const std::lock_guard lock(mutex); - settings = s; + return data; } -bool binary_cache::enabled() +void binary_cache::stats::reused() { const std::lock_guard lock(mutex); - return not settings.path.empty(); + data.reused++; +} + +void binary_cache::stats::hit() +{ + const std::lock_guard lock(mutex); + data.hits++; +} + +void binary_cache::stats::miss() +{ + const std::lock_guard lock(mutex); + data.misses++; +} + +void binary_cache::stats::compiled() +{ + const std::lock_guard lock(mutex); + data.compiled++; +} + +void binary_cache::record_reused() +{ + if(st) + st->reused(); +} + +void binary_cache::record_hit() +{ + if(st) + st->hit(); +} + +optional binary_cache::record_miss() +{ + if(st) + st->miss(); + return nullopt; +} + +void binary_cache::record_compiled() +{ + if(st) + st->compiled(); +} + +void binary_cache::configure(const binary_cache_settings& s) +{ + const std::lock_guard lock(mutex); + settings = s; } bool binary_cache::verify() @@ -124,7 +163,7 @@ const std::string& binary_cache::version_dir() /// Entries are grouped by the device they were compiled for. The architecture is part of this /// because it is passed to the compiler rather than appearing in the source; the core count and /// wavefront size are here to keep the directory self-describing, since they already reach the -/// key through the launch bounds baked into the source. +/// key through the launch bounds and -D defines in the parameter list. static std::string device_dir(const context& ctx) { const auto& device = ctx.get_current_device(); @@ -166,104 +205,80 @@ optional binary_cache::get(const context& ctx, const std::string& auto it = memo.find(key); if(it != memo.end()) { - stats.hits++; + record_reused(); return it->second; } root = settings.path; } if(root.empty()) - return nullopt; + return record_miss(); auto path = entry_path(root, ctx, key); if(path.empty() or not fs::exists(path)) - { - const std::lock_guard lock(mutex); - stats.misses++; - return nullopt; - } + return record_miss(); - binary_cache_entry entry; + entry e; try { - migraphx::from_value(from_msgpack(read_buffer(path)), entry); + migraphx::from_value(from_msgpack(read_buffer(path)), e); } - catch(const std::exception& e) + catch(const std::exception& ex) { // A damaged or stale entry is only worth a recompile, so treat every failure as a miss // and let the result be written over the top. - log::warn() << "Ignoring unreadable binary cache entry " << path << ": " << e.what(); - const std::lock_guard lock(mutex); - stats.misses++; - return nullopt; + log::warn() << "Ignoring unreadable binary cache entry " << path << ": " << ex.what(); + return record_miss(); } - if(entry.key != key) + if(e.key != key) { log::warn() << "Ignoring binary cache entry with mismatched key: " << path; - const std::lock_guard lock(mutex); - stats.misses++; - return nullopt; + return record_miss(); } + record_hit(); const std::lock_guard lock(mutex); - memo[key] = entry.code; - stats.hits++; - return entry.code; + memo[key] = std::move(e.code); + return memo[key]; } -void binary_cache::insert(const context& ctx, const binary_cache_entry& entry) +void binary_cache::insert(const context& ctx, const entry& e) { - if(entry.key.empty()) + if(e.key.empty()) return; + record_compiled(); fs::path root; { const std::lock_guard lock(mutex); - memo[entry.key] = entry.code; - root = settings.path; + memo[e.key] = e.code; + root = settings.path; } if(root.empty()) return; - auto path = entry_path(root, ctx, entry.key); + auto path = entry_path(root, ctx, e.key); if(path.empty()) return; + // Publish by rename so a reader never sees a half-written entry. The content is decided + // entirely by the key, so a writer that loses the race replaces the file with the same bytes + // and no locking is needed. + auto tmp = path; + tmp += "." + unique_string("tmp"); try { fs::create_directories(path.parent_path()); write_stamp(path.parent_path().parent_path()); - // Publish by rename so a reader never sees a half-written entry. The content is decided - // entirely by the key, so a writer that loses the race replaces the file with the same - // bytes and no locking is needed. - std::stringstream suffix; - suffix << "." << getpid() << "." << std::this_thread::get_id() << ".tmp"; - auto tmp = path; - tmp += suffix.str(); - write_buffer(tmp, to_msgpack(migraphx::to_value(entry))); + write_buffer(tmp, to_msgpack(migraphx::to_value(e))); fs::rename(tmp, path); } - catch(const std::exception& e) + catch(const std::exception& ex) { - log::warn() << "Failed to write binary cache entry " << path << ": " << e.what(); + // Leaving the temporary behind would accumulate in the cache directory. + std::error_code ec; + fs::remove(tmp, ec); + log::warn() << "Failed to write binary cache entry " << path << ": " << ex.what(); } } -void binary_cache::record_compiled(std::size_t n) -{ - const std::lock_guard lock(mutex); - stats.compiled += n; -} - -void binary_cache::record_reused(std::size_t n) -{ - const std::lock_guard lock(mutex); - stats.reused += n; -} - -binary_cache_stats binary_cache::get_stats() -{ - const std::lock_guard lock(mutex); - return stats; -} - } // namespace gpu } // namespace MIGRAPHX_INLINE_NS } // namespace migraphx diff --git a/src/targets/gpu/compile_hip.cpp b/src/targets/gpu/compile_hip.cpp index 7a140ed6246..a2f470467f8 100644 --- a/src/targets/gpu/compile_hip.cpp +++ b/src/targets/gpu/compile_hip.cpp @@ -383,8 +383,8 @@ static constexpr std::string_view version_suffix = ">>>"; // accepts. Keeping it fixed means the answer does not depend on the device present. static const char* const version_probe_arch = "gfx950"; -// Has the compiler record its own version, since the compiler that generates device code is not -// the one this was built with and is only known once it runs. +// Probe that makes the device compiler embed its own version string in the object. It is not +// the compiler this library was built with, so its version is only known at runtime. static const char* const version_probe = R"__migraphx__( #define MIGRAPHX_STRINGIZE_(x) #x #define MIGRAPHX_STRINGIZE(x) MIGRAPHX_STRINGIZE_(x) diff --git a/src/targets/gpu/compile_hip_code_object.cpp b/src/targets/gpu/compile_hip_code_object.cpp index a7c2c973d7d..9cc4982f465 100644 --- a/src/targets/gpu/compile_hip_code_object.cpp +++ b/src/targets/gpu/compile_hip_code_object.cpp @@ -202,11 +202,12 @@ std::size_t compute_block_size(const context& ctx, std::size_t n, std::size_t ma return std::min(std::max(min_block_size, block_size), max_block_size); } -// Append the parameters that are derived from the options and the device rather than passed in -// by the compiler. Shared with hip_compile_key so the key sees the same final parameter list -// that the compiler does. +// Append the parameters derived from the options and the device rather than set by the caller. +// Shared with hip_compile_key so the key sees the same final parameter list clang does. static void add_derived_params(const context& ctx, hip_compile_options& options) { + assert(options.global > 0); + assert(options.local > 0); if(options.global % options.local != 0 and hip_accept_non_uniform_wg()) options.emplace_param("-fno-offload-uniform-block"); else @@ -247,6 +248,9 @@ compile_hip_raw(context& ctx, const std::string& content, hip_compile_options op std::string hip_compile_key(const context& ctx, const hip_src& src) { + assert(not src.options.inputs.empty()); + assert(src.options.inputs.size() == src.options.virtual_inputs.size() or + src.options.virtual_inputs.empty()); auto options = src.options; add_derived_params(ctx, options); diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 8a20c18819e..6a1aa3a02b4 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -58,17 +59,6 @@ MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_BENCHMARKING); MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_SKIP_BENCHMARKING); MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR); -// Inner repeat count when timing a candidate, raised for split-k (kernel + prefill). -static std::size_t compute_benchmark_bundle(const module& m) -{ - // Count context-requiring ops (kernel + prefills); skip context-free and @-builtins. - int n = std::count_if(m.begin(), m.end(), [](const auto& ins) { - return not migraphx::is_context_free(ins.get_operator()) and - not starts_with(ins.name(), "@"); - }); - return std::max(1, 4 * n - 2); -} - struct precompile_op { operation op = op::identity{}; @@ -115,15 +105,15 @@ MIGRAPHX_REGISTER_OP(precompile_op); static program make_fragment(const compiler_replace& cr, instruction_ref ins) { program frag; - auto* fm = frag.get_main_module(); + auto* fm = frag.get_main_module(); + const auto& args = ins->inputs(); std::vector inputs; - std::transform(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(inputs), - [&](const auto& arg) { - return fm->add_parameter(compiled_code::input_name(inputs.size()), - arg->get_shape()); - }); + // Named by position rather than by a counter the transform mutates, since compiled_code + // maps each parameter back onto the input in the same position. + auto idx = range(args.size()); + std::transform(idx.begin(), idx.end(), std::back_inserter(inputs), [&](auto i) { + return fm->add_parameter(compiled_code::input_name(i), args[i]->get_shape()); + }); auto frag_ins = fm->add_instruction(ins->get_operator(), inputs, ins->module_inputs()); fm->add_return({frag_ins}); cr.replace(*fm, frag_ins); @@ -152,6 +142,66 @@ static std::string compile_key_or_empty(context& ctx, } } +/// Compile, and normalize the result so a fragment is always what gets replaced. Reusing a +/// result then takes the same path as compiling one and cannot drift from it. +static compiler_replace +compile_fragment(context& ctx, instruction_ref ins, const operation& preop, const value& solution) +{ + auto cr = compile(ctx, ins, preop, solution); + if(cr.code.empty()) + cr.code.fragment = make_fragment(cr, ins); + return cr; +} + +/** + * Compile anyway and check the reused result matches. + * + * A key that fails to capture something the compiler depends on does not cause a miss, it hands + * back the wrong code, so there is nothing to notice at runtime beyond bad numbers. + */ +static void verify_reuse(context& ctx, + instruction_ref ins, + const operation& preop, + const value& solution, + const compiled_code& reused) +{ + auto fresh = compile_fragment(ctx, ins, preop, solution); + if(fresh.code.fragment == reused.fragment) + return; + MIGRAPHX_THROW("Binary cache reused a result that does not match a fresh compile of " + + preop.name() + ".\nReused:\n" + to_string(reused.fragment) + "\nCompiled:\n" + + to_string(fresh.code.fragment)); +} + +/// Reuse an earlier result for this key, or compile and record one. Both the compile pass and +/// the dynamic shapes compiled during evaluation go through here, so they cache alike. +static compiler_replace compile_cached(context& ctx, + instruction_ref ins, + const operation& preop, + const value& solution, + const std::string& key, + const value& problem) +{ + auto& cache = ctx.get_binary_cache(); + if(auto cached = cache.get(ctx, key)) + { + if(cache.verify()) + verify_reuse(ctx, ins, preop, solution, *cached); + compiler_replace cr; + cr.code = std::move(*cached); + return cr; + } + auto cr = compile_fragment(ctx, ins, preop, solution); + binary_cache::entry entry; + entry.key = key; + entry.op_name = preop.name(); + entry.problem = problem; + entry.solution = solution; + entry.code = cr.code; + cache.insert(ctx, entry); + return cr; +} + struct dynamic_op_cache { module mod; @@ -281,29 +331,10 @@ struct dynamic_code_object_op { solution = config->solutions.front(); } - // This runs while the program is being evaluated, so reusing an earlier result matters - // more here than anywhere else. - auto key = compile_key_or_empty(ctx, ins, preop, solution); - auto& bin_cache = ctx.get_binary_cache(); - compiler_replace compiled_op; - if(auto cached = bin_cache.get(ctx, key)) - { - compiled_op.code = *cached; - } - else - { - compiled_op = compile(ctx, ins, preop, solution); - if(compiled_op.code.empty()) - compiled_op.code.fragment = make_fragment(compiled_op, ins); - bin_cache.record_compiled(1); - binary_cache_entry entry; - entry.key = key; - entry.op_name = preop.name(); - entry.problem = config ? config->problem : value{}; - entry.solution = solution; - entry.code = compiled_op.code; - bin_cache.insert(ctx, entry); - } + // Dynamic shapes compile during eval, so reusing a result here avoids stalling the run. + auto key = compile_key_or_empty(ctx, ins, preop, solution); + auto compiled_op = + compile_cached(ctx, ins, preop, solution, key, config ? config->problem : value{}); compiled_op.replace(runtime_mod, ins); run_passes(runtime_mod, {dead_code_elimination{}}); @@ -399,52 +430,23 @@ struct compile_plan return compile_key_or_empty(*ctx, ins, preop, solution); } - /** - * Compile anyway and check the reused result matches. - * - * A key that fails to capture something the compiler depends on does not cause a miss, it - * hands back the wrong code, so there is nothing to notice at runtime beyond bad numbers. - * Running with this on turns that into a loud failure. - */ - void verify_reuse(const value& solution, const compiled_code& reused) const + /// Inner repeat count when timing a candidate, raised for split-k (kernel + prefill). + static std::size_t benchmark_bundle(const module& m) { - auto fresh = compile(*ctx, ins, preop, solution); - if(fresh.code.empty()) - fresh.code.fragment = make_fragment(fresh, ins); - if(fresh.code.fragment == reused.fragment) - return; - MIGRAPHX_THROW("Binary cache reused a result that does not match a fresh compile of " + - preop.name() + ".\nReused:\n" + to_string(reused.fragment) + - "\nCompiled:\n" + to_string(fresh.code.fragment)); + // Count context-requiring ops (kernel + prefills); skip context-free and @-builtins. + int n = std::count_if(m.begin(), m.end(), [](const auto& ins2) { + return not migraphx::is_context_free(ins2.get_operator()) and + not starts_with(ins2.name(), "@"); + }); + return std::max(1, 4 * n - 2); } optional run_compile(const value& solution, const std::string& key) const { - auto& cache = ctx->get_binary_cache(); - if(auto cached = cache.get(*ctx, key)) - { - compiler_replace cr; - cr.code = *cached; - if(cache.verify()) - verify_reuse(solution, *cached); - return cr; - } try { - auto cr = compile(*ctx, ins, preop, solution); - // Always replace through the fragment, so the path taken when the result is reused - // is the only path and cannot drift from the one taken when it is compiled. - if(cr.code.empty()) - cr.code.fragment = make_fragment(cr, ins); - cache.record_compiled(1); - binary_cache_entry entry; - entry.key = key; - entry.op_name = preop.name(); - entry.problem = config ? config->problem : value{}; - entry.solution = solution; - entry.code = cr.code; - cache.insert(*ctx, entry); - return cr; + return compile_cached( + *ctx, ins, preop, solution, key, config ? config->problem : value{}); } catch(const std::exception& e) { @@ -591,7 +593,7 @@ struct compile_plan auto bench_prog = cr->make_program(); if(trace_level > 2) std::cout << bench_prog << std::endl; - auto bundle = compute_benchmark_bundle(*bench_prog.get_main_module()); + auto bundle = benchmark_bundle(*bench_prog.get_main_module()); auto t = time_program(*ctx, std::move(bench_prog), cr->replace.code.fill_map, @@ -672,13 +674,14 @@ static void par_compile(std::size_t n, F f) /// A compile that one or more plans are waiting on, along with the places its result goes. struct compile_task { - /// The plan whose instruction drives the compile. Any plan in targets would do, since they - /// all produce the same code. - std::size_t plan_index = 0; - value solution = {}; - std::string key = {}; + value solution = {}; + std::string key = {}; + /// Where the result goes, as (plan, result) slots. The first one also drives the compile, + /// since every target shares this key and therefore compiles to the same code. std::vector> targets = {}; optional result = nullopt; + + std::size_t plan_index() const { return targets.front().first; } }; struct compile_manager @@ -703,22 +706,24 @@ struct compile_manager * Since every key is known before any compile starts, each group can be handed to exactly * one task and the results shared afterwards, so the compiles never have to coordinate. */ - std::vector make_tasks(const std::vector& candidates) const + static std::vector make_tasks(std::vector& candidates) { std::vector tasks; - std::unordered_map task_index; - for(const auto& c : candidates) + // The keys are whole kernel sources, so the index views them rather than copying them; + // candidates outlives the tasks built here. + std::unordered_map task_index; + for(auto& c : candidates) { std::size_t i = tasks.size(); if(c.key.empty()) { - tasks.push_back({c.plan_index, c.solution, c.key}); + tasks.push_back({c.solution, c.key}); } else { auto [it, inserted] = task_index.emplace(c.key, i); if(inserted) - tasks.push_back({c.plan_index, c.solution, c.key}); + tasks.push_back({c.solution, c.key}); i = it->second; } tasks[i].targets.emplace_back(c.plan_index, c.result_index); @@ -738,26 +743,24 @@ struct compile_manager }); auto tasks = make_tasks(candidates); - if(not cps.empty()) - { - cps.front().ctx->get_binary_cache().record_reused(candidates.size() - tasks.size()); - } + assert(tasks.size() <= candidates.size()); par_compile(tasks.size(), [&](auto i) { - tasks[i].result = cps[tasks[i].plan_index].run_compile(tasks[i].solution, tasks[i].key); + tasks[i].result = + cps[tasks[i].plan_index()].run_compile(tasks[i].solution, tasks[i].key); }); for(const auto& task : tasks) { + // The slots start out empty, so a task that failed to compile needs no write. + if(not task.result.has_value()) + continue; + assert(not task.result->code.empty()); for(auto [pi, ri] : task.targets) { - if(not task.result.has_value()) - { - cps[pi].results[ri] = nullopt; - continue; - } auto cr = *task.result; - // The fragment is what gets used from here on. The replace function is bound to - // the instruction that produced it, so it must not run against another one. + // Only the fragment is used from here on. Dropping the replace function releases + // the code objects and split modules its closure holds, and keeps it from being + // run against an instruction other than the one it was built for. cr.replace_fn = nullptr; cps[pi].results[ri] = compiled_result{std::move(cr), cps[pi].ins}; } diff --git a/src/targets/gpu/compiled_code.cpp b/src/targets/gpu/compiled_code.cpp index b5b8609c93a..3d869e113f9 100644 --- a/src/targets/gpu/compiled_code.cpp +++ b/src/targets/gpu/compiled_code.cpp @@ -37,12 +37,17 @@ void compiled_code::replace(module& m, instruction_ref ins) const { const auto* frag = fragment.get_main_module(); const auto& inputs = ins->inputs(); + assert(frag->get_parameter_names().size() == inputs.size()); // Seeding the map with the parameters keeps insert_instructions from copying them, so the // fragment is wired directly onto the inputs of ins. std::unordered_map map_ins; for(auto i : range(inputs.size())) { - map_ins[frag->get_parameter(input_name(i))] = inputs[i]; + auto param = frag->get_parameter(input_name(i)); + // get_parameter returns end() for an unknown name, which would splice a stray parameter + // into m instead of wiring the input. + assert(param != frag->end()); + map_ins[param] = inputs[i]; } auto returns = m.insert_instructions(ins, frag, &map_ins); if(returns.size() != 1) diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index 54317b70647..de659ec6afe 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -25,10 +25,12 @@ #define MIGRAPHX_GUARD_GPU_BINARY_CACHE_HPP #include +#include #include #include #include #include +#include #include #include #include @@ -39,78 +41,78 @@ namespace gpu { struct context; -/// What gets written to disk for one compiled kernel. The problem and solution are recorded so -/// an entry can be identified without recompiling anything. -struct binary_cache_entry -{ - std::string key = {}; - std::string op_name = {}; - value problem = {}; - value solution = {}; - compiled_code code = {}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.key, "key"), - f(self.op_name, "op_name"), - f(self.problem, "problem"), - f(self.solution, "solution"), - f(self.code, "code")); - } -}; - -/// Counts of what the cache did, for tests and for reporting. -struct binary_cache_stats -{ - std::size_t compiled = 0; - std::size_t reused = 0; - std::size_t hits = 0; - std::size_t misses = 0; -}; - -/// How the cache behaves for one compile. Supplied through the target's backend options, with -/// the MIGRAPHX_BINARY_CACHE environment variable as the default for the directory. -struct binary_cache_settings -{ - /// Where entries are kept. Nothing is written to disk when this is empty, though results - /// are still shared in memory. - std::string path = {}; - /// Compile even when a result can be reused, and fail if the two disagree. - bool verify = false; - - MIGRAPHX_GPU_EXPORT static binary_cache_settings defaults(); -}; - /** * Compiled kernels, keyed by a string describing what the compiler was given. * * Results are held in memory for the life of the context and, when a cache directory is - * configured, written to disk so later runs can reuse them. Entries are keyed on the exact - * source handed to the backend, so anything that would change the generated code changes the - * key; things outside the source, such as the toolchain and the embedded kernel headers, are - * separated by the directory the entries live in. + * configured, written to disk so later runs can reuse them. Things outside the key, such as the + * compiler and the embedded kernel headers, are separated by the directory the entries live in. */ struct MIGRAPHX_GPU_EXPORT binary_cache { + /// What gets written to disk for one compiled kernel. The op name, problem and solution are + /// stored for offline inspection; only the key is checked when an entry is loaded. + struct entry + { + std::string key = {}; + std::string op_name = {}; + value problem = {}; + value solution = {}; + compiled_code code = {}; + + template + static auto reflect(Self& self, F f) + { + return pack(f(self.key, "key"), + f(self.op_name, "op_name"), + f(self.problem, "problem"), + f(self.solution, "solution"), + f(self.code, "code")); + } + }; + + /** + * Counts of what the cache did. + * + * Only the tests ask for these, so a cache built without one keeps no counters at all. It + * carries its own lock because one may be shared by several caches. + */ + struct stats + { + struct counts + { + /// Served from memory, so an earlier compile in this process was shared. + std::size_t reused = 0; + /// Served from the cache directory. + std::size_t hits = 0; + /// Not found, so the caller had to compile. + std::size_t misses = 0; + /// Written to the cache after being compiled. + std::size_t compiled = 0; + }; + + counts get(); + + void reused(); + void hit(); + void miss(); + void compiled(); + + private: + std::mutex mutex; + counts data; + }; + + explicit binary_cache(std::shared_ptr s = nullptr) : st(std::move(s)) {} + /// Look up a key, consulting memory first and then the cache directory. optional get(const context& ctx, const std::string& key); /// Record a compiled result under its key. - void insert(const context& ctx, const binary_cache_entry& entry); - - /// Note that a result was shared rather than compiled again. - void record_compiled(std::size_t n); - void record_reused(std::size_t n); - - binary_cache_stats get_stats(); + void insert(const context& ctx, const entry& e); void configure(const binary_cache_settings& s); - /// True when a cache directory is configured. Nothing is written to disk otherwise, but - /// results are still shared in memory. - bool enabled(); - /// True when reused results should be checked against a fresh compile. bool verify(); @@ -121,10 +123,18 @@ struct MIGRAPHX_GPU_EXPORT binary_cache static const std::string& version_dir(); private: + /// Each records one outcome, and does nothing when no stats were supplied. record_miss + /// returns the empty result its callers hand back, so the outcome is recorded exactly where + /// the lookup gives up. + void record_reused(); + void record_hit(); + optional record_miss(); + void record_compiled(); + std::mutex mutex; std::unordered_map memo; - binary_cache_stats stats; binary_cache_settings settings = binary_cache_settings::defaults(); + std::shared_ptr st = nullptr; }; } // namespace gpu diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp index 3a3b5ce9c6e..72b538f54c2 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp @@ -57,7 +57,7 @@ struct hiprtc_src_file /// What the compiler that turns kernel source into code objects reports about itself. struct hip_compiler_info { - /// Version numbers from __clang_major__ and __clang_minor__, short enough to read. + /// __clang_major__ and __clang_minor__, kept separate so the cache directory stays readable. std::string major = {}; std::string minor = {}; /// The full __clang_version__ string, which also names the source revision. diff --git a/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp b/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp index 979b182c6ea..491341c686b 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp @@ -25,7 +25,7 @@ #define MIGRAPHX_GUARD_GPU_COMPILE_OPS_HPP #include -#include +#include #include namespace migraphx { diff --git a/src/targets/gpu/include/migraphx/gpu/compiler.hpp b/src/targets/gpu/include/migraphx/gpu/compiler.hpp index d6b29774d55..c2eda1f43db 100644 --- a/src/targets/gpu/include/migraphx/gpu/compiler.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compiler.hpp @@ -150,10 +150,10 @@ get_tuning_config(context& ctx, instruction_ref ins, const operation& op, bool e /** * A string that uniquely identifies the code the compiler would produce, without producing it. * - * This is the text handed to the backend compiler, so it captures the generated source along - * with everything else that shapes the result, and no separate accounting of shapes, launch - * parameters, or device properties is needed. An empty string means the compiler cannot - * describe its output this way, and its result must not be cached. + * The key must cover everything that shapes the result: the generated source, the final + * parameter list, the launch bounds, the target architecture, and the shapes recorded on the + * code object. An empty string means the compiler cannot describe its output this way, and its + * result must not be cached. */ MIGRAPHX_GPU_EXPORT std::string compile_key(context& ctx, instruction_ref ins, const operation& op, const value& solution); @@ -197,8 +197,8 @@ struct compiler : auto_register_compiler return nullopt; } - /// Compilers opt in to caching by overriding this. Until one does, an empty key keeps its - /// results out of the cache, which is always correct if never fast. + /// Compilers opt in to caching by overriding this; the default empty key keeps their + /// results out of the cache. std::string compile_key(context&, instruction_ref, const operation&, const value&) const { return {}; diff --git a/src/targets/gpu/include/migraphx/gpu/context.hpp b/src/targets/gpu/include/migraphx/gpu/context.hpp index 1061a91b019..86e75dc1c47 100644 --- a/src/targets/gpu/include/migraphx/gpu/context.hpp +++ b/src/targets/gpu/include/migraphx/gpu/context.hpp @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include #include #include @@ -490,7 +490,7 @@ struct context } /// Compiled kernels shared across every module compiled with this context. - binary_cache& get_binary_cache() const { return *bc; } + binary_cache& get_binary_cache() { return *bc; } private: // TODO: Make this a vector to support multiple devices @@ -505,7 +505,7 @@ struct context shared begin_event = nullptr; shared finish_event = nullptr; std::shared_ptr pc = nullptr; - std::shared_ptr bc = std::make_shared(); + std::shared_ptr bc = make_binary_cache(); }; inline void migraphx_to_value(value& v, const context& ctx) { v = ctx.to_value(); } diff --git a/src/targets/gpu/include/migraphx/gpu/mlir.hpp b/src/targets/gpu/include/migraphx/gpu/mlir.hpp index 795f44e1ed0..89675064c60 100644 --- a/src/targets/gpu/include/migraphx/gpu/mlir.hpp +++ b/src/targets/gpu/include/migraphx/gpu/mlir.hpp @@ -64,8 +64,7 @@ MIGRAPHX_GPU_EXPORT mlir_code_object compile_mlir(const context& migraphx_ctx, * A string identifying the code object compile_mlir would produce for these arguments. * * This is the mlir module as parsed, before any pipeline runs, so it is far cheaper than - * compiling. Callers that already ran get_tuning_config_mlir have the same text available as - * tuning_config::detailed_problem_info and can use that instead of calling this. + * compiling. */ MIGRAPHX_GPU_EXPORT std::string mlir_compile_key(const context& migraphx_ctx, module m, diff --git a/src/targets/gpu/jit/mlir.cpp b/src/targets/gpu/jit/mlir.cpp index 8e043f7eb41..3151be44bfe 100644 --- a/src/targets/gpu/jit/mlir.cpp +++ b/src/targets/gpu/jit/mlir.cpp @@ -241,8 +241,10 @@ struct mlir_compiler : compiler const operation&, const value& solution) const { - // The split into a separate gemm and pointwise kernel is decided from the submodule, - // the context and the solution, so the key covers both shapes of the result. + // The split into separate gemm and pointwise kernels is decided from the submodule, the + // context and the solution, so a key built from the unsplit submodule already + // distinguishes the split and unsplit forms. + assert(not ins->module_inputs().empty()); auto* smod = ins->module_inputs().front(); return mlir_compile_key(ctx, *smod, to_shapes(ins->inputs()), solution); } diff --git a/src/targets/gpu/target.cpp b/src/targets/gpu/target.cpp index a23b81ae0b5..b4c89cec2c9 100644 --- a/src/targets/gpu/target.cpp +++ b/src/targets/gpu/target.cpp @@ -106,7 +106,7 @@ namespace { struct backend_options { std::vector mlss_use_specific_ops = {}; - /// Where compiled kernels are cached between runs. Defaults to the environment setting. + /// Where compiled kernels are cached between runs. Defaults to ``MIGRAPHX_BINARY_CACHE``. std::string binary_cache = binary_cache_settings::defaults().path; /// Compile even when a kernel could be reused, and fail if the two disagree. bool binary_cache_verify = false; diff --git a/src/tmp_dir.cpp b/src/tmp_dir.cpp index 2509646c59f..c8e9f6b0a44 100644 --- a/src/tmp_dir.cpp +++ b/src/tmp_dir.cpp @@ -66,7 +66,7 @@ static std::string random_string(std::string::size_type length) return str; } -static std::string unique_string(const std::string& prefix) +std::string unique_string(const std::string& prefix) { auto pid = getpid(); auto tid = std::this_thread::get_id(); diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index b1de64bc62b..fecc223d7cc 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -38,6 +38,7 @@ #include #include #include +#include static migraphx::program pointwise_program() { @@ -53,25 +54,6 @@ static migraphx::program pointwise_program() return p; } -// Independent branches of the same shape lower to separate instructions that cannot fuse into -// each other, but they all produce the same kernel, so only one of them should be compiled. -static migraphx::program repeated_reduce(std::size_t n) -{ - migraphx::program p; - auto* mm = p.get_main_module(); - migraphx::shape s{migraphx::shape::float_type, {4, 8}}; - std::vector branches; - for(std::size_t i = 0; i < n; i++) - { - auto x = mm->add_parameter("x" + std::to_string(i), s); - auto mul = mm->add_instruction(migraphx::make_op("mul"), x, x); - branches.push_back( - mm->add_instruction(migraphx::make_op("reduce_sum", {{"axes", {1}}}), mul)); - } - mm->add_return({mm->add_instruction(migraphx::make_op("concat", {{"axis", 1}}), branches)}); - return p; -} - static migraphx::compile_options cache_options(const migraphx::fs::path& dir, bool verify = false) { migraphx::compile_options options; @@ -80,54 +62,130 @@ static migraphx::compile_options cache_options(const migraphx::fs::path& dir, bo return options; } -static migraphx::gpu::binary_cache_stats -compile_and_get_stats(migraphx::program p, const migraphx::compile_options& options) +static migraphx::gpu::compiled_code make_code() { - p.compile(migraphx::make_target("gpu"), options); - return migraphx::any_cast(p.get_context()) - .get_binary_cache() - .get_stats(); + migraphx::gpu::compiled_code code; + auto* fm = code.fragment.get_main_module(); + auto x = fm->add_parameter(migraphx::gpu::compiled_code::input_name(0), + {migraphx::shape::float_type, {2}}); + fm->add_return({fm->add_instruction(migraphx::make_op("abs"), x)}); + code.fill_map["float_type{2}"] = 3.0; + return code; +} + +static migraphx::gpu::binary_cache::entry make_entry(const std::string& key) +{ + migraphx::gpu::binary_cache::entry e; + e.key = key; + e.op_name = "pointwise"; + e.solution = migraphx::value{{"algo", "block"}}; + e.code = make_code(); + return e; +} + +// A cache built without stats keeps no counters, so the tests that assert on counts supply one. +static std::shared_ptr make_stats() +{ + return std::make_shared(); } -TEST_CASE(identical_kernels_compile_once) +TEST_CASE(lookup_records_a_miss) +{ + migraphx::gpu::context ctx; + auto st = make_stats(); + migraphx::gpu::binary_cache cache{st}; + + EXPECT(not cache.get(ctx, "absent").has_value()); + EXPECT(st->get().misses == 1); + EXPECT(st->get().hits == 0); + EXPECT(st->get().reused == 0); +} + +// An entry served out of memory is one an earlier compile in this process already paid for. +TEST_CASE(memory_lookup_records_reuse) +{ + migraphx::gpu::context ctx; + auto st = make_stats(); + migraphx::gpu::binary_cache cache{st}; + + cache.insert(ctx, make_entry("a-key")); + EXPECT(st->get().compiled == 1); + + auto found = cache.get(ctx, "a-key"); + EXPECT(found.has_value()); + EXPECT(st->get().reused == 1); + EXPECT(st->get().misses == 0); +} + +// A second cache shares nothing in memory, so anything it finds came off disk. +TEST_CASE(disk_lookup_records_a_hit) { migraphx::tmp_dir td{"binary-cache"}; - const std::size_t n = 6; - auto stats = compile_and_get_stats(repeated_reduce(n), cache_options(td.path)); - EXPECT(stats.reused >= n - 1); - EXPECT(stats.compiled < n); + migraphx::gpu::context ctx; + migraphx::gpu::binary_cache_settings settings{td.path.string(), false}; + + migraphx::gpu::binary_cache writer; + writer.configure(settings); + writer.insert(ctx, make_entry("shared-key")); + + auto st = make_stats(); + migraphx::gpu::binary_cache reader{st}; + reader.configure(settings); + + auto found = reader.get(ctx, "shared-key"); + EXPECT(found.has_value()); + EXPECT(st->get().hits == 1); + EXPECT(st->get().misses == 0); + EXPECT(*found->fragment.get_main_module() == *make_code().fragment.get_main_module()); } -// The whole point of the cache: a program compiled once should not compile again. The second -// compile gets a fresh context, so an empty in-memory cache, and everything it reuses has to -// come back off disk. -TEST_CASE(second_compile_reads_from_disk) +// A damaged entry must cost a recompile and nothing more. +TEST_CASE(corrupt_entry_is_ignored) { migraphx::tmp_dir td{"binary-cache"}; - auto options = cache_options(td.path); + migraphx::gpu::context ctx; + migraphx::gpu::binary_cache_settings settings{td.path.string(), false}; + + migraphx::gpu::binary_cache writer; + writer.configure(settings); + writer.insert(ctx, make_entry("damaged")); + + std::size_t truncated = 0; + for(const auto& file : migraphx::fs::recursive_directory_iterator(td.path)) + { + if(file.path().extension() != ".mxr") + continue; + migraphx::write_buffer(file.path(), std::vector(8, 0)); + truncated++; + } + EXPECT(truncated > 0); - auto cold = compile_and_get_stats(pointwise_program(), options); - EXPECT(cold.compiled > 0); - EXPECT(cold.misses > 0); - EXPECT(cold.hits == 0); + auto st = make_stats(); + migraphx::gpu::binary_cache reader{st}; + reader.configure(settings); - auto warm = compile_and_get_stats(pointwise_program(), options); - EXPECT(warm.hits == cold.misses); - EXPECT(warm.compiled == 0); - EXPECT(warm.misses == 0); + EXPECT(not reader.get(ctx, "damaged").has_value()); + EXPECT(st->get().misses == 1); } -// Without a directory nothing is written, but results are still shared within the compile. -TEST_CASE(no_directory_keeps_results_in_memory) +// Without a directory nothing reaches disk, though results are still shared in memory. +TEST_CASE(no_directory_writes_nothing) { migraphx::tmp_dir td{"binary-cache"}; - auto stats = compile_and_get_stats(repeated_reduce(4), cache_options("")); - EXPECT(stats.reused > 0); + migraphx::gpu::context ctx; + auto st = make_stats(); + migraphx::gpu::binary_cache cache{st}; + cache.configure({"", false}); + + cache.insert(ctx, make_entry("in-memory-only")); + EXPECT(cache.get(ctx, "in-memory-only").has_value()); + EXPECT(st->get().reused == 1); EXPECT(migraphx::fs::is_empty(td.path)); } -// A result taken from the cache has to produce the same numbers as one that was just compiled. -TEST_CASE(cached_results_match_reference) +// Compiling twice against the same directory has to leave entries behind and keep producing the +// same numbers as the reference, whichever half of the run they came from. +TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) { migraphx::tmp_dir td{"binary-cache"}; auto options = cache_options(td.path); @@ -141,10 +199,15 @@ TEST_CASE(cached_results_match_reference) auto ref_result = p_ref.eval({{"x", x}, {"y", y}}).back(); - // Compile once to fill the cache, then again so the result comes back out of it. auto warmup = pointwise_program(); warmup.compile(migraphx::make_target("gpu"), options); + auto entries = + std::count_if(migraphx::fs::recursive_directory_iterator{td.path}, + migraphx::fs::recursive_directory_iterator{}, + [](const auto& file) { return file.path().extension() == ".mxr"; }); + EXPECT(entries > 0); + auto t = migraphx::make_target("gpu"); auto p = pointwise_program(); p.compile(t, options); @@ -161,76 +224,37 @@ TEST_CASE(cached_results_match_reference) } auto gpu_result = t.copy_from(p.eval(params).back()); - std::vector ref_vec; - std::vector gpu_vec; - ref_result.visit([&](auto v) { ref_vec.assign(v.begin(), v.end()); }); - gpu_result.visit([&](auto v) { gpu_vec.assign(v.begin(), v.end()); }); - - EXPECT(migraphx::verify::verify_rms_range(ref_vec, gpu_vec)); + EXPECT(migraphx::verify::verify_rms_range(ref_result.to_vector(), + gpu_result.to_vector())); } -// With verification on, every reused result is checked against a fresh compile. A clean run -// means the keys really do capture what the compilers depend on. +// With verification on, every reused result is compiled again and compared, so a run that does +// not throw is one where the keys really do capture what the compilers depend on. TEST_CASE(verified_reuse_matches_fresh_compiles) { migraphx::tmp_dir td{"binary-cache"}; auto options = cache_options(td.path, /* verify */ true); - compile_and_get_stats(pointwise_program(), options); - auto stats = compile_and_get_stats(pointwise_program(), options); - EXPECT(stats.hits > 0); -} - -// A damaged entry must cost a recompile and nothing more. -TEST_CASE(corrupt_entry_is_ignored) -{ - migraphx::tmp_dir td{"binary-cache"}; - auto options = cache_options(td.path); + auto warmup = pointwise_program(); + warmup.compile(migraphx::make_target("gpu"), options); auto p = pointwise_program(); p.compile(migraphx::make_target("gpu"), options); - - std::size_t truncated = 0; - for(const auto& entry : migraphx::fs::recursive_directory_iterator(td.path)) - { - if(entry.path().extension() != ".mxr") - continue; - migraphx::write_buffer(entry.path(), std::vector(8, 0)); - truncated++; - } - EXPECT(truncated > 0); - - auto stats = compile_and_get_stats(pointwise_program(), options); - EXPECT(stats.hits == 0); - EXPECT(stats.compiled > 0); } TEST_CASE(entry_round_trip) { - migraphx::gpu::compiled_code code; - { - auto* fm = code.fragment.get_main_module(); - auto x = fm->add_parameter(migraphx::gpu::compiled_code::input_name(0), - {migraphx::shape::float_type, {2}}); - fm->add_return({fm->add_instruction(migraphx::make_op("abs"), x)}); - } - code.fill_map["float_type{2}"] = 3.0; - - migraphx::gpu::binary_cache_entry entry; - entry.key = "some-key"; - entry.op_name = "pointwise"; - entry.solution = migraphx::value{{"algo", "block"}}; - entry.code = code; + auto e = make_entry("some-key"); + auto buffer = migraphx::to_msgpack(migraphx::to_value(e)); - auto buffer = migraphx::to_msgpack(migraphx::to_value(entry)); - migraphx::gpu::binary_cache_entry loaded; + migraphx::gpu::binary_cache::entry loaded; migraphx::from_value(migraphx::from_msgpack(buffer), loaded); - EXPECT(loaded.key == entry.key); - EXPECT(loaded.op_name == entry.op_name); - EXPECT(loaded.solution == entry.solution); - EXPECT(loaded.code.fill_map == entry.code.fill_map); - EXPECT(*loaded.code.fragment.get_main_module() == *entry.code.fragment.get_main_module()); + EXPECT(loaded.key == e.key); + EXPECT(loaded.op_name == e.op_name); + EXPECT(loaded.solution == e.solution); + EXPECT(loaded.code.fill_map == e.code.fill_map); + EXPECT(*loaded.code.fragment.get_main_module() == *e.code.fragment.get_main_module()); } // The key has to cover everything handed to the compiler, not just the source text. Two From f5bdcabc8fc6fa6b59e9a71dc59b0939d97f7a44 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 29 Jul 2026 14:40:44 -0500 Subject: [PATCH 04/49] Remove locks --- src/targets/gpu/binary_cache.cpp | 73 +++--------- src/targets/gpu/compile_ops.cpp | 106 ++++++++++++++---- .../gpu/include/migraphx/gpu/binary_cache.hpp | 46 +++----- test/gpu/binary_cache.cpp | 20 ++-- 4 files changed, 123 insertions(+), 122 deletions(-) diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index 1616d0de6da..1f3c79f0ad1 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -62,72 +62,34 @@ binary_cache_settings binary_cache_settings::defaults() return {path, false}; } -binary_cache::stats::counts binary_cache::stats::get() -{ - const std::lock_guard lock(mutex); - return data; -} - -void binary_cache::stats::reused() -{ - const std::lock_guard lock(mutex); - data.reused++; -} - -void binary_cache::stats::hit() -{ - const std::lock_guard lock(mutex); - data.hits++; -} - -void binary_cache::stats::miss() -{ - const std::lock_guard lock(mutex); - data.misses++; -} - -void binary_cache::stats::compiled() -{ - const std::lock_guard lock(mutex); - data.compiled++; -} - void binary_cache::record_reused() { if(st) - st->reused(); + st->reused++; } void binary_cache::record_hit() { if(st) - st->hit(); + st->hits++; } optional binary_cache::record_miss() { if(st) - st->miss(); + st->misses++; return nullopt; } void binary_cache::record_compiled() { if(st) - st->compiled(); + st->compiled++; } -void binary_cache::configure(const binary_cache_settings& s) -{ - const std::lock_guard lock(mutex); - settings = s; -} +void binary_cache::configure(const binary_cache_settings& s) { settings = s; } -bool binary_cache::verify() -{ - const std::lock_guard lock(mutex); - return settings.verify; -} +bool binary_cache::verify() const { return settings.verify; } /// A digest of the kernel headers compiled into this build. Taken from the embedded sources /// rather than the files on disk, so it tracks what is actually compiled even when the build @@ -199,17 +161,13 @@ optional binary_cache::get(const context& ctx, const std::string& { if(key.empty()) return nullopt; - fs::path root; + auto it = memo.find(key); + if(it != memo.end()) { - const std::lock_guard lock(mutex); - auto it = memo.find(key); - if(it != memo.end()) - { - record_reused(); - return it->second; - } - root = settings.path; + record_reused(); + return it->second; } + const auto& root = settings.path; if(root.empty()) return record_miss(); @@ -236,7 +194,6 @@ optional binary_cache::get(const context& ctx, const std::string& } record_hit(); - const std::lock_guard lock(mutex); memo[key] = std::move(e.code); return memo[key]; } @@ -246,12 +203,8 @@ void binary_cache::insert(const context& ctx, const entry& e) if(e.key.empty()) return; record_compiled(); - fs::path root; - { - const std::lock_guard lock(mutex); - memo[e.key] = e.code; - root = settings.path; - } + memo[e.key] = e.code; + const auto& root = settings.path; if(root.empty()) return; diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 6a1aa3a02b4..17a709b193e 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -173,8 +173,37 @@ static void verify_reuse(context& ctx, to_string(fresh.code.fragment)); } -/// Reuse an earlier result for this key, or compile and record one. Both the compile pass and -/// the dynamic shapes compiled during evaluation go through here, so they cache alike. +/// Look for an earlier result. The cache is not guarded, so callers must not do this while +/// compiles are running. +static optional cache_lookup(context& ctx, const std::string& key) +{ + auto cached = ctx.get_binary_cache().get(ctx, key); + if(not cached.has_value()) + return nullopt; + compiler_replace cr; + cr.code = std::move(*cached); + return cr; +} + +/// Record a freshly compiled result, under the same restriction as cache_lookup. +static void cache_store(context& ctx, + const operation& preop, + const value& solution, + const std::string& key, + const value& problem, + const compiled_code& code) +{ + binary_cache::entry e; + e.key = key; + e.op_name = preop.name(); + e.problem = problem; + e.solution = solution; + e.code = code; + ctx.get_binary_cache().insert(ctx, e); +} + +/// Reuse an earlier result for this key, or compile and record one. For callers with a single +/// instruction to compile; the compile pass splits these steps around its parallel compiles. static compiler_replace compile_cached(context& ctx, instruction_ref ins, const operation& preop, @@ -182,23 +211,14 @@ static compiler_replace compile_cached(context& ctx, const std::string& key, const value& problem) { - auto& cache = ctx.get_binary_cache(); - if(auto cached = cache.get(ctx, key)) + if(auto cached = cache_lookup(ctx, key)) { - if(cache.verify()) - verify_reuse(ctx, ins, preop, solution, *cached); - compiler_replace cr; - cr.code = std::move(*cached); - return cr; + if(ctx.get_binary_cache().verify()) + verify_reuse(ctx, ins, preop, solution, cached->code); + return *cached; } auto cr = compile_fragment(ctx, ins, preop, solution); - binary_cache::entry entry; - entry.key = key; - entry.op_name = preop.name(); - entry.problem = problem; - entry.solution = solution; - entry.code = cr.code; - cache.insert(ctx, entry); + cache_store(ctx, preop, solution, key, problem, cr.code); return cr; } @@ -430,6 +450,22 @@ struct compile_plan return compile_key_or_empty(*ctx, ins, preop, solution); } + optional lookup(const std::string& key) const + { + return cache_lookup(*ctx, key); + } + + void store(const value& solution, const std::string& key, const compiled_code& code) const + { + cache_store(*ctx, preop, solution, key, config ? config->problem : value{}, code); + } + + /// Compile again and compare, so a key that fails to describe its result fails loudly. + void verify(const value& solution, const compiled_code& reused) const + { + verify_reuse(*ctx, ins, preop, solution, reused); + } + /// Inner repeat count when timing a candidate, raised for split-k (kernel + prefill). static std::size_t benchmark_bundle(const module& m) { @@ -441,12 +477,12 @@ struct compile_plan return std::max(1, 4 * n - 2); } - optional run_compile(const value& solution, const std::string& key) const + /// Compile without touching the cache, so this can run in parallel with other compiles. + optional run_compile(const value& solution) const { try { - return compile_cached( - *ctx, ins, preop, solution, key, config ? config->problem : value{}); + return compile_fragment(*ctx, ins, preop, solution); } catch(const std::exception& e) { @@ -680,6 +716,8 @@ struct compile_task /// since every target shares this key and therefore compiles to the same code. std::vector> targets = {}; optional result = nullopt; + /// Set when the result came from the cache, so it is not stored back afterwards. + bool reused = false; std::size_t plan_index() const { return targets.front().first; } }; @@ -741,14 +779,40 @@ struct compile_manager par_compile(candidates.size(), [&](auto i) { candidates[i].key = cps[candidates[i].plan_index].get_key(candidates[i].solution); }); + const bool verify_reused = not cps.empty() and cps.front().ctx->get_binary_cache().verify(); auto tasks = make_tasks(candidates); assert(tasks.size() <= candidates.size()); + + // The cache is unguarded, so everything it can answer is looked up before any compile + // starts, and everything compiled is stored after they all finish. + for(auto& task : tasks) + { + task.result = cps[task.plan_index()].lookup(task.key); + task.reused = task.result.has_value(); + } + par_compile(tasks.size(), [&](auto i) { - tasks[i].result = - cps[tasks[i].plan_index()].run_compile(tasks[i].solution, tasks[i].key); + auto& task = tasks[i]; + const auto& plan = cps[task.plan_index()]; + if(task.reused) + { + if(verify_reused) + plan.verify(task.solution, task.result->code); + } + else + { + task.result = plan.run_compile(task.solution); + } }); + for(const auto& task : tasks) + { + if(task.reused or not task.result.has_value()) + continue; + cps[task.plan_index()].store(task.solution, task.key, task.result->code); + } + for(const auto& task : tasks) { // The slots start out empty, so a task that failed to compile needs no write. diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index de659ec6afe..b3631bb7c17 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include @@ -47,6 +46,10 @@ struct context; * Results are held in memory for the life of the context and, when a cache directory is * configured, written to disk so later runs can reuse them. Things outside the key, such as the * compiler and the embedded kernel headers, are separated by the directory the entries live in. + * + * This is not thread safe, and deliberately so. Every key is known before any compile begins, so + * the compile pass looks results up and stores them in serial passes on either side of its + * parallel compiles, and nothing has to be guarded. */ struct MIGRAPHX_GPU_EXPORT binary_cache { @@ -71,36 +74,18 @@ struct MIGRAPHX_GPU_EXPORT binary_cache } }; - /** - * Counts of what the cache did. - * - * Only the tests ask for these, so a cache built without one keeps no counters at all. It - * carries its own lock because one may be shared by several caches. - */ + /// Counts of what the cache did. Only the tests ask for these, so a cache built without one + /// keeps no counters at all. struct stats { - struct counts - { - /// Served from memory, so an earlier compile in this process was shared. - std::size_t reused = 0; - /// Served from the cache directory. - std::size_t hits = 0; - /// Not found, so the caller had to compile. - std::size_t misses = 0; - /// Written to the cache after being compiled. - std::size_t compiled = 0; - }; - - counts get(); - - void reused(); - void hit(); - void miss(); - void compiled(); - - private: - std::mutex mutex; - counts data; + /// Served from memory, so an earlier compile in this process was shared. + std::size_t reused = 0; + /// Served from the cache directory. + std::size_t hits = 0; + /// Not found, so the caller had to compile. + std::size_t misses = 0; + /// Written to the cache after being compiled. + std::size_t compiled = 0; }; explicit binary_cache(std::shared_ptr s = nullptr) : st(std::move(s)) {} @@ -114,7 +99,7 @@ struct MIGRAPHX_GPU_EXPORT binary_cache void configure(const binary_cache_settings& s); /// True when reused results should be checked against a fresh compile. - bool verify(); + bool verify() const; /// Names the directory holding entries that this build can use: the entry format, the /// compiler, a digest of the embedded kernel headers, and the rocMLIR build. Empty when the @@ -131,7 +116,6 @@ struct MIGRAPHX_GPU_EXPORT binary_cache optional record_miss(); void record_compiled(); - std::mutex mutex; std::unordered_map memo; binary_cache_settings settings = binary_cache_settings::defaults(); std::shared_ptr st = nullptr; diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index fecc223d7cc..8dc74dce981 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -96,9 +96,9 @@ TEST_CASE(lookup_records_a_miss) migraphx::gpu::binary_cache cache{st}; EXPECT(not cache.get(ctx, "absent").has_value()); - EXPECT(st->get().misses == 1); - EXPECT(st->get().hits == 0); - EXPECT(st->get().reused == 0); + EXPECT(st->misses == 1); + EXPECT(st->hits == 0); + EXPECT(st->reused == 0); } // An entry served out of memory is one an earlier compile in this process already paid for. @@ -109,12 +109,12 @@ TEST_CASE(memory_lookup_records_reuse) migraphx::gpu::binary_cache cache{st}; cache.insert(ctx, make_entry("a-key")); - EXPECT(st->get().compiled == 1); + EXPECT(st->compiled == 1); auto found = cache.get(ctx, "a-key"); EXPECT(found.has_value()); - EXPECT(st->get().reused == 1); - EXPECT(st->get().misses == 0); + EXPECT(st->reused == 1); + EXPECT(st->misses == 0); } // A second cache shares nothing in memory, so anything it finds came off disk. @@ -134,8 +134,8 @@ TEST_CASE(disk_lookup_records_a_hit) auto found = reader.get(ctx, "shared-key"); EXPECT(found.has_value()); - EXPECT(st->get().hits == 1); - EXPECT(st->get().misses == 0); + EXPECT(st->hits == 1); + EXPECT(st->misses == 0); EXPECT(*found->fragment.get_main_module() == *make_code().fragment.get_main_module()); } @@ -165,7 +165,7 @@ TEST_CASE(corrupt_entry_is_ignored) reader.configure(settings); EXPECT(not reader.get(ctx, "damaged").has_value()); - EXPECT(st->get().misses == 1); + EXPECT(st->misses == 1); } // Without a directory nothing reaches disk, though results are still shared in memory. @@ -179,7 +179,7 @@ TEST_CASE(no_directory_writes_nothing) cache.insert(ctx, make_entry("in-memory-only")); EXPECT(cache.get(ctx, "in-memory-only").has_value()); - EXPECT(st->get().reused == 1); + EXPECT(st->reused == 1); EXPECT(migraphx::fs::is_empty(td.path)); } From 1dd35193c771217a582bf7777cb7e568302c65a4 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 31 Jul 2026 18:26:48 -0500 Subject: [PATCH 05/49] Add missing files --- .../migraphx/gpu/binary_cache_settings.hpp | 59 +++++++++++++++ .../include/migraphx/gpu/compiled_code.hpp | 74 +++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 src/targets/gpu/include/migraphx/gpu/binary_cache_settings.hpp create mode 100644 src/targets/gpu/include/migraphx/gpu/compiled_code.hpp diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache_settings.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache_settings.hpp new file mode 100644 index 00000000000..bd985f543eb --- /dev/null +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache_settings.hpp @@ -0,0 +1,59 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_SETTINGS_HPP +#define MIGRAPHX_GUARD_GPU_BINARY_CACHE_SETTINGS_HPP + +// What a caller needs to hold or configure a binary cache, separated from the cache itself so +// that context.hpp and compile_ops.hpp do not pull the IR headers into every GPU source file. + +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +struct binary_cache; + +/// How the cache behaves for one compile. Supplied through the target's backend options, with +/// the MIGRAPHX_BINARY_CACHE environment variable as the default for the directory. +struct binary_cache_settings +{ + /// Where entries are kept. Nothing is written to disk when this is empty, though results + /// are still shared in memory. + std::string path = {}; + /// Compile even when a result can be reused, and fail if the two disagree. + bool verify = false; + + MIGRAPHX_GPU_EXPORT static binary_cache_settings defaults(); +}; + +/// Lets a context own a cache without seeing its definition. +MIGRAPHX_GPU_EXPORT std::shared_ptr make_binary_cache(); + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx +#endif // MIGRAPHX_GUARD_GPU_BINARY_CACHE_SETTINGS_HPP diff --git a/src/targets/gpu/include/migraphx/gpu/compiled_code.hpp b/src/targets/gpu/include/migraphx/gpu/compiled_code.hpp new file mode 100644 index 00000000000..fa1c84d45f1 --- /dev/null +++ b/src/targets/gpu/include/migraphx/gpu/compiled_code.hpp @@ -0,0 +1,74 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MIGRAPHX_GUARD_GPU_COMPILED_CODE_HPP +#define MIGRAPHX_GUARD_GPU_COMPILED_CODE_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +/** + * The result of compiling one precompile_op, in a form that can be serialized and reused. + * + * The fragment holds the compiled code objects along with any prefills and allocations they + * need. Its parameters correspond by position to the inputs of the instruction it replaces, so + * the fragment does not refer to the instruction that produced it and can be reused for every + * instruction that compiles to the same code. It is kept as a program because that is what + * already converts to and from a value. + */ +struct MIGRAPHX_GPU_EXPORT compiled_code +{ + program fragment = {}; + /// Values used to prefill inputs while benchmarking, keyed by type and size string. + std::unordered_map fill_map = {}; + + template + static auto reflect(Self& self, F f) + { + return pack(f(self.fragment, "fragment"), f(self.fill_map, "fill_map")); + } + + bool empty() const { return fragment.get_main_module()->size() == 0; } + + /// Splice the fragment into m in place of ins, mapping each fragment parameter to the + /// input of ins in the same position. + void replace(module& m, instruction_ref ins) const; + + /// Name of the fragment parameter standing in for the input at index i. + static std::string input_name(std::size_t i); +}; + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx + +#endif // MIGRAPHX_GUARD_GPU_COMPILED_CODE_HPP From 9a80988d3d28ea75cc1e2956033d56a4e7244f5c Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 31 Jul 2026 18:27:01 -0500 Subject: [PATCH 06/49] Update license --- src/include/migraphx/tmp_dir.hpp | 2 +- src/targets/gpu/compile_pointwise.cpp | 2 +- src/targets/gpu/compiler.cpp | 2 +- src/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp | 2 +- src/targets/gpu/include/migraphx/gpu/compiler.hpp | 2 +- src/targets/gpu/jit/gather.cpp | 2 +- src/targets/gpu/jit/pad.cpp | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/include/migraphx/tmp_dir.hpp b/src/include/migraphx/tmp_dir.hpp index f5c06b95f24..8a851124089 100644 --- a/src/include/migraphx/tmp_dir.hpp +++ b/src/include/migraphx/tmp_dir.hpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/src/targets/gpu/compile_pointwise.cpp b/src/targets/gpu/compile_pointwise.cpp index b209caaae0d..0fff76aba7a 100644 --- a/src/targets/gpu/compile_pointwise.cpp +++ b/src/targets/gpu/compile_pointwise.cpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/src/targets/gpu/compiler.cpp b/src/targets/gpu/compiler.cpp index bc7ae92f849..642fe4016ed 100644 --- a/src/targets/gpu/compiler.cpp +++ b/src/targets/gpu/compiler.cpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/src/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp b/src/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp index 35d12795eb4..4793fe417a8 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/src/targets/gpu/include/migraphx/gpu/compiler.hpp b/src/targets/gpu/include/migraphx/gpu/compiler.hpp index c2eda1f43db..90e098e4e83 100644 --- a/src/targets/gpu/include/migraphx/gpu/compiler.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compiler.hpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/src/targets/gpu/jit/gather.cpp b/src/targets/gpu/jit/gather.cpp index 66028fcea3f..80d5341db0c 100644 --- a/src/targets/gpu/jit/gather.cpp +++ b/src/targets/gpu/jit/gather.cpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/src/targets/gpu/jit/pad.cpp b/src/targets/gpu/jit/pad.cpp index a4830624d83..d67616b3fb4 100644 --- a/src/targets/gpu/jit/pad.cpp +++ b/src/targets/gpu/jit/pad.cpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal From 625ad00ad0d79a1efdfac799c78e5257cd6bccce Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 17:20:50 -0500 Subject: [PATCH 07/49] Cleanup binary settings --- src/targets/gpu/binary_cache.cpp | 11 ---- src/targets/gpu/compile_ops.cpp | 3 - .../gpu/include/migraphx/gpu/binary_cache.hpp | 26 ++++++-- .../migraphx/gpu/binary_cache_settings.hpp | 59 ------------------- .../migraphx/gpu/compile_hipblaslt.hpp | 2 +- .../include/migraphx/gpu/compile_miopen.hpp | 2 +- .../gpu/include/migraphx/gpu/compile_ops.hpp | 2 - .../gpu/include/migraphx/gpu/context.hpp | 20 +++++-- src/targets/gpu/target.cpp | 29 +++++---- test/gpu/binary_cache.cpp | 19 +++--- 10 files changed, 61 insertions(+), 112 deletions(-) delete mode 100644 src/targets/gpu/include/migraphx/gpu/binary_cache_settings.hpp diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index 1f3c79f0ad1..629d9554fbc 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -41,8 +40,6 @@ namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { namespace gpu { -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_BINARY_CACHE) - // Bump when the shape of a stored fragment changes, which happens when a compiler changes the // instructions it replaces with or when a serialized operator gains or loses a field. Such a // change is invisible to the key, since the source handed to the backend is unaffected. @@ -56,12 +53,6 @@ static constexpr const char* rocmlir_id = "nomlir"; std::shared_ptr make_binary_cache() { return std::make_shared(); } -binary_cache_settings binary_cache_settings::defaults() -{ - static const std::string path = string_value_of(MIGRAPHX_BINARY_CACHE{}); - return {path, false}; -} - void binary_cache::record_reused() { if(st) @@ -87,8 +78,6 @@ void binary_cache::record_compiled() st->compiled++; } -void binary_cache::configure(const binary_cache_settings& s) { settings = s; } - bool binary_cache::verify() const { return settings.verify; } /// A digest of the kernel headers compiled into this build. Taken from the embedded sources diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 17a709b193e..63b1bc99147 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -894,9 +894,6 @@ void compile_ops::apply(module_pass_manager& mpm) const { bool is_root = &mpm.get_module() == mpm.get_root_module(); auto& m = mpm.get_module(); - // The cache outlives this pass, since compiling a dynamic shape at evaluation time uses it - // too, so the settings are recorded on it rather than passed along. - ctx->get_binary_cache().configure(cache_settings); compile_manager cm; cm.exhaustive = exhaustive_tune; // Find all precompile ops diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index b3631bb7c17..336f191615c 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -25,8 +25,8 @@ #define MIGRAPHX_GUARD_GPU_BINARY_CACHE_HPP #include -#include #include +#include #include #include #include @@ -38,8 +38,21 @@ namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { namespace gpu { +MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_BINARY_CACHE) + struct context; +/// How the cache behaves for one compile. Supplied through the target's backend options, with +/// the MIGRAPHX_BINARY_CACHE environment variable as the default for the directory. +struct binary_cache_settings +{ + /// Where entries are kept. Nothing is written to disk when this is empty, though results + /// are still shared in memory. + std::string path = string_value_of(MIGRAPHX_BINARY_CACHE{}); + /// Compile even when a result can be reused, and fail if the two disagree. + bool verify = false; +}; + /** * Compiled kernels, keyed by a string describing what the compiler was given. * @@ -88,7 +101,10 @@ struct MIGRAPHX_GPU_EXPORT binary_cache std::size_t compiled = 0; }; - explicit binary_cache(std::shared_ptr s = nullptr) : st(std::move(s)) {} + explicit binary_cache(binary_cache_settings s = {}, std::shared_ptr counters = nullptr) + : settings(std::move(s)), st(std::move(counters)) + { + } /// Look up a key, consulting memory first and then the cache directory. optional get(const context& ctx, const std::string& key); @@ -96,8 +112,6 @@ struct MIGRAPHX_GPU_EXPORT binary_cache /// Record a compiled result under its key. void insert(const context& ctx, const entry& e); - void configure(const binary_cache_settings& s); - /// True when reused results should be checked against a fresh compile. bool verify() const; @@ -117,8 +131,8 @@ struct MIGRAPHX_GPU_EXPORT binary_cache void record_compiled(); std::unordered_map memo; - binary_cache_settings settings = binary_cache_settings::defaults(); - std::shared_ptr st = nullptr; + binary_cache_settings settings; + std::shared_ptr st; }; } // namespace gpu diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache_settings.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache_settings.hpp deleted file mode 100644 index bd985f543eb..00000000000 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache_settings.hpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_SETTINGS_HPP -#define MIGRAPHX_GUARD_GPU_BINARY_CACHE_SETTINGS_HPP - -// What a caller needs to hold or configure a binary cache, separated from the cache itself so -// that context.hpp and compile_ops.hpp do not pull the IR headers into every GPU source file. - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct binary_cache; - -/// How the cache behaves for one compile. Supplied through the target's backend options, with -/// the MIGRAPHX_BINARY_CACHE environment variable as the default for the directory. -struct binary_cache_settings -{ - /// Where entries are kept. Nothing is written to disk when this is empty, though results - /// are still shared in memory. - std::string path = {}; - /// Compile even when a result can be reused, and fail if the two disagree. - bool verify = false; - - MIGRAPHX_GPU_EXPORT static binary_cache_settings defaults(); -}; - -/// Lets a context own a cache without seeing its definition. -MIGRAPHX_GPU_EXPORT std::shared_ptr make_binary_cache(); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_BINARY_CACHE_SETTINGS_HPP diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hipblaslt.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hipblaslt.hpp index 3ad029b3aec..a4c64744e5b 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hipblaslt.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hipblaslt.hpp @@ -64,7 +64,7 @@ struct hipblaslt_op struct compile_hipblaslt { - context* ctx = nullptr; + migraphx::context* ctx = nullptr; std::string name() const { return "gpu::compile_hipblaslt"; } void apply(module& m) const; }; diff --git a/src/targets/gpu/include/migraphx/gpu/compile_miopen.hpp b/src/targets/gpu/include/migraphx/gpu/compile_miopen.hpp index 03dd669e536..ec58b77e40b 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_miopen.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_miopen.hpp @@ -39,7 +39,7 @@ namespace gpu { struct compile_miopen { - context* ctx = nullptr; + migraphx::context* ctx = nullptr; std::string name() const { return "gpu::compile_miopen"; } void apply(module& m) const; std::size_t compile(operation& op, instruction_ref ins) const; diff --git a/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp b/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp index 491341c686b..1898899f1b5 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_ops.hpp @@ -25,7 +25,6 @@ #define MIGRAPHX_GUARD_GPU_COMPILE_OPS_HPP #include -#include #include namespace migraphx { @@ -41,7 +40,6 @@ struct MIGRAPHX_GPU_EXPORT compile_ops { context* ctx = nullptr; bool exhaustive_tune = false; - binary_cache_settings cache_settings{}; std::string name() const { return "gpu::compile_ops"; } void apply(module_pass_manager& mpm) const; }; diff --git a/src/targets/gpu/include/migraphx/gpu/context.hpp b/src/targets/gpu/include/migraphx/gpu/context.hpp index 92e069c4e27..918a17f1b20 100644 --- a/src/targets/gpu/include/migraphx/gpu/context.hpp +++ b/src/targets/gpu/include/migraphx/gpu/context.hpp @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -52,6 +51,11 @@ namespace gpu { MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_NULL_STREAM) MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_NSTREAMS) +struct binary_cache; + +/// Lets a context own a cache without seeing its definition. +MIGRAPHX_GPU_EXPORT std::shared_ptr make_binary_cache(); + using hip_event_ptr = MIGRAPHX_MANAGE_PTR(hipEvent_t, hipEventDestroy); struct hip_device @@ -284,10 +288,13 @@ struct context this->save(); } }; - context(std::size_t device_id = 0, std::size_t n = value_of(MIGRAPHX_NSTREAMS{}, 1)) + context(std::size_t device_id = 0, + std::size_t n = value_of(MIGRAPHX_NSTREAMS{}, 1), + std::shared_ptr bcache = make_binary_cache()) : current_device(std::make_shared(device_id, n)), begin_event(create_event()), - finish_event(create_event()) + finish_event(create_event()), + bc(std::move(bcache)) { // Bind the cache to this context's device (cross-compile safe: the // key is derived from the context, not a live HIP query). @@ -296,8 +303,9 @@ struct context /// Construct a context for a device that is not present, which can only be /// used to compile and not to execute. - explicit context(const device_description& desc) - : current_device(std::make_shared(desc)) + explicit context(const device_description& desc, + std::shared_ptr bcache = make_binary_cache()) + : current_device(std::make_shared(desc)), bc(std::move(bcache)) { } @@ -480,7 +488,7 @@ struct context shared begin_event = nullptr; shared finish_event = nullptr; std::shared_ptr pc = std::make_shared(); - std::shared_ptr bc = make_binary_cache(); + std::shared_ptr bc; }; inline void migraphx_to_value(value& v, const context& ctx) { v = ctx.to_value(); } diff --git a/src/targets/gpu/target.cpp b/src/targets/gpu/target.cpp index 9dad24e625d..2de845604c4 100644 --- a/src/targets/gpu/target.cpp +++ b/src/targets/gpu/target.cpp @@ -66,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -107,7 +108,7 @@ struct backend_options { std::vector mlss_use_specific_ops = {}; /// Where compiled kernels are cached between runs. Defaults to ``MIGRAPHX_BINARY_CACHE``. - std::string binary_cache = binary_cache_settings::defaults().path; + std::string binary_cache = binary_cache_settings{}.path; /// Compile even when a kernel could be reused, and fail if the two disagree. bool binary_cache_verify = false; @@ -257,9 +258,7 @@ struct pipeline_factory adjust_allocation{gpu_allocation_model{}}, dead_code_elimination{}, lower_device_ops{}, - compile_ops{get_context(), - options.exhaustive_tune, - {backend_opts.binary_cache, backend_opts.binary_cache_verify}}, + compile_ops{get_context(), options.exhaustive_tune}, dead_code_elimination{}, promote_literals{}, dead_code_elimination{}, @@ -280,13 +279,26 @@ struct pipeline_factory }; } // namespace +static migraphx::context make_context(const target& t, std::shared_ptr bc) +{ + if(t.is_cross_compile()) + return context(t.desc, std::move(bc)); + return context(gpu::get_device_id(), value_of(MIGRAPHX_NSTREAMS{}, 1), std::move(bc)); +} + std::vector target::get_passes(migraphx::context& gctx, const compile_options& options) const { + auto backend_opts = from_value(value(options.backend_options)); + // The initial context predates the compile options, so rebuild it around a cache + // constructed with the settings they carry. + gctx = make_context(*this, + std::make_shared(binary_cache_settings{ + backend_opts.binary_cache, backend_opts.binary_cache_verify})); auto& ctx = any_cast(gctx); ctx.set_exhaustive_tune_flag(options.exhaustive_tune); ctx.load_problem_cache(); // TODO: update load_problem_cache to include gpu arch - pipeline_factory p{&gctx, options, from_value(value(options.backend_options))}; + pipeline_factory p{&gctx, options, backend_opts}; std::vector> pipelines = { p.dynamic_shapes_pipeline(), @@ -303,12 +315,7 @@ std::vector target::get_passes(migraphx::context& gctx, const compile_opti std::string target::name() const { return "gpu"; } -migraphx::context target::get_context() const -{ - if(is_cross_compile()) - return context(desc); - return context(gpu::get_device_id()); -} +migraphx::context target::get_context() const { return make_context(*this, make_binary_cache()); } argument target::copy_to(const argument& arg) const { diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 8dc74dce981..c364ad6a05b 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -93,7 +93,7 @@ TEST_CASE(lookup_records_a_miss) { migraphx::gpu::context ctx; auto st = make_stats(); - migraphx::gpu::binary_cache cache{st}; + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}, st}; EXPECT(not cache.get(ctx, "absent").has_value()); EXPECT(st->misses == 1); @@ -106,7 +106,7 @@ TEST_CASE(memory_lookup_records_reuse) { migraphx::gpu::context ctx; auto st = make_stats(); - migraphx::gpu::binary_cache cache{st}; + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}, st}; cache.insert(ctx, make_entry("a-key")); EXPECT(st->compiled == 1); @@ -124,13 +124,11 @@ TEST_CASE(disk_lookup_records_a_hit) migraphx::gpu::context ctx; migraphx::gpu::binary_cache_settings settings{td.path.string(), false}; - migraphx::gpu::binary_cache writer; - writer.configure(settings); + migraphx::gpu::binary_cache writer{settings}; writer.insert(ctx, make_entry("shared-key")); auto st = make_stats(); - migraphx::gpu::binary_cache reader{st}; - reader.configure(settings); + migraphx::gpu::binary_cache reader{settings, st}; auto found = reader.get(ctx, "shared-key"); EXPECT(found.has_value()); @@ -146,8 +144,7 @@ TEST_CASE(corrupt_entry_is_ignored) migraphx::gpu::context ctx; migraphx::gpu::binary_cache_settings settings{td.path.string(), false}; - migraphx::gpu::binary_cache writer; - writer.configure(settings); + migraphx::gpu::binary_cache writer{settings}; writer.insert(ctx, make_entry("damaged")); std::size_t truncated = 0; @@ -161,8 +158,7 @@ TEST_CASE(corrupt_entry_is_ignored) EXPECT(truncated > 0); auto st = make_stats(); - migraphx::gpu::binary_cache reader{st}; - reader.configure(settings); + migraphx::gpu::binary_cache reader{settings, st}; EXPECT(not reader.get(ctx, "damaged").has_value()); EXPECT(st->misses == 1); @@ -174,8 +170,7 @@ TEST_CASE(no_directory_writes_nothing) migraphx::tmp_dir td{"binary-cache"}; migraphx::gpu::context ctx; auto st = make_stats(); - migraphx::gpu::binary_cache cache{st}; - cache.configure({"", false}); + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}, st}; cache.insert(ctx, make_entry("in-memory-only")); EXPECT(cache.get(ctx, "in-memory-only").has_value()); From 148a1cda18bb77599ddcc3d2c15b8d56f726364c Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 18:12:11 -0500 Subject: [PATCH 08/49] Simplify --- src/targets/gpu/binary_cache.cpp | 122 ++++++++---------- src/targets/gpu/compile_hip.cpp | 18 +-- src/targets/gpu/compile_hip_code_object.cpp | 20 ++- src/targets/gpu/compile_ops.cpp | 70 ++++++---- src/targets/gpu/compiled_code.cpp | 20 +-- .../gpu/include/migraphx/gpu/binary_cache.hpp | 22 +--- .../gpu/include/migraphx/gpu/compile_hip.hpp | 2 +- .../migraphx/gpu/compile_hip_code_object.hpp | 3 + .../gpu/include/migraphx/gpu/context.hpp | 6 +- src/targets/gpu/include/migraphx/gpu/mlir.hpp | 2 +- src/targets/gpu/jit/gather.cpp | 3 +- src/targets/gpu/jit/pad.cpp | 3 +- src/targets/gpu/jit/pointwise.cpp | 3 +- src/targets/gpu/jit/reduce.cpp | 6 +- test/gpu/binary_cache.cpp | 41 +++--- 15 files changed, 162 insertions(+), 179 deletions(-) diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index 629d9554fbc..65b107fe523 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -53,33 +52,10 @@ static constexpr const char* rocmlir_id = "nomlir"; std::shared_ptr make_binary_cache() { return std::make_shared(); } -void binary_cache::record_reused() -{ - if(st) - st->reused++; -} - -void binary_cache::record_hit() -{ - if(st) - st->hits++; -} - -optional binary_cache::record_miss() -{ - if(st) - st->misses++; - return nullopt; -} - -void binary_cache::record_compiled() -{ - if(st) - st->compiled++; -} - bool binary_cache::verify() const { return settings.verify; } +static std::string short_digest(const std::string& s) { return md5(s).substr(0, 12); } + /// A digest of the kernel headers compiled into this build. Taken from the embedded sources /// rather than the files on disk, so it tracks what is actually compiled even when the build /// system has not reconfigured. @@ -91,7 +67,7 @@ static const std::string& kernels_digest() { ss << path << "\n" << content << "\n"; } - return md5(ss.str()).substr(0, 12); + return short_digest(ss.str()); }(); return digest; } @@ -105,16 +81,15 @@ const std::string& binary_cache::version_dir() // The version numbers make the directory readable; the hash of the full version string // separates builds that share them, since it also covers the source revision. return std::string{binary_cache_format} + "-hip" + compiler.major + "." + compiler.minor + - "." + md5(compiler.version).substr(0, 12) + "-kernels" + kernels_digest() + - "-rocmlir" + rocmlir_id; + "." + short_digest(compiler.version) + "-kernels" + kernels_digest() + "-rocmlir" + + rocmlir_id; }(); return dir; } -/// Entries are grouped by the device they were compiled for. The architecture is part of this -/// because it is passed to the compiler rather than appearing in the source; the core count and -/// wavefront size are here to keep the directory self-describing, since they already reach the -/// key through the launch bounds and -D defines in the parameter list. +/// Entries are grouped by the device they were compiled for. This keeps the directory +/// self-describing; the arch, core count and wavefront size already reach the key through the +/// arch line, the launch bounds and the -D defines. static std::string device_dir(const context& ctx) { const auto& device = ctx.get_current_device(); @@ -143,7 +118,11 @@ static void write_stamp(const fs::path& dir) ss << "hip: " << hip_compiler_version().version << "\n"; ss << "kernels: " << kernels_digest() << "\n"; ss << "rocmlir: " << rocmlir_id << "\n"; - write_string(stamp, ss.str()); + // Publish by rename like the entries, so concurrent writers cannot tear the file. + auto tmp = stamp; + tmp += "." + unique_string("tmp"); + write_string(tmp, ss.str()); + fs::rename(tmp, stamp); } optional binary_cache::get(const context& ctx, const std::string& key) @@ -153,17 +132,21 @@ optional binary_cache::get(const context& ctx, const std::string& auto it = memo.find(key); if(it != memo.end()) { - record_reused(); + counters.reused++; return it->second; } const auto& root = settings.path; if(root.empty()) - return record_miss(); - + { + counters.misses++; + return nullopt; + } auto path = entry_path(root, ctx, key); if(path.empty() or not fs::exists(path)) - return record_miss(); - + { + counters.misses++; + return nullopt; + } entry e; try { @@ -174,51 +157,50 @@ optional binary_cache::get(const context& ctx, const std::string& // A damaged or stale entry is only worth a recompile, so treat every failure as a miss // and let the result be written over the top. log::warn() << "Ignoring unreadable binary cache entry " << path << ": " << ex.what(); - return record_miss(); + counters.misses++; + return nullopt; } if(e.key != key) { log::warn() << "Ignoring binary cache entry with mismatched key: " << path; - return record_miss(); + counters.misses++; + return nullopt; } - record_hit(); - memo[key] = std::move(e.code); - return memo[key]; + counters.hits++; + return memo[key] = std::move(e.code); } -void binary_cache::insert(const context& ctx, const entry& e) +void binary_cache::insert(const context& ctx, entry e) { if(e.key.empty()) return; - record_compiled(); - memo[e.key] = e.code; + counters.compiled++; const auto& root = settings.path; - if(root.empty()) - return; - - auto path = entry_path(root, ctx, e.key); - if(path.empty()) - return; - // Publish by rename so a reader never sees a half-written entry. The content is decided - // entirely by the key, so a writer that loses the race replaces the file with the same bytes - // and no locking is needed. - auto tmp = path; - tmp += "." + unique_string("tmp"); - try - { - fs::create_directories(path.parent_path()); - write_stamp(path.parent_path().parent_path()); - write_buffer(tmp, to_msgpack(migraphx::to_value(e))); - fs::rename(tmp, path); - } - catch(const std::exception& ex) + auto path = root.empty() ? fs::path{} : entry_path(root, ctx, e.key); + if(not path.empty()) { - // Leaving the temporary behind would accumulate in the cache directory. - std::error_code ec; - fs::remove(tmp, ec); - log::warn() << "Failed to write binary cache entry " << path << ": " << ex.what(); + // Publish by rename so a reader never sees a half-written entry. The content is decided + // entirely by the key, so a writer that loses the race replaces the file with the same + // bytes and no locking is needed. + auto tmp = path; + tmp += "." + unique_string("tmp"); + try + { + fs::create_directories(path.parent_path()); + write_stamp(fs::path(root) / version_dir()); + write_buffer(tmp, to_msgpack(migraphx::to_value(e))); + fs::rename(tmp, path); + } + catch(const std::exception& ex) + { + // Leaving the temporary behind would accumulate in the cache directory. + std::error_code ec; + fs::remove(tmp, ec); + log::warn() << "Failed to write binary cache entry " << path << ": " << ex.what(); + } } + memo[std::move(e.key)] = std::move(e.code); } } // namespace gpu diff --git a/src/targets/gpu/compile_hip.cpp b/src/targets/gpu/compile_hip.cpp index a2f470467f8..a5a664ab6e5 100644 --- a/src/targets/gpu/compile_hip.cpp +++ b/src/targets/gpu/compile_hip.cpp @@ -26,15 +26,12 @@ #include #include #include -#include #include #include #include #include #include -#include #include -#include #ifdef MIGRAPHX_USE_HIPRTC #include @@ -383,8 +380,8 @@ static constexpr std::string_view version_suffix = ">>>"; // accepts. Keeping it fixed means the answer does not depend on the device present. static const char* const version_probe_arch = "gfx950"; -// Probe that makes the device compiler embed its own version string in the object. It is not -// the compiler this library was built with, so its version is only known at runtime. +// Probe that makes the device compiler embed its own version string in the object. It need not +// be the compiler this library was built with, so its version is only known at runtime. static const char* const version_probe = R"__migraphx__( #define MIGRAPHX_STRINGIZE_(x) #x #define MIGRAPHX_STRINGIZE(x) MIGRAPHX_STRINGIZE_(x) @@ -400,8 +397,8 @@ static const char* const version_probe = R"__migraphx__( #endif extern "C" __attribute__((used)) __device__ const char migraphx_compiler_version[] = - "<<>>"; + "${prefix}" MIGRAPHX_COMPILER_MAJOR "|" MIGRAPHX_COMPILER_MINOR "|" + MIGRAPHX_COMPILER_VERSION "${suffix}"; extern "C" __global__ void migraphx_version_probe(char* p) { @@ -430,7 +427,12 @@ const hip_compiler_info& hip_compiler_version() static const hip_compiler_info info = [] { try { - auto cos = compile_hip_src({src_file{"main.cpp", version_probe}}, + // Interpolated so the markers parse_version searches for cannot drift from the ones + // the probe embeds. + auto probe = interpolate_string( + version_probe, + {{"prefix", std::string{version_prefix}}, {"suffix", std::string{version_suffix}}}); + auto cos = compile_hip_src({src_file{"main.cpp", probe}}, {"-std=c++17"}, version_probe_arch, /* quiet */ true); diff --git a/src/targets/gpu/compile_hip_code_object.cpp b/src/targets/gpu/compile_hip_code_object.cpp index 9cc4982f465..b517e8221c2 100644 --- a/src/targets/gpu/compile_hip_code_object.cpp +++ b/src/targets/gpu/compile_hip_code_object.cpp @@ -260,17 +260,22 @@ std::string hip_compile_key(const context& ctx, const hip_src& src) ss << "global=" << options.global << "\n"; ss << "local=" << options.local << "\n"; ss << "output_arg=" << options.output_arg << "\n"; - ss << "params=" << join_strings(options.params, " ") << "\n"; + // One param per line so the key keeps the boundaries the compiler command line has. + for(const auto& p : options.params) + ss << "param=" << p << "\n"; // The shapes are recorded because they become fields of the code object, not because they // are part of the source; the source sees them through the generated tensor views below. for(const auto& s : options.inputs) ss << "input=" << s << "\n"; ss << "output=" << options.output << "\n"; + // The sources are length-prefixed so two compiles cannot serialize to the same key by their + // contents running together. for(const auto& f : options.additional_src_files) - ss << "src=" << f.path << "\n" << f.content << "\n"; - ss << generate_args_hpp(options.virtual_inputs.empty() ? options.inputs - : options.virtual_inputs); - ss << "main.cpp\n" << src.content; + ss << "src=" << f.path << ":" << f.content.size() << "\n" << f.content << "\n"; + auto args_hpp = + generate_args_hpp(options.virtual_inputs.empty() ? options.inputs : options.virtual_inputs); + ss << "args.hpp:" << args_hpp.size() << "\n" << args_hpp; + ss << "main.cpp:" << src.content.size() << "\n" << src.content; return ss.str(); } @@ -293,6 +298,11 @@ compile_hip_code_object(context& ctx, const std::string& content, hip_compile_op options.output_arg}; } +operation compile_hip_code_object(context& ctx, hip_src src) +{ + return compile_hip_code_object(ctx, src.content, std::move(src.options)); +} + } // namespace gpu } // namespace MIGRAPHX_INLINE_NS } // namespace migraphx diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 63b1bc99147..7af32c01a6c 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include namespace migraphx { @@ -108,8 +109,8 @@ static program make_fragment(const compiler_replace& cr, instruction_ref ins) auto* fm = frag.get_main_module(); const auto& args = ins->inputs(); std::vector inputs; - // Named by position rather than by a counter the transform mutates, since compiled_code - // maps each parameter back onto the input in the same position. + // Parameters are named by input position, since compiled_code::replace maps each parameter + // back onto the input in the same position. auto idx = range(args.size()); std::transform(idx.begin(), idx.end(), std::back_inserter(inputs), [&](auto i) { return fm->add_parameter(compiled_code::input_name(i), args[i]->get_shape()); @@ -199,7 +200,7 @@ static void cache_store(context& ctx, e.problem = problem; e.solution = solution; e.code = code; - ctx.get_binary_cache().insert(ctx, e); + ctx.get_binary_cache().insert(ctx, std::move(e)); } /// Reuse an earlier result for this key, or compile and record one. For callers with a single @@ -461,8 +462,11 @@ struct compile_plan } /// Compile again and compare, so a key that fails to describe its result fails loudly. + /// Does nothing unless the cache was configured to verify reuse. void verify(const value& solution, const compiled_code& reused) const { + if(not ctx->get_binary_cache().verify()) + return; verify_reuse(*ctx, ins, preop, solution, reused); } @@ -530,13 +534,8 @@ struct compile_plan { ctx->get_problem_cache().mark(preop.name(), problem); results.resize(solutions.size()); - for(auto i : range(solutions.size())) - { - auto& candidate = candidates.emplace_back(); - candidate.plan_index = plan_index; - candidate.result_index = i; - candidate.solution = solutions[i]; - } + for(std::size_t i : range(solutions.size())) + candidates.push_back({plan_index, i, solutions[i]}); } } } @@ -718,8 +717,14 @@ struct compile_task optional result = nullopt; /// Set when the result came from the cache, so it is not stored back afterwards. bool reused = false; + /// A failure raised while verifying the reused result, rethrown after the workers join. + std::exception_ptr error = nullptr; - std::size_t plan_index() const { return targets.front().first; } + std::size_t plan_index() const + { + assert(not targets.empty()); + return targets.front().first; + } }; struct compile_manager @@ -744,13 +749,13 @@ struct compile_manager * Since every key is known before any compile starts, each group can be handed to exactly * one task and the results shared afterwards, so the compiles never have to coordinate. */ - static std::vector make_tasks(std::vector& candidates) + static std::vector make_tasks(const std::vector& candidates) { std::vector tasks; - // The keys are whole kernel sources, so the index views them rather than copying them; - // candidates outlives the tasks built here. + // The keys are whole kernel sources, so the index views them instead of copying; the + // index does not outlive this function. std::unordered_map task_index; - for(auto& c : candidates) + for(const auto& c : candidates) { std::size_t i = tasks.size(); if(c.key.empty()) @@ -779,8 +784,6 @@ struct compile_manager par_compile(candidates.size(), [&](auto i) { candidates[i].key = cps[candidates[i].plan_index].get_key(candidates[i].solution); }); - const bool verify_reused = not cps.empty() and cps.front().ctx->get_binary_cache().verify(); - auto tasks = make_tasks(candidates); assert(tasks.size() <= candidates.size()); @@ -797,14 +800,26 @@ struct compile_manager const auto& plan = cps[task.plan_index()]; if(task.reused) { - if(verify_reused) + // A throw would escape the worker thread and terminate the process, so carry + // the failure back and rethrow it after the workers join. + try + { plan.verify(task.solution, task.result->code); + } + catch(...) + { + task.error = std::current_exception(); + } } else { task.result = plan.run_compile(task.solution); } }); + auto failed = std::find_if( + tasks.begin(), tasks.end(), [](const auto& task) { return task.error != nullptr; }); + if(failed != tasks.end()) + std::rethrow_exception(failed->error); for(const auto& task : tasks) { @@ -813,19 +828,24 @@ struct compile_manager cps[task.plan_index()].store(task.solution, task.key, task.result->code); } - for(const auto& task : tasks) + for(auto& task : tasks) { // The slots start out empty, so a task that failed to compile needs no write. if(not task.result.has_value()) continue; assert(not task.result->code.empty()); - for(auto [pi, ri] : task.targets) + // Only the serializable code is used from here on. Dropping the replace function + // releases the code objects and split modules its closure holds, and keeps it from + // being run against an instruction other than the one it was built for. + task.result->replace_fn = nullptr; + // Every target gets its own copy; the last takes the result itself. + const auto last = task.targets.size() - 1; + for(auto i : range(task.targets.size())) { - auto cr = *task.result; - // Only the fragment is used from here on. Dropping the replace function releases - // the code objects and split modules its closure holds, and keeps it from being - // run against an instruction other than the one it was built for. - cr.replace_fn = nullptr; + auto [pi, ri] = task.targets[i]; + assert(pi < cps.size()); + assert(ri < cps[pi].results.size()); + auto cr = i == last ? std::move(*task.result) : *task.result; cps[pi].results[ri] = compiled_result{std::move(cr), cps[pi].ins}; } } diff --git a/src/targets/gpu/compiled_code.cpp b/src/targets/gpu/compiled_code.cpp index 3d869e113f9..b301bffe277 100644 --- a/src/targets/gpu/compiled_code.cpp +++ b/src/targets/gpu/compiled_code.cpp @@ -23,8 +23,8 @@ */ #include #include +#include #include -#include #include namespace migraphx { @@ -35,21 +35,9 @@ std::string compiled_code::input_name(std::size_t i) { return param_name(i); } void compiled_code::replace(module& m, instruction_ref ins) const { - const auto* frag = fragment.get_main_module(); - const auto& inputs = ins->inputs(); - assert(frag->get_parameter_names().size() == inputs.size()); - // Seeding the map with the parameters keeps insert_instructions from copying them, so the - // fragment is wired directly onto the inputs of ins. - std::unordered_map map_ins; - for(auto i : range(inputs.size())) - { - auto param = frag->get_parameter(input_name(i)); - // get_parameter returns end() for an unknown name, which would splice a stray parameter - // into m instead of wiring the input. - assert(param != frag->end()); - map_ins[param] = inputs[i]; - } - auto returns = m.insert_instructions(ins, frag, &map_ins); + // insert_inline wires each fragment parameter onto the input in the same position, matching + // how input_name named them. + auto returns = m.insert_inline(ins, *fragment.get_main_module(), ins->inputs()); if(returns.size() != 1) MIGRAPHX_THROW("compiled_code: expected a single output from the fragment but got " + std::to_string(returns.size())); diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index 336f191615c..8066fab4866 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -87,8 +87,7 @@ struct MIGRAPHX_GPU_EXPORT binary_cache } }; - /// Counts of what the cache did. Only the tests ask for these, so a cache built without one - /// keeps no counters at all. + /// Counts of what the cache did. struct stats { /// Served from memory, so an earlier compile in this process was shared. @@ -101,20 +100,19 @@ struct MIGRAPHX_GPU_EXPORT binary_cache std::size_t compiled = 0; }; - explicit binary_cache(binary_cache_settings s = {}, std::shared_ptr counters = nullptr) - : settings(std::move(s)), st(std::move(counters)) - { - } + explicit binary_cache(binary_cache_settings s = {}) : settings(std::move(s)) {} /// Look up a key, consulting memory first and then the cache directory. optional get(const context& ctx, const std::string& key); /// Record a compiled result under its key. - void insert(const context& ctx, const entry& e); + void insert(const context& ctx, entry e); /// True when reused results should be checked against a fresh compile. bool verify() const; + const stats& get_stats() const { return counters; } + /// Names the directory holding entries that this build can use: the entry format, the /// compiler, a digest of the embedded kernel headers, and the rocMLIR build. Empty when the /// compiler cannot be identified, in which case nothing is read from or written to disk, @@ -122,17 +120,9 @@ struct MIGRAPHX_GPU_EXPORT binary_cache static const std::string& version_dir(); private: - /// Each records one outcome, and does nothing when no stats were supplied. record_miss - /// returns the empty result its callers hand back, so the outcome is recorded exactly where - /// the lookup gives up. - void record_reused(); - void record_hit(); - optional record_miss(); - void record_compiled(); - std::unordered_map memo; binary_cache_settings settings; - std::shared_ptr st; + stats counters; }; } // namespace gpu diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp index 72b538f54c2..43fe63171e7 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp @@ -69,7 +69,7 @@ struct hip_compiler_info /** * Ask the device compiler what version it is. * - * It is not the compiler this library was built with, so the answer comes from compiling a + * It need not be the compiler this library was built with, so the answer comes from compiling a * probe and reading back what it recorded. The result is determined once and reused; it is * empty if the compiler could not be asked. */ diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp index 48951486e90..0f3b208f09f 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp @@ -100,6 +100,9 @@ MIGRAPHX_GPU_EXPORT operation compile_hip_code_object(context& ctx, const std::string& content, hip_compile_options options); +/// Compile the kernel a hip_src describes. +MIGRAPHX_GPU_EXPORT operation compile_hip_code_object(context& ctx, hip_src src); + MIGRAPHX_GPU_EXPORT std::size_t compute_block_size(const context& ctx, std::size_t n, std::size_t max_block_size = 1024); diff --git a/src/targets/gpu/include/migraphx/gpu/context.hpp b/src/targets/gpu/include/migraphx/gpu/context.hpp index 918a17f1b20..cab865ee35b 100644 --- a/src/targets/gpu/include/migraphx/gpu/context.hpp +++ b/src/targets/gpu/include/migraphx/gpu/context.hpp @@ -473,7 +473,11 @@ struct context } /// Compiled kernels shared across every module compiled with this context. - binary_cache& get_binary_cache() { return *bc; } + binary_cache& get_binary_cache() + { + assert(bc != nullptr); + return *bc; + } private: // TODO: Make this a vector to support multiple devices diff --git a/src/targets/gpu/include/migraphx/gpu/mlir.hpp b/src/targets/gpu/include/migraphx/gpu/mlir.hpp index 89675064c60..e556121aa6b 100644 --- a/src/targets/gpu/include/migraphx/gpu/mlir.hpp +++ b/src/targets/gpu/include/migraphx/gpu/mlir.hpp @@ -63,7 +63,7 @@ MIGRAPHX_GPU_EXPORT mlir_code_object compile_mlir(const context& migraphx_ctx, /** * A string identifying the code object compile_mlir would produce for these arguments. * - * This is the mlir module as parsed, before any pipeline runs, so it is far cheaper than + * Built from the mlir module as parsed, before any pipeline runs, so it is far cheaper than * compiling. */ MIGRAPHX_GPU_EXPORT std::string mlir_compile_key(const context& migraphx_ctx, diff --git a/src/targets/gpu/jit/gather.cpp b/src/targets/gpu/jit/gather.cpp index 80d5341db0c..a8ffa53f848 100644 --- a/src/targets/gpu/jit/gather.cpp +++ b/src/targets/gpu/jit/gather.cpp @@ -80,8 +80,7 @@ struct gather_compiler : compiler operation compile_op(context& ctx, const std::vector& inputs, const value& v) const { - auto src = make_src(ctx, inputs, v); - return compile_hip_code_object(ctx, src.content, src.options); + return compile_hip_code_object(ctx, make_src(ctx, inputs, v)); } compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const diff --git a/src/targets/gpu/jit/pad.cpp b/src/targets/gpu/jit/pad.cpp index d67616b3fb4..1bb84902c60 100644 --- a/src/targets/gpu/jit/pad.cpp +++ b/src/targets/gpu/jit/pad.cpp @@ -123,8 +123,7 @@ struct pad_compiler : compiler operation compile_op(context& ctx, const std::vector& inputs, const value& v) const { - auto src = make_src(ctx, inputs, v); - return compile_hip_code_object(ctx, src.content, src.options); + return compile_hip_code_object(ctx, make_src(ctx, inputs, v)); } compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const diff --git a/src/targets/gpu/jit/pointwise.cpp b/src/targets/gpu/jit/pointwise.cpp index 9377471b389..87560d2ba07 100644 --- a/src/targets/gpu/jit/pointwise.cpp +++ b/src/targets/gpu/jit/pointwise.cpp @@ -105,8 +105,7 @@ struct pointwise_compiler : compiler operation compile_op(context& ctx, const std::vector& inputs, const value& v) const { - auto src = make_src(ctx, inputs, v); - return compile_hip_code_object(ctx, src.content, src.options); + return compile_hip_code_object(ctx, make_src(ctx, inputs, v)); } static value make_value(instruction_ref ins, const operation& op) diff --git a/src/targets/gpu/jit/reduce.cpp b/src/targets/gpu/jit/reduce.cpp index 46227da3fb3..e8a8a935747 100644 --- a/src/targets/gpu/jit/reduce.cpp +++ b/src/targets/gpu/jit/reduce.cpp @@ -416,8 +416,7 @@ struct simple_reduce_compiler : compiler operation compile_op(context& ctx, const std::vector& inputs, const value& v) const { - auto src = make_src(ctx, inputs, v); - return compile_hip_code_object(ctx, src.content, src.options); + return compile_hip_code_object(ctx, make_src(ctx, inputs, v)); } static value make_value(instruction_ref ins, const operation& op) @@ -636,8 +635,7 @@ struct fused_reduce_compiler : compiler operation compile_op(context& ctx, const std::vector& inputs, const value& v) const { - auto src = make_src(ctx, inputs, v); - return compile_hip_code_object(ctx, src.content, src.options); + return compile_hip_code_object(ctx, make_src(ctx, inputs, v)); } static value make_value(instruction_ref ins, const operation& op, const value& solution) diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index c364ad6a05b..640e786018a 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -83,38 +83,30 @@ static migraphx::gpu::binary_cache::entry make_entry(const std::string& key) return e; } -// A cache built without stats keeps no counters, so the tests that assert on counts supply one. -static std::shared_ptr make_stats() -{ - return std::make_shared(); -} - TEST_CASE(lookup_records_a_miss) { migraphx::gpu::context ctx; - auto st = make_stats(); - migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}, st}; + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}}; EXPECT(not cache.get(ctx, "absent").has_value()); - EXPECT(st->misses == 1); - EXPECT(st->hits == 0); - EXPECT(st->reused == 0); + EXPECT(cache.get_stats().misses == 1); + EXPECT(cache.get_stats().hits == 0); + EXPECT(cache.get_stats().reused == 0); } // An entry served out of memory is one an earlier compile in this process already paid for. TEST_CASE(memory_lookup_records_reuse) { migraphx::gpu::context ctx; - auto st = make_stats(); - migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}, st}; + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}}; cache.insert(ctx, make_entry("a-key")); - EXPECT(st->compiled == 1); + EXPECT(cache.get_stats().compiled == 1); auto found = cache.get(ctx, "a-key"); EXPECT(found.has_value()); - EXPECT(st->reused == 1); - EXPECT(st->misses == 0); + EXPECT(cache.get_stats().reused == 1); + EXPECT(cache.get_stats().misses == 0); } // A second cache shares nothing in memory, so anything it finds came off disk. @@ -127,13 +119,12 @@ TEST_CASE(disk_lookup_records_a_hit) migraphx::gpu::binary_cache writer{settings}; writer.insert(ctx, make_entry("shared-key")); - auto st = make_stats(); - migraphx::gpu::binary_cache reader{settings, st}; + migraphx::gpu::binary_cache reader{settings}; auto found = reader.get(ctx, "shared-key"); EXPECT(found.has_value()); - EXPECT(st->hits == 1); - EXPECT(st->misses == 0); + EXPECT(reader.get_stats().hits == 1); + EXPECT(reader.get_stats().misses == 0); EXPECT(*found->fragment.get_main_module() == *make_code().fragment.get_main_module()); } @@ -157,11 +148,10 @@ TEST_CASE(corrupt_entry_is_ignored) } EXPECT(truncated > 0); - auto st = make_stats(); - migraphx::gpu::binary_cache reader{settings, st}; + migraphx::gpu::binary_cache reader{settings}; EXPECT(not reader.get(ctx, "damaged").has_value()); - EXPECT(st->misses == 1); + EXPECT(reader.get_stats().misses == 1); } // Without a directory nothing reaches disk, though results are still shared in memory. @@ -169,12 +159,11 @@ TEST_CASE(no_directory_writes_nothing) { migraphx::tmp_dir td{"binary-cache"}; migraphx::gpu::context ctx; - auto st = make_stats(); - migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}, st}; + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}}; cache.insert(ctx, make_entry("in-memory-only")); EXPECT(cache.get(ctx, "in-memory-only").has_value()); - EXPECT(st->reused == 1); + EXPECT(cache.get_stats().reused == 1); EXPECT(migraphx::fs::is_empty(td.path)); } From 1bde05c831c484fc0e52bff99b5aa1b32514f571 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 14 Aug 2026 18:03:15 -0500 Subject: [PATCH 09/49] Add license --- src/targets/gpu/include/migraphx/gpu/compile_miopen.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/targets/gpu/include/migraphx/gpu/compile_miopen.hpp b/src/targets/gpu/include/migraphx/gpu/compile_miopen.hpp index ec58b77e40b..ddb422e7c8d 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_miopen.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_miopen.hpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal From 735d946dfb9e21a89c9f64788bc8ae391f988dae Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 14 Aug 2026 19:49:36 -0500 Subject: [PATCH 10/49] Fix cppcheck --- src/targets/gpu/compile_ops.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index cd06a92cd9e..0655cc22bc7 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -536,8 +536,13 @@ struct compile_plan { ctx->get_problem_cache().mark(preop.name(), problem); results.resize(solutions.size()); - for(std::size_t i : range(solutions.size())) - candidates.push_back({plan_index, i, solutions[i]}); + auto indices = range(solutions.size()); + std::transform(indices.begin(), + indices.end(), + std::back_inserter(candidates), + [&](std::size_t i) { + return compile_candidate{plan_index, i, solutions[i]}; + }); } } } From 7a5b11aa1ae816a87da008cf36857074dba5557e Mon Sep 17 00:00:00 2001 From: Paul Date: Sat, 15 Aug 2026 10:48:38 -0500 Subject: [PATCH 11/49] Hash all libraries with rocMLIR --- src/targets/gpu/CMakeLists.txt | 67 +++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/src/targets/gpu/CMakeLists.txt b/src/targets/gpu/CMakeLists.txt index b0ff0003dfa..7dd95e09fd6 100644 --- a/src/targets/gpu/CMakeLists.txt +++ b/src/targets/gpu/CMakeLists.txt @@ -25,24 +25,46 @@ find_package(hip REQUIRED) # Produce a short identity for a toolchain file, used to separate binary cache entries that -# different toolchains cannot share. The content hash is exact but slow on the larger libraries, -# so fall back to size and timestamp, which still changes whenever the file is replaced. +# different toolchains cannot share. function(migraphx_hash_file FILE_PATH OUT_VAR) if(NOT FILE_PATH OR NOT EXISTS "${FILE_PATH}") set(${OUT_VAR} "unknown" PARENT_SCOPE) return() endif() - file(SIZE "${FILE_PATH}" HASH_FILE_SIZE) - if(HASH_FILE_SIZE LESS 268435456) - file(MD5 "${FILE_PATH}" HASH_RESULT) - else() - file(TIMESTAMP "${FILE_PATH}" HASH_FILE_TIME "%s" UTC) - string(MD5 HASH_RESULT "${FILE_PATH}-${HASH_FILE_SIZE}-${HASH_FILE_TIME}") - endif() + file(MD5 "${FILE_PATH}" HASH_RESULT) string(SUBSTRING "${HASH_RESULT}" 0 12 HASH_RESULT) set(${OUT_VAR} "${HASH_RESULT}" PARENT_SCOPE) endfunction() +# Hash a list of library files, skipping entries that are not files (e.g. target names). +# CMake cannot stream one hash across several files, so hash the size and content hash +# of each file instead. +function(migraphx_hash_libraries LIB_PATHS OUT_VAR) + set(HASH_FILES "") + foreach(LIB_PATH IN LISTS LIB_PATHS) + if(EXISTS "${LIB_PATH}" AND NOT IS_DIRECTORY "${LIB_PATH}") + list(APPEND HASH_FILES "${LIB_PATH}") + endif() + endforeach() + list(LENGTH HASH_FILES HASH_FILE_COUNT) + if(HASH_FILE_COUNT EQUAL 0) + set(${OUT_VAR} "unknown" PARENT_SCOPE) + elseif(HASH_FILE_COUNT EQUAL 1) + migraphx_hash_file("${HASH_FILES}" HASH_RESULT) + set(${OUT_VAR} "${HASH_RESULT}" PARENT_SCOPE) + else() + set(HASH_INPUT "") + foreach(HASH_FILE IN LISTS HASH_FILES) + file(SIZE "${HASH_FILE}" HASH_FILE_SIZE) + file(MD5 "${HASH_FILE}" HASH_FILE_MD5) + string(APPEND HASH_INPUT "${HASH_FILE_SIZE}-${HASH_FILE_MD5};") + endforeach() + string(MD5 HASH_RESULT "${HASH_INPUT}") + string(SUBSTRING "${HASH_RESULT}" 0 12 HASH_RESULT) + set(${OUT_VAR} "${HASH_RESULT}" PARENT_SCOPE) + endif() +endfunction() + # MIGRAPHX_USE_AMDMLSS is resolved to a plain boolean at the top-level CMakeLists. if(MIGRAPHX_USE_AMDMLSS) message(STATUS "MIGraphX is using AMDMLSS") @@ -335,17 +357,28 @@ if(MIGRAPHX_ENABLE_MLIR) message(STATUS "Build with rocMLIR::rockCompiler ${rocMLIR_VERSION}") target_compile_definitions(migraphx_gpu PRIVATE "-DMIGRAPHX_MLIR") - # Identify the rocMLIR build for the binary cache by hashing the library that gets linked. + # Identify the rocMLIR build for the binary cache by hashing the libraries that get linked. # That covers a locally built or substituted rocMLIR, which a recorded version or commit # would not. - get_target_property(MIGRAPHX_ROCKCOMPILER_LIB rocMLIR::rockCompiler IMPORTED_LOCATION) - if(NOT MIGRAPHX_ROCKCOMPILER_LIB) - # rockCompiler is an interface target, so the archive is named as a link dependency. - get_target_property(MIGRAPHX_ROCKCOMPILER_LIB rocMLIR::rockCompiler INTERFACE_LINK_LIBRARIES) - list(FILTER MIGRAPHX_ROCKCOMPILER_LIB INCLUDE REGEX "rockCompiler") - list(GET MIGRAPHX_ROCKCOMPILER_LIB 0 MIGRAPHX_ROCKCOMPILER_LIB) + get_target_property(MIGRAPHX_ROCKCOMPILER_LIBS rocMLIR::rockCompiler IMPORTED_LOCATION) + if(NOT MIGRAPHX_ROCKCOMPILER_LIBS) + # rockCompiler is an interface target that names its archives as link dependencies: + # the fat librockCompiler.a on Linux, many smaller archives on Windows. + get_target_property(MIGRAPHX_ROCKCOMPILER_LIBS rocMLIR::rockCompiler + INTERFACE_LINK_LIBRARIES) endif() - migraphx_hash_file("${MIGRAPHX_ROCKCOMPILER_LIB}" MIGRAPHX_ROCMLIR_ID) + # Only rocMLIR's own archives identify the build, so skip the upstream LLVM/MLIR archives + # also linked on Windows. Match the file name alone since paths can contain these names. + set(MIGRAPHX_ROCMLIR_LIBS "") + foreach(MIGRAPHX_ROCKCOMPILER_LIB IN LISTS MIGRAPHX_ROCKCOMPILER_LIBS) + get_filename_component(MIGRAPHX_ROCKCOMPILER_LIB_NAME "${MIGRAPHX_ROCKCOMPILER_LIB}" + NAME) + if(MIGRAPHX_ROCKCOMPILER_LIB_NAME MATCHES + "rockCompiler|Rock|MLIRRoc|RocMLIR|Rocmlir|MIGraphX|MHAL") + list(APPEND MIGRAPHX_ROCMLIR_LIBS "${MIGRAPHX_ROCKCOMPILER_LIB}") + endif() + endforeach() + migraphx_hash_libraries("${MIGRAPHX_ROCMLIR_LIBS}" MIGRAPHX_ROCMLIR_ID) message(STATUS "rocMLIR binary cache id: ${MIGRAPHX_ROCMLIR_ID}") target_compile_definitions(migraphx_gpu PRIVATE "-DMIGRAPHX_ROCMLIR_ID=\"${MIGRAPHX_ROCMLIR_ID}\"") From 53c0e2c204a44c23ddff5f76900bb4edcfbca782 Mon Sep 17 00:00:00 2001 From: Paul Date: Sat, 15 Aug 2026 10:52:45 -0500 Subject: [PATCH 12/49] Use case-insensitive match --- src/targets/gpu/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/targets/gpu/CMakeLists.txt b/src/targets/gpu/CMakeLists.txt index 7dd95e09fd6..46adcd8c85c 100644 --- a/src/targets/gpu/CMakeLists.txt +++ b/src/targets/gpu/CMakeLists.txt @@ -373,8 +373,8 @@ if(MIGRAPHX_ENABLE_MLIR) foreach(MIGRAPHX_ROCKCOMPILER_LIB IN LISTS MIGRAPHX_ROCKCOMPILER_LIBS) get_filename_component(MIGRAPHX_ROCKCOMPILER_LIB_NAME "${MIGRAPHX_ROCKCOMPILER_LIB}" NAME) - if(MIGRAPHX_ROCKCOMPILER_LIB_NAME MATCHES - "rockCompiler|Rock|MLIRRoc|RocMLIR|Rocmlir|MIGraphX|MHAL") + string(TOLOWER "${MIGRAPHX_ROCKCOMPILER_LIB_NAME}" MIGRAPHX_ROCKCOMPILER_LIB_NAME) + if(MIGRAPHX_ROCKCOMPILER_LIB_NAME MATCHES "rock|roctarget|rocmlir|migraphx|mhal") list(APPEND MIGRAPHX_ROCMLIR_LIBS "${MIGRAPHX_ROCKCOMPILER_LIB}") endif() endforeach() From 754a4a422f1f450c123e62d710f66a107b57635f Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 25 Aug 2026 19:44:29 -0500 Subject: [PATCH 13/49] Add internal flags to the digest --- src/targets/gpu/compile_hip.cpp | 61 ++++++++----------- .../gpu/include/migraphx/gpu/compile_hip.hpp | 2 + 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/targets/gpu/compile_hip.cpp b/src/targets/gpu/compile_hip.cpp index a5a664ab6e5..562ecaf42f4 100644 --- a/src/targets/gpu/compile_hip.cpp +++ b/src/targets/gpu/compile_hip.cpp @@ -59,6 +59,29 @@ MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_DUMP_ASM); MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_DUMP_SRC); MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_HIP_FLAGS); +std::vector compile_hip_options(const std::vector& params, + const std::string& arch) +{ + auto options = params; + if(enabled(MIGRAPHX_GPU_DEBUG{})) + options.push_back("-DMIGRAPHX_DEBUG"); + if(std::none_of(options.begin(), options.end(), [](const std::string& s) { + return starts_with(s, "--std=") or starts_with(s, "-std="); + })) + options.push_back("-std=c++17"); + if(enabled(MIGRAPHX_GPU_DEBUG_SYM{})) + options.emplace_back("-g"); + options.push_back("-fno-gpu-rdc"); + options.push_back("-O" + string_value_of(MIGRAPHX_GPU_OPTIMIZE{}, "3")); + options.push_back("-Wno-cuda-compat"); + options.push_back("--offload-arch=" + arch); + std::vector extra_flags = + split_string(string_value_of(MIGRAPHX_GPU_HIP_FLAGS{}, ""), ' '); + options.insert(options.end(), extra_flags.begin(), extra_flags.end()); + + return options; +} + #ifdef MIGRAPHX_USE_HIPRTC namespace { @@ -207,21 +230,8 @@ std::vector> compile_hip_src_with_hiprtc(std::vector> compile_hip_src(const std::vector& srcs MIGRAPHX_THROW("Unknown hip compiler: " MIGRAPHX_HIP_COMPILER); src_compiler compiler; - compiler.flags = params; + compiler.flags = compile_hip_options(params, arch); compiler.compiler = MIGRAPHX_HIP_COMPILER; #ifdef MIGRAPHX_HIP_COMPILER_LAUNCHER if(has_compiler_launcher()) compiler.launcher = MIGRAPHX_HIP_COMPILER_LAUNCHER; #endif - if(std::none_of(params.begin(), params.end(), [](const std::string& s) { - return starts_with(s, "--std=") or starts_with(s, "-std="); - })) - compiler.flags.emplace_back("--std=c++17"); - compiler.flags.emplace_back(" -fno-gpu-rdc"); - if(enabled(MIGRAPHX_GPU_DEBUG_SYM{})) - compiler.flags.emplace_back("-g"); compiler.flags.emplace_back("-c"); - compiler.flags.emplace_back("--offload-arch=" + arch); - compiler.flags.emplace_back("--cuda-device-only"); - compiler.flags.emplace_back("-O" + string_value_of(MIGRAPHX_GPU_OPTIMIZE{}, "3") + " "); - - if(enabled(MIGRAPHX_GPU_DEBUG{})) - compiler.flags.emplace_back("-DMIGRAPHX_DEBUG"); - - compiler.flags.emplace_back("-Wno-unused-command-line-argument"); - compiler.flags.emplace_back("-Wno-cuda-compat"); compiler.flags.emplace_back(MIGRAPHX_HIP_COMPILER_FLAGS); - std::vector extra_flags = - split_string(string_value_of(MIGRAPHX_GPU_HIP_FLAGS{}, ""), ' '); - compiler.flags.insert(compiler.flags.end(), extra_flags.begin(), extra_flags.end()); if(enabled(MIGRAPHX_GPU_DUMP_SRC{})) { @@ -419,7 +410,7 @@ static hip_compiler_info parse_version(const std::vector& obj) auto fields = split_string(std::string{begin, end}, '|'); if(fields.size() != 3) return {}; - return {fields[0], fields[1], fields[2]}; + return {.major = fields[0], .minor = fields[1], .version = fields[2], .flags = compile_hip_options({}, version_probe_arch)}; } const hip_compiler_info& hip_compiler_version() diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp index 43fe63171e7..e3864be03ce 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp @@ -63,6 +63,8 @@ struct hip_compiler_info /// The full __clang_version__ string, which also names the source revision. std::string version = {}; + std::vector flags = {}; + bool empty() const { return version.empty(); } }; From 8d4da9280e0e86d3bc390940413d3070a5b7d0b5 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 26 Aug 2026 16:55:44 -0500 Subject: [PATCH 14/49] Add back compiler flags --- src/targets/gpu/compile_hip.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/targets/gpu/compile_hip.cpp b/src/targets/gpu/compile_hip.cpp index 562ecaf42f4..fafa264863e 100644 --- a/src/targets/gpu/compile_hip.cpp +++ b/src/targets/gpu/compile_hip.cpp @@ -339,6 +339,8 @@ std::vector> compile_hip_src(const std::vector& srcs #endif compiler.flags.emplace_back("-c"); + compiler.flags.emplace_back("--cuda-device-only"); + compiler.flags.emplace_back("-Wno-unused-command-line-argument"); compiler.flags.emplace_back(MIGRAPHX_HIP_COMPILER_FLAGS); if(enabled(MIGRAPHX_GPU_DUMP_SRC{})) From 74d95f2eef26ab573163e755265878858a1659e9 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 26 Aug 2026 17:07:29 -0500 Subject: [PATCH 15/49] Update key for internal compiler options --- src/targets/gpu/compile_hip_code_object.cpp | 1 + src/targets/gpu/include/migraphx/gpu/compile_hip.hpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/targets/gpu/compile_hip_code_object.cpp b/src/targets/gpu/compile_hip_code_object.cpp index b517e8221c2..940b5619425 100644 --- a/src/targets/gpu/compile_hip_code_object.cpp +++ b/src/targets/gpu/compile_hip_code_object.cpp @@ -253,6 +253,7 @@ std::string hip_compile_key(const context& ctx, const hip_src& src) src.options.virtual_inputs.empty()); auto options = src.options; add_derived_params(ctx, options); + options.params = compile_hip_options(options.params, ctx.get_current_device().get_device_name()); std::stringstream ss; ss << "arch=" << ctx.get_current_device().get_device_name() << "\n"; diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp index e3864be03ce..793494df7d3 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp @@ -68,6 +68,9 @@ struct hip_compiler_info bool empty() const { return version.empty(); } }; +std::vector compile_hip_options(const std::vector& params, + const std::string& arch); + /** * Ask the device compiler what version it is. * From 418b43fc55e15edbf4b06bac87ff08fec9fef49b Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 26 Aug 2026 17:36:39 -0500 Subject: [PATCH 16/49] Add unit tests for in-memory cache --- test/gpu/binary_cache.cpp | 47 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 640e786018a..d506d7c745b 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -37,7 +37,13 @@ #include #include #include +#include +#include +#include +#include +#include #include +#include #include static migraphx::program pointwise_program() @@ -167,6 +173,47 @@ TEST_CASE(no_directory_writes_nothing) EXPECT(migraphx::fs::is_empty(td.path)); } +static migraphx::program two_identical_pointwise() +{ + migraphx::program p; + auto* mm = p.get_main_module(); + migraphx::shape s{migraphx::shape::float_type, {4, 8}}; + auto x = mm->add_parameter("x", s); + auto y = mm->add_parameter("y", s); + auto pw1 = add_pointwise(p, "main:pointwise0", {x, y}, single_pointwise("add")); + auto pw2 = add_pointwise(p, "main:pointwise1", {y, x}, single_pointwise("add")); + mm->add_return({pw1, pw2}); + return p; +} + +static std::size_t count_code_objects(const migraphx::module& m) +{ + return std::count_if( + m.begin(), m.end(), [](const auto& ins) { return ins.name() == "gpu::code_object"; }); +} + +// The in-memory cache needs no directory: two identical kernels in one model compile once, and +// a later module compiled with the same context reuses the result without compiling at all. +TEST_CASE(duplicate_kernels_compile_once_without_a_directory) +{ + auto cache = std::make_shared( + migraphx::gpu::binary_cache_settings{.path = ""}); + migraphx::gpu::context ctx{0, 1, cache}; + + auto p1 = two_identical_pointwise(); + migraphx::run_passes(*p1.get_main_module(), + {migraphx::gpu::lowering{&ctx, false}, migraphx::gpu::compile_ops{&ctx}}); + EXPECT(count_code_objects(*p1.get_main_module()) == 2); + EXPECT(cache->get_stats().compiled == 1); + + auto p2 = two_identical_pointwise(); + migraphx::run_passes(*p2.get_main_module(), + {migraphx::gpu::lowering{&ctx, false}, migraphx::gpu::compile_ops{&ctx}}); + EXPECT(count_code_objects(*p2.get_main_module()) == 2); + EXPECT(cache->get_stats().compiled == 1); + EXPECT(cache->get_stats().reused == 1); +} + // Compiling twice against the same directory has to leave entries behind and keep producing the // same numbers as the reference, whichever half of the run they came from. TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) From a134d9eb693ecd7bb9977f5d2a4a5c4b4a76bd55 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 10:51:06 -0500 Subject: [PATCH 17/49] Refactor compile_ops --- src/targets/gpu/compile_ops.cpp | 308 ++++++++++++++------------------ 1 file changed, 136 insertions(+), 172 deletions(-) diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 0655cc22bc7..1d7ba7713a7 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -50,6 +50,7 @@ #include #include #include +#include namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { @@ -382,55 +383,29 @@ struct dynamic_code_object_op }; MIGRAPHX_REGISTER_OP(dynamic_code_object_op); -struct compiled_result -{ - compiler_replace replace; - instruction_ref ins; - - friend std::ostream& operator<<(std::ostream& os, const compiled_result& cr) - { - cr.replace.trace(os, cr.ins); - return os; - } - - program make_program() const - { - program bench_prog; - auto* mm = bench_prog.get_main_module(); - - std::vector bench_ins_inputs; - std::transform(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(bench_ins_inputs), - [&](const auto& arg) { - return mm->add_parameter(std::to_string(bench_ins_inputs.size()), - arg->get_shape()); - }); - auto bench_ins = - mm->add_instruction(ins->get_operator(), bench_ins_inputs, ins->module_inputs()); - mm->add_return({bench_ins}); - replace.replace(*mm, bench_ins); - run_passes(*mm, - { - eliminate_identity{}, - dead_code_elimination{}, - memory_coloring{"hip::allocate"}, - }); - return bench_prog; - } -}; - // forward declared since it requires compile_manager static void replace_inserted_device_ops(context& ctx, module& m); -/// One compilation a plan is waiting on. Candidates are collected before anything is compiled, -/// so identical ones can be found and compiled only once. -struct compile_candidate +/** + * One compilation and its result, shared by every result slot whose key matches. + * + * Duplicate compiles are grouped by pointing the slots of every matching plan at the same cell, + * so writing the result once puts it in place everywhere. During the parallel compile each cell + * is written by exactly one worker; the plans sharing it read it only after the workers join. + */ +struct compile_cell { - std::size_t plan_index = 0; - std::size_t result_index = 0; - value solution = {}; - std::string key = {}; + explicit compile_cell(value s) : solution(std::move(s)) {} + + value solution = {}; + /// Identifies the code the compile would produce. Empty when the compiler cannot say, which + /// keeps the cell out of the cache and out of the grouping. + std::string key = {}; + optional result = nullopt; + /// Set when the result came from the cache, so it is not stored back afterwards. + bool reused = false; + /// A failure raised while verifying the reused result, rethrown after the workers join. + std::exception_ptr error = nullptr; }; struct compile_plan @@ -439,8 +414,10 @@ struct compile_plan operation preop; instruction_ref ins; module_ref mod; - optional config = nullopt; - std::vector> results = {}; + optional config = nullopt; + /// One slot per candidate solution, aligned with config->solutions when benchmarking. + /// Slots of different plans point at the same cell when their keys match. + std::vector> results = {}; void update_config(bool exhaustive) { config = get_tuning_config(*ctx, ins, preop, exhaustive); @@ -501,10 +478,37 @@ struct compile_plan } } - void add_candidates(std::vector& candidates, - std::size_t plan_index, - bool skip_benchmark) + /// A standalone program with just this instruction and the compiled code objects inserted, + /// used for benchmarking a result and for dumping it. + program make_program(const compiler_replace& cr) const + { + program bench_prog; + auto* mm = bench_prog.get_main_module(); + + std::vector bench_ins_inputs; + std::transform(ins->inputs().begin(), + ins->inputs().end(), + std::back_inserter(bench_ins_inputs), + [&](const auto& arg) { + return mm->add_parameter(std::to_string(bench_ins_inputs.size()), + arg->get_shape()); + }); + auto bench_ins = + mm->add_instruction(ins->get_operator(), bench_ins_inputs, ins->module_inputs()); + mm->add_return({bench_ins}); + cr.replace(*mm, bench_ins); + run_passes(*mm, + { + eliminate_identity{}, + dead_code_elimination{}, + memory_coloring{"hip::allocate"}, + }); + return bench_prog; + } + + void add_cells(bool skip_benchmark) { + assert(results.empty()); if(config.has_value()) { const auto& problem = config->problem; @@ -514,8 +518,7 @@ struct compile_plan // No solution yet until benchmarked so skip for now if(solution.is_null()) return; - results.resize(1); - candidates.push_back({plan_index, 0, solution}); + results.push_back(std::make_shared(solution)); } else { @@ -529,27 +532,23 @@ struct compile_plan (ctx->is_cross_compile() and not dump_mxr) or solutions.size() == 1) { ctx->get_problem_cache().insert(preop.name(), problem, solutions.front()); - results.resize(1); - candidates.push_back({plan_index, 0, solutions.front()}); + results.push_back(std::make_shared(solutions.front())); } else { ctx->get_problem_cache().mark(preop.name(), problem); - results.resize(solutions.size()); - auto indices = range(solutions.size()); - std::transform(indices.begin(), - indices.end(), - std::back_inserter(candidates), - [&](std::size_t i) { - return compile_candidate{plan_index, i, solutions[i]}; + std::transform(solutions.begin(), + solutions.end(), + std::back_inserter(results), + [](const value& solution) { + return std::make_shared(solution); }); } } } else { - results.resize(1); - candidates.push_back({plan_index, 0, value{}}); + results.push_back(std::make_shared(value{})); } } std::string problem_string() const @@ -588,7 +587,7 @@ struct compile_plan return input_shapes.str(); } - const compiled_result& benchmark() const + const compiler_replace& benchmark() const { const auto trace_level = value_of(MIGRAPHX_TRACE_BENCHMARKING{}); if(trace_level > 0 and not results.empty()) @@ -601,10 +600,10 @@ struct compile_plan problem_string() + "\n\n" + print_modules()); if(results.size() == 1) { - if(not results.front().has_value()) + if(not results.front()->result.has_value()) MIGRAPHX_THROW("No valid tuned compilation for " + preop.name() + " with " + problem_string() + "\n\n" + print_modules()); - return *results.front(); + return *results.front()->result; } if(not config) MIGRAPHX_THROW("Multiple kernels without config for " + preop.name()); @@ -616,29 +615,32 @@ struct compile_plan results.end(), config->solutions.begin(), std::back_inserter(times), - [&](const auto& cr, const auto& solution) { + [&](const auto& cell, const auto& solution) { if(trace_level > 1) std::cout << "Benchmarking solution: " << solution << std::endl; - if(not cr.has_value()) + if(not cell->result.has_value()) { if(trace_level > 1) std::cout << "No binary" << std::endl; return std::numeric_limits::max(); } if(trace_level > 2) - std::cout << *cr << std::endl; + { + cell->result->trace(std::cout, ins); + std::cout << std::endl; + } /* create a small program with insturction being compiled and call "replace" on that which would insert all the compiled code objects, prefills etc. necessary to run candidate code object */ - auto bench_prog = cr->make_program(); + auto bench_prog = make_program(*cell->result); if(trace_level > 2) std::cout << bench_prog << std::endl; auto bundle = benchmark_bundle(*bench_prog.get_main_module()); auto t = time_program(*ctx, std::move(bench_prog), - cr->replace.code.fill_map, + cell->result->code.fill_map, bundle, /* nrun */ 20); if(trace_level > 1) @@ -653,22 +655,19 @@ struct compile_plan std::cout << "Fastest solution: " << config->solutions.at(i) << std::endl; ctx->get_problem_cache().save(); } - if(not results[i].has_value()) + if(not results[i]->result.has_value()) MIGRAPHX_THROW("No valid tuned compilation for " + preop.name() + " with " + problem_string() + "\n\n" + print_modules()); - auto skipped = std::count_if( - results.begin(), results.end(), [](const auto& cr) { return not cr.has_value(); }); + auto skipped = std::count_if(results.begin(), results.end(), [](const auto& cell) { + return not cell->result.has_value(); + }); if(skipped > 0) log::info() << "Skipped " << skipped << " configs for " << preop.name(); - return *results[i]; + return *results[i]->result; } - void replace(module& m) const - { - const auto& cr = benchmark(); - cr.replace.replace(m, cr.ins); - } + void replace(module& m) const { benchmark().replace(m, ins); } void save_binaries(const fs::path& mxr_dir) const { @@ -676,10 +675,10 @@ struct compile_plan return; for(auto i : range(results.size())) { - if(not results[i].has_value()) + if(not results[i]->result.has_value()) continue; const auto& solution = config->solutions[i]; - auto bench_prog = results[i]->make_program(); + auto bench_prog = make_program(*results[i]->result); auto* mm = bench_prog.get_main_module(); replace_inserted_device_ops(*ctx, *mm); @@ -713,27 +712,6 @@ static void par_compile(std::size_t n, F f) par_for(n, n / d, f); } -/// A compile that one or more plans are waiting on, along with the places its result goes. -struct compile_task -{ - value solution = {}; - std::string key = {}; - /// Where the result goes, as (plan, result) slots. The first one also drives the compile, - /// since every target shares this key and therefore compiles to the same code. - std::vector> targets = {}; - optional result = nullopt; - /// Set when the result came from the cache, so it is not stored back afterwards. - bool reused = false; - /// A failure raised while verifying the reused result, rethrown after the workers join. - std::exception_ptr error = nullptr; - - std::size_t plan_index() const - { - assert(not targets.empty()); - return targets.front().first; - } -}; - struct compile_manager { std::vector cps; @@ -751,111 +729,97 @@ struct compile_manager par_compile(cps.size(), [&](auto i) { cps[i].update_config(exhaustive); }); } - /** - * Group the candidates that would compile to the same code. - * - * Since every key is known before any compile starts, each group can be handed to exactly - * one task and the results shared afterwards, so the compiles never have to coordinate. - */ - static std::vector make_tasks(const std::vector& candidates) + void compile(module& m, bool is_root) { - std::vector tasks; - // The keys are whole kernel sources, so the index views them instead of copying; the - // index does not outlive this function. - std::unordered_map task_index; - for(const auto& c : candidates) + for(auto& cp : cps) + cp.add_cells(skip_benchmark); + { - std::size_t i = tasks.size(); - if(c.key.empty()) + // Every slot is collected so the keys can be computed in parallel. + std::vector> slots; + for(auto& cp : cps) { - tasks.push_back({c.solution, c.key}); + std::transform(cp.results.begin(), + cp.results.end(), + std::back_inserter(slots), + [&](const auto& cell) { return std::make_pair(&cp, cell.get()); }); } - else - { - auto [it, inserted] = task_index.emplace(c.key, i); - if(inserted) - tasks.push_back({c.solution, c.key}); - i = it->second; - } - tasks[i].targets.emplace_back(c.plan_index, c.result_index); + par_compile(slots.size(), [&](auto i) { + auto [cp, cell] = slots[i]; + cell->key = cp->get_key(cell->solution); + }); } - return tasks; - } - void compile(module& m, bool is_root) - { - std::vector candidates; - for(auto i : range(cps.size())) + // Group the slots that would compile to the same code by pointing them at one shared + // cell; since every key is known before any compile starts, each cell can be handed to + // exactly one task and the compiles never have to coordinate. The plan kept with the + // cell drives the compile, which any of the sharers could do since they compile to the + // same code. + std::vector>> tasks; + // The keys are whole kernel sources, so the index views the canonical cell's key + // instead of copying; the index does not outlive this function. + std::unordered_map> index; + for(auto& cp : cps) { - cps[i].add_candidates(candidates, i, skip_benchmark); + for(auto& cell : cp.results) + { + if(not cell->key.empty()) + { + auto [it, inserted] = index.emplace(cell->key, cell); + if(not inserted) + { + cell = it->second; + continue; + } + } + tasks.emplace_back(&cp, cell); + } } - par_compile(candidates.size(), [&](auto i) { - candidates[i].key = cps[candidates[i].plan_index].get_key(candidates[i].solution); - }); - auto tasks = make_tasks(candidates); - assert(tasks.size() <= candidates.size()); // The cache is unguarded, so everything it can answer is looked up before any compile // starts, and everything compiled is stored after they all finish. - for(auto& task : tasks) + for(const auto& [cp, cell] : tasks) { - task.result = cps[task.plan_index()].lookup(task.key); - task.reused = task.result.has_value(); + cell->result = cp->lookup(cell->key); + cell->reused = cell->result.has_value(); } par_compile(tasks.size(), [&](auto i) { - auto& task = tasks[i]; - const auto& plan = cps[task.plan_index()]; - if(task.reused) + const auto& [cp, cell] = tasks[i]; + if(cell->reused) { // A throw would escape the worker thread and terminate the process, so carry // the failure back and rethrow it after the workers join. try { - plan.verify(task.solution, task.result->code); + cp->verify(cell->solution, cell->result->code); } catch(...) { - task.error = std::current_exception(); + cell->error = std::current_exception(); } } else { - task.result = plan.run_compile(task.solution); + cell->result = cp->run_compile(cell->solution); } }); - auto failed = std::find_if( - tasks.begin(), tasks.end(), [](const auto& task) { return task.error != nullptr; }); + auto failed = std::find_if(tasks.begin(), tasks.end(), [](const auto& task) { + return task.second->error != nullptr; + }); if(failed != tasks.end()) - std::rethrow_exception(failed->error); + std::rethrow_exception(failed->second->error); - for(const auto& task : tasks) + for(const auto& [cp, cell] : tasks) { - if(task.reused or not task.result.has_value()) + if(cell->reused or not cell->result.has_value()) continue; - cps[task.plan_index()].store(task.solution, task.key, task.result->code); - } - - for(auto& task : tasks) - { - // The slots start out empty, so a task that failed to compile needs no write. - if(not task.result.has_value()) - continue; - assert(not task.result->code.empty()); + cp->store(cell->solution, cell->key, cell->result->code); + assert(not cell->result->code.empty()); // Only the serializable code is used from here on. Dropping the replace function // releases the code objects and split modules its closure holds, and keeps it from // being run against an instruction other than the one it was built for. - task.result->replace_fn = nullptr; - // Every target gets its own copy; the last takes the result itself. - const auto last = task.targets.size() - 1; - for(auto i : range(task.targets.size())) - { - auto [pi, ri] = task.targets[i]; - assert(pi < cps.size()); - assert(ri < cps[pi].results.size()); - auto cr = i == last ? std::move(*task.result) : *task.result; - cps[pi].results[ri] = compiled_result{std::move(cr), cps[pi].ins}; - } + cell->result->replace_fn = nullptr; } static const auto mxr_path = string_value_of(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR{}); From cac0edd85760f7bcc8fb714f8697952e84f91a2a Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 11:03:01 -0500 Subject: [PATCH 18/49] Skip tasks that dont need compile --- src/targets/gpu/compile_ops.cpp | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 1d7ba7713a7..8a70fb48a4e 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -438,12 +438,13 @@ struct compile_plan cache_store(*ctx, preop, solution, key, config ? config->problem : value{}, code); } + /// True when the cache was configured to check reused results against a fresh compile. + bool verify_enabled() const { return ctx->get_binary_cache().verify(); } + /// Compile again and compare, so a key that fails to describe its result fails loudly. - /// Does nothing unless the cache was configured to verify reuse. + /// Only called when verify_enabled. void verify(const value& solution, const compiled_code& reused) const { - if(not ctx->get_binary_cache().verify()) - return; verify_reuse(*ctx, ins, preop, solution, reused); } @@ -751,10 +752,13 @@ struct compile_manager } // Group the slots that would compile to the same code by pointing them at one shared - // cell; since every key is known before any compile starts, each cell can be handed to - // exactly one task and the compiles never have to coordinate. The plan kept with the - // cell drives the compile, which any of the sharers could do since they compile to the - // same code. + // cell, and look each unique cell up in the cache as it is first seen; the cache is + // unguarded, so everything it can answer is looked up here before any compile starts, + // and everything compiled is stored after they all finish. Only the cells that still + // need parallel work, a compile or a verify of a reused result, become tasks; since + // every key is known before any compile starts, each cell is handed to exactly one task + // and the compiles never have to coordinate. The plan kept with the cell drives the + // compile, which any of the sharers could do since they compile to the same code. std::vector>> tasks; // The keys are whole kernel sources, so the index views the canonical cell's key // instead of copying; the index does not outlive this function. @@ -772,18 +776,14 @@ struct compile_manager continue; } } + cell->result = cp.lookup(cell->key); + cell->reused = cell->result.has_value(); + if(cell->reused and not cp.verify_enabled()) + continue; tasks.emplace_back(&cp, cell); } } - // The cache is unguarded, so everything it can answer is looked up before any compile - // starts, and everything compiled is stored after they all finish. - for(const auto& [cp, cell] : tasks) - { - cell->result = cp->lookup(cell->key); - cell->reused = cell->result.has_value(); - } - par_compile(tasks.size(), [&](auto i) { const auto& [cp, cell] = tasks[i]; if(cell->reused) From aacdc979fe0c11bba8df8a00b1cdb267d32cd836 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 11:38:59 -0500 Subject: [PATCH 19/49] Seperate the loops --- src/targets/gpu/compile_ops.cpp | 56 +++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 8a70fb48a4e..f0a79551bf0 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -752,38 +752,48 @@ struct compile_manager } // Group the slots that would compile to the same code by pointing them at one shared - // cell, and look each unique cell up in the cache as it is first seen; the cache is - // unguarded, so everything it can answer is looked up here before any compile starts, - // and everything compiled is stored after they all finish. Only the cells that still - // need parallel work, a compile or a verify of a reused result, become tasks; since - // every key is known before any compile starts, each cell is handed to exactly one task - // and the compiles never have to coordinate. The plan kept with the cell drives the - // compile, which any of the sharers could do since they compile to the same code. - std::vector>> tasks; - // The keys are whole kernel sources, so the index views the canonical cell's key - // instead of copying; the index does not outlive this function. - std::unordered_map> index; - for(auto& cp : cps) + // cell; writing that cell's result once then puts it in place for every plan that + // shares it. The plan kept with the cell drives the compile, which any of the sharers + // could do since they compile to the same code. + std::vector>> cells; { - for(auto& cell : cp.results) + // The keys are whole kernel sources, so the index views the canonical cell's key + // instead of copying; the index does not outlive this block. + std::unordered_map> index; + for(auto& cp : cps) { - if(not cell->key.empty()) + for(auto& cell : cp.results) { - auto [it, inserted] = index.emplace(cell->key, cell); - if(not inserted) + if(not cell->key.empty()) { - cell = it->second; - continue; + auto [it, inserted] = index.emplace(cell->key, cell); + if(not inserted) + { + cell = it->second; + continue; + } } + cells.emplace_back(&cp, cell); } - cell->result = cp.lookup(cell->key); - cell->reused = cell->result.has_value(); - if(cell->reused and not cp.verify_enabled()) - continue; - tasks.emplace_back(&cp, cell); } } + // The cache is unguarded, so everything it can answer is looked up before any compile + // starts, and everything compiled is stored after they all finish. + for(const auto& [cp, cell] : cells) + { + cell->result = cp->lookup(cell->key); + cell->reused = cell->result.has_value(); + } + + // Only the cells that still need parallel work, a compile or a verify of a reused + // result, become tasks; since every key is known before any compile starts, each cell + // is handed to exactly one task and the compiles never have to coordinate. + std::vector>> tasks; + std::copy_if(cells.begin(), cells.end(), std::back_inserter(tasks), [](const auto& task) { + return not task.second->reused or task.first->verify_enabled(); + }); + par_compile(tasks.size(), [&](auto i) { const auto& [cp, cell] = tasks[i]; if(cell->reused) From 4ada6a49dae4c468b6abe2fe473e99c8412facea Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 11:57:48 -0500 Subject: [PATCH 20/49] Get rid of extra vector --- src/targets/gpu/compile_ops.cpp | 66 +++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index f0a79551bf0..0c83ff4f808 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -175,10 +175,19 @@ static void verify_reuse(context& ctx, to_string(fresh.code.fragment)); } +/// Marks keys invented for grouping only, given to cells whose compiler cannot describe its +/// output so they are never grouped with another cell. The cache must ignore them: they are +/// numbered by slot, so the same key names different code from one compile batch to the next. +static constexpr const char* private_key_prefix = "__migraphx_private_unique_key"; + +static bool is_private_key(const std::string& key) { return starts_with(key, private_key_prefix); } + /// Look for an earlier result. The cache is not guarded, so callers must not do this while /// compiles are running. static optional cache_lookup(context& ctx, const std::string& key) { + if(is_private_key(key)) + return nullopt; auto cached = ctx.get_binary_cache().get(ctx, key); if(not cached.has_value()) return nullopt; @@ -195,6 +204,8 @@ static void cache_store(context& ctx, const value& problem, const compiled_code& code) { + if(is_private_key(key)) + return; binary_cache::entry e; e.key = key; e.op_name = preop.name(); @@ -398,8 +409,8 @@ struct compile_cell explicit compile_cell(value s) : solution(std::move(s)) {} value solution = {}; - /// Identifies the code the compile would produce. Empty when the compiler cannot say, which - /// keeps the cell out of the cache and out of the grouping. + /// Identifies the code the compile would produce. When the compiler cannot say, a private + /// key is invented instead, which keeps the cell out of the cache and out of the grouping. std::string key = {}; optional result = nullopt; /// Set when the result came from the cache, so it is not stored back afterwards. @@ -748,51 +759,52 @@ struct compile_manager par_compile(slots.size(), [&](auto i) { auto [cp, cell] = slots[i]; cell->key = cp->get_key(cell->solution); + if(cell->key.empty()) + cell->key = private_key_prefix + std::to_string(i); }); } // Group the slots that would compile to the same code by pointing them at one shared // cell; writing that cell's result once then puts it in place for every plan that // shares it. The plan kept with the cell drives the compile, which any of the sharers - // could do since they compile to the same code. - std::vector>> cells; + // could do since they compile to the same code. The keys are whole kernel sources, so + // the index views the canonical cell's key instead of copying; it does not outlive + // this function. + std::unordered_map>> + index; + for(auto& cp : cps) { - // The keys are whole kernel sources, so the index views the canonical cell's key - // instead of copying; the index does not outlive this block. - std::unordered_map> index; - for(auto& cp : cps) + for(auto& cell : cp.results) { - for(auto& cell : cp.results) - { - if(not cell->key.empty()) - { - auto [it, inserted] = index.emplace(cell->key, cell); - if(not inserted) - { - cell = it->second; - continue; - } - } - cells.emplace_back(&cp, cell); - } + auto [it, inserted] = index.emplace(cell->key, std::make_pair(&cp, cell)); + if(not inserted) + cell = it->second.second; } } // The cache is unguarded, so everything it can answer is looked up before any compile // starts, and everything compiled is stored after they all finish. - for(const auto& [cp, cell] : cells) + for(const auto& entry : index) { - cell->result = cp->lookup(cell->key); - cell->reused = cell->result.has_value(); + const auto& [cp, cell] = entry.second; + cell->result = cp->lookup(cell->key); + cell->reused = cell->result.has_value(); } // Only the cells that still need parallel work, a compile or a verify of a reused // result, become tasks; since every key is known before any compile starts, each cell // is handed to exactly one task and the compiles never have to coordinate. std::vector>> tasks; - std::copy_if(cells.begin(), cells.end(), std::back_inserter(tasks), [](const auto& task) { - return not task.second->reused or task.first->verify_enabled(); - }); + transform_if( + index.begin(), + index.end(), + std::back_inserter(tasks), + [](const auto& entry) { + const auto& [cp, cell] = entry.second; + return not cell->reused or cp->verify_enabled(); + }, + [](const auto& entry) { return entry.second; }); par_compile(tasks.size(), [&](auto i) { const auto& [cp, cell] = tasks[i]; From d58f5ea4cac47e1a7a6c2aec99b3cfcafd6c5fc9 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 12:02:10 -0500 Subject: [PATCH 21/49] Remove error marshalling --- src/targets/gpu/compile_ops.cpp | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 0c83ff4f808..f6ee2739daa 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -415,8 +415,6 @@ struct compile_cell optional result = nullopt; /// Set when the result came from the cache, so it is not stored back afterwards. bool reused = false; - /// A failure raised while verifying the reused result, rethrown after the workers join. - std::exception_ptr error = nullptr; }; struct compile_plan @@ -809,28 +807,10 @@ struct compile_manager par_compile(tasks.size(), [&](auto i) { const auto& [cp, cell] = tasks[i]; if(cell->reused) - { - // A throw would escape the worker thread and terminate the process, so carry - // the failure back and rethrow it after the workers join. - try - { - cp->verify(cell->solution, cell->result->code); - } - catch(...) - { - cell->error = std::current_exception(); - } - } + cp->verify(cell->solution, cell->result->code); else - { cell->result = cp->run_compile(cell->solution); - } - }); - auto failed = std::find_if(tasks.begin(), tasks.end(), [](const auto& task) { - return task.second->error != nullptr; }); - if(failed != tasks.end()) - std::rethrow_exception(failed->second->error); for(const auto& [cp, cell] : tasks) { From fa41b247dbd6cc8aed20b44713d69a43260756eb Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 12:05:41 -0500 Subject: [PATCH 22/49] Remove the reused flag --- src/targets/gpu/compile_ops.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index f6ee2739daa..2a0f76a5fc3 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -413,8 +413,6 @@ struct compile_cell /// key is invented instead, which keeps the cell out of the cache and out of the grouping. std::string key = {}; optional result = nullopt; - /// Set when the result came from the cache, so it is not stored back afterwards. - bool reused = false; }; struct compile_plan @@ -782,12 +780,12 @@ struct compile_manager } // The cache is unguarded, so everything it can answer is looked up before any compile - // starts, and everything compiled is stored after they all finish. + // starts, and everything compiled is stored after they all finish. Until the compiles + // run below, a cell holding a result is one whose result was reused from the cache. for(const auto& entry : index) { const auto& [cp, cell] = entry.second; cell->result = cp->lookup(cell->key); - cell->reused = cell->result.has_value(); } // Only the cells that still need parallel work, a compile or a verify of a reused @@ -800,13 +798,13 @@ struct compile_manager std::back_inserter(tasks), [](const auto& entry) { const auto& [cp, cell] = entry.second; - return not cell->reused or cp->verify_enabled(); + return not cell->result.has_value() or cp->verify_enabled(); }, [](const auto& entry) { return entry.second; }); par_compile(tasks.size(), [&](auto i) { const auto& [cp, cell] = tasks[i]; - if(cell->reused) + if(cell->result.has_value()) cp->verify(cell->solution, cell->result->code); else cell->result = cp->run_compile(cell->solution); @@ -814,8 +812,10 @@ struct compile_manager for(const auto& [cp, cell] : tasks) { - if(cell->reused or not cell->result.has_value()) + if(not cell->result.has_value()) continue; + // When verifying, this also stores reused results back, which rewrites the same + // bytes under the same key and is harmless. cp->store(cell->solution, cell->key, cell->result->code); assert(not cell->result->code.empty()); // Only the serializable code is used from here on. Dropping the replace function From 579920b8ac0516e373766df762e2a854610382f1 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 14:10:53 -0500 Subject: [PATCH 23/49] Simplify --- src/targets/gpu/CMakeLists.txt | 44 +++------ src/targets/gpu/binary_cache.cpp | 89 +++++++++++-------- src/targets/gpu/compile_hip.cpp | 2 +- src/targets/gpu/compile_hip_code_object.cpp | 27 +++--- src/targets/gpu/compile_ops.cpp | 17 ++-- .../gpu/include/migraphx/gpu/binary_cache.hpp | 5 +- .../gpu/include/migraphx/gpu/compile_hip.hpp | 2 - .../gpu/include/migraphx/gpu/context.hpp | 18 ++-- src/targets/gpu/target.cpp | 19 ++-- test/gpu/binary_cache.cpp | 3 +- 10 files changed, 114 insertions(+), 112 deletions(-) diff --git a/src/targets/gpu/CMakeLists.txt b/src/targets/gpu/CMakeLists.txt index 46adcd8c85c..7cc43e625b3 100644 --- a/src/targets/gpu/CMakeLists.txt +++ b/src/targets/gpu/CMakeLists.txt @@ -24,45 +24,25 @@ find_package(hip REQUIRED) -# Produce a short identity for a toolchain file, used to separate binary cache entries that -# different toolchains cannot share. -function(migraphx_hash_file FILE_PATH OUT_VAR) - if(NOT FILE_PATH OR NOT EXISTS "${FILE_PATH}") - set(${OUT_VAR} "unknown" PARENT_SCOPE) - return() - endif() - file(MD5 "${FILE_PATH}" HASH_RESULT) - string(SUBSTRING "${HASH_RESULT}" 0 12 HASH_RESULT) - set(${OUT_VAR} "${HASH_RESULT}" PARENT_SCOPE) -endfunction() - -# Hash a list of library files, skipping entries that are not files (e.g. target names). -# CMake cannot stream one hash across several files, so hash the size and content hash -# of each file instead. +# Produce a short identity for a list of toolchain library files, used to separate binary cache +# entries that different toolchains cannot share. Entries that are not files (e.g. target names) +# are skipped; CMake cannot stream one hash across several files, so hash the per-file content +# hashes instead. function(migraphx_hash_libraries LIB_PATHS OUT_VAR) - set(HASH_FILES "") + set(HASH_INPUT "") foreach(LIB_PATH IN LISTS LIB_PATHS) if(EXISTS "${LIB_PATH}" AND NOT IS_DIRECTORY "${LIB_PATH}") - list(APPEND HASH_FILES "${LIB_PATH}") + file(MD5 "${LIB_PATH}" HASH_FILE_MD5) + string(APPEND HASH_INPUT "${HASH_FILE_MD5};") endif() endforeach() - list(LENGTH HASH_FILES HASH_FILE_COUNT) - if(HASH_FILE_COUNT EQUAL 0) + if(HASH_INPUT STREQUAL "") set(${OUT_VAR} "unknown" PARENT_SCOPE) - elseif(HASH_FILE_COUNT EQUAL 1) - migraphx_hash_file("${HASH_FILES}" HASH_RESULT) - set(${OUT_VAR} "${HASH_RESULT}" PARENT_SCOPE) - else() - set(HASH_INPUT "") - foreach(HASH_FILE IN LISTS HASH_FILES) - file(SIZE "${HASH_FILE}" HASH_FILE_SIZE) - file(MD5 "${HASH_FILE}" HASH_FILE_MD5) - string(APPEND HASH_INPUT "${HASH_FILE_SIZE}-${HASH_FILE_MD5};") - endforeach() - string(MD5 HASH_RESULT "${HASH_INPUT}") - string(SUBSTRING "${HASH_RESULT}" 0 12 HASH_RESULT) - set(${OUT_VAR} "${HASH_RESULT}" PARENT_SCOPE) + return() endif() + string(MD5 HASH_RESULT "${HASH_INPUT}") + string(SUBSTRING "${HASH_RESULT}" 0 12 HASH_RESULT) + set(${OUT_VAR} "${HASH_RESULT}" PARENT_SCOPE) endfunction() # MIGRAPHX_USE_AMDMLSS is resolved to a plain boolean at the top-level CMakeLists. diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index 65b107fe523..a88c78d39a0 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -52,8 +52,6 @@ static constexpr const char* rocmlir_id = "nomlir"; std::shared_ptr make_binary_cache() { return std::make_shared(); } -bool binary_cache::verify() const { return settings.verify; } - static std::string short_digest(const std::string& s) { return md5(s).substr(0, 12); } /// A digest of the kernel headers compiled into this build. Taken from the embedded sources @@ -107,6 +105,26 @@ static fs::path entry_path(const fs::path& root, const context& ctx, const std:: return root / version / device_dir(ctx) / (md5(key) + ".mxr"); } +/// Publish content by writing a temporary and renaming it into place, so a reader never sees a +/// half-written file and concurrent writers cannot tear it. The temporary is removed on +/// failure, since leaving it behind would accumulate in the cache directory. +static void write_atomically(const fs::path& dest, const std::vector& content) +{ + auto tmp = dest; + tmp += "." + unique_string("tmp"); + try + { + write_buffer(tmp, content); + fs::rename(tmp, dest); + } + catch(...) + { + std::error_code ec; + fs::remove(tmp, ec); + throw; + } +} + /// Record what this build is, so a directory full of hashes can be identified later. static void write_stamp(const fs::path& dir) { @@ -118,57 +136,57 @@ static void write_stamp(const fs::path& dir) ss << "hip: " << hip_compiler_version().version << "\n"; ss << "kernels: " << kernels_digest() << "\n"; ss << "rocmlir: " << rocmlir_id << "\n"; - // Publish by rename like the entries, so concurrent writers cannot tear the file. - auto tmp = stamp; - tmp += "." + unique_string("tmp"); - write_string(tmp, ss.str()); - fs::rename(tmp, stamp); + auto s = ss.str(); + write_atomically(stamp, std::vector(s.begin(), s.end())); } -optional binary_cache::get(const context& ctx, const std::string& key) +/// Read the entry for a key off disk, or nullopt when there is no usable one. A damaged or +/// stale entry is only worth a recompile, so every failure is a miss and the result is written +/// over the top. +static optional +read_entry(const fs::path& root, const context& ctx, const std::string& key) { - if(key.empty()) - return nullopt; - auto it = memo.find(key); - if(it != memo.end()) - { - counters.reused++; - return it->second; - } - const auto& root = settings.path; if(root.empty()) - { - counters.misses++; return nullopt; - } auto path = entry_path(root, ctx, key); if(path.empty() or not fs::exists(path)) - { - counters.misses++; return nullopt; - } - entry e; + binary_cache::entry e; try { migraphx::from_value(from_msgpack(read_buffer(path)), e); } catch(const std::exception& ex) { - // A damaged or stale entry is only worth a recompile, so treat every failure as a miss - // and let the result be written over the top. log::warn() << "Ignoring unreadable binary cache entry " << path << ": " << ex.what(); - counters.misses++; return nullopt; } if(e.key != key) { log::warn() << "Ignoring binary cache entry with mismatched key: " << path; - counters.misses++; return nullopt; } + return e; +} +optional binary_cache::get(const context& ctx, const std::string& key) +{ + if(key.empty()) + return nullopt; + auto it = memo.find(key); + if(it != memo.end()) + { + counters.reused++; + return it->second; + } + auto e = read_entry(settings.path, ctx, key); + if(not e.has_value()) + { + counters.misses++; + return nullopt; + } counters.hits++; - return memo[key] = std::move(e.code); + return memo.emplace(key, std::move(e->code)).first->second; } void binary_cache::insert(const context& ctx, entry e) @@ -180,23 +198,16 @@ void binary_cache::insert(const context& ctx, entry e) auto path = root.empty() ? fs::path{} : entry_path(root, ctx, e.key); if(not path.empty()) { - // Publish by rename so a reader never sees a half-written entry. The content is decided - // entirely by the key, so a writer that loses the race replaces the file with the same - // bytes and no locking is needed. - auto tmp = path; - tmp += "." + unique_string("tmp"); + // The content is decided entirely by the key, so a writer that loses the publish race + // replaces the file with the same bytes and no locking is needed. try { fs::create_directories(path.parent_path()); write_stamp(fs::path(root) / version_dir()); - write_buffer(tmp, to_msgpack(migraphx::to_value(e))); - fs::rename(tmp, path); + write_atomically(path, to_msgpack(migraphx::to_value(e))); } catch(const std::exception& ex) { - // Leaving the temporary behind would accumulate in the cache directory. - std::error_code ec; - fs::remove(tmp, ec); log::warn() << "Failed to write binary cache entry " << path << ": " << ex.what(); } } diff --git a/src/targets/gpu/compile_hip.cpp b/src/targets/gpu/compile_hip.cpp index fafa264863e..cc3b838bc00 100644 --- a/src/targets/gpu/compile_hip.cpp +++ b/src/targets/gpu/compile_hip.cpp @@ -412,7 +412,7 @@ static hip_compiler_info parse_version(const std::vector& obj) auto fields = split_string(std::string{begin, end}, '|'); if(fields.size() != 3) return {}; - return {.major = fields[0], .minor = fields[1], .version = fields[2], .flags = compile_hip_options({}, version_probe_arch)}; + return {.major = fields[0], .minor = fields[1], .version = fields[2]}; } const hip_compiler_info& hip_compiler_version() diff --git a/src/targets/gpu/compile_hip_code_object.cpp b/src/targets/gpu/compile_hip_code_object.cpp index 940b5619425..66a38f56da2 100644 --- a/src/targets/gpu/compile_hip_code_object.cpp +++ b/src/targets/gpu/compile_hip_code_object.cpp @@ -246,14 +246,23 @@ compile_hip_raw(context& ctx, const std::string& content, hip_compile_options op return cos.front(); } +/// The generated args.hpp for a compile, from the same inputs the compile hands to clang. Shared +/// with hip_compile_key so the key cannot drift from what is compiled. +static std::string make_args_hpp(const hip_compile_options& options) +{ + assert(not options.inputs.empty()); + assert(options.inputs.size() == options.virtual_inputs.size() or + options.virtual_inputs.empty()); + return generate_args_hpp(options.virtual_inputs.empty() ? options.inputs + : options.virtual_inputs); +} + std::string hip_compile_key(const context& ctx, const hip_src& src) { - assert(not src.options.inputs.empty()); - assert(src.options.inputs.size() == src.options.virtual_inputs.size() or - src.options.virtual_inputs.empty()); auto options = src.options; add_derived_params(ctx, options); - options.params = compile_hip_options(options.params, ctx.get_current_device().get_device_name()); + options.params = + compile_hip_options(options.params, ctx.get_current_device().get_device_name()); std::stringstream ss; ss << "arch=" << ctx.get_current_device().get_device_name() << "\n"; @@ -273,8 +282,7 @@ std::string hip_compile_key(const context& ctx, const hip_src& src) // contents running together. for(const auto& f : options.additional_src_files) ss << "src=" << f.path << ":" << f.content.size() << "\n" << f.content << "\n"; - auto args_hpp = - generate_args_hpp(options.virtual_inputs.empty() ? options.inputs : options.virtual_inputs); + auto args_hpp = make_args_hpp(options); ss << "args.hpp:" << args_hpp.size() << "\n" << args_hpp; ss << "main.cpp:" << src.content.size() << "\n" << src.content; return ss.str(); @@ -283,11 +291,8 @@ std::string hip_compile_key(const context& ctx, const hip_src& src) operation compile_hip_code_object(context& ctx, const std::string& content, hip_compile_options options) { - assert(not options.inputs.empty()); - assert(options.inputs.size() == options.virtual_inputs.size() or - options.virtual_inputs.empty()); - auto args_hpp = - generate_args_hpp(options.virtual_inputs.empty() ? options.inputs : options.virtual_inputs); + // src_file views the string, so args_hpp must outlive the compile below. + auto args_hpp = make_args_hpp(options); options.additional_src_files.emplace_back("args.hpp", args_hpp); return code_object_op{value::binary{compile_hip_raw(ctx, content, options)}, diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 2a0f76a5fc3..e756aa2e724 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -150,8 +150,8 @@ static compiler_replace compile_fragment(context& ctx, instruction_ref ins, const operation& preop, const value& solution) { auto cr = compile(ctx, ins, preop, solution); - if(cr.code.empty()) - cr.code.fragment = make_fragment(cr, ins); + assert(cr.code.empty()); + cr.code.fragment = make_fragment(cr, ins); return cr; } @@ -175,9 +175,10 @@ static void verify_reuse(context& ctx, to_string(fresh.code.fragment)); } -/// Marks keys invented for grouping only, given to cells whose compiler cannot describe its -/// output so they are never grouped with another cell. The cache must ignore them: they are -/// numbered by slot, so the same key names different code from one compile batch to the next. +/// Prefix of placeholder keys for cells whose compiler cannot describe its output. Each +/// placeholder is unique, so such cells never group with another cell, and the cache must +/// ignore them: they are numbered by slot, so the same key names different code from one +/// compile batch to the next. static constexpr const char* private_key_prefix = "__migraphx_private_unique_key"; static bool is_private_key(const std::string& key) { return starts_with(key, private_key_prefix); } @@ -617,6 +618,9 @@ struct compile_plan MIGRAPHX_THROW("Multiple kernels without config for " + preop.name()); if(trace_level > 1) std::cout << "Problem: " << config->problem << std::endl; + // One slot was created per solution, and grouping only redirects slots, never adds or + // removes them. + assert(results.size() == config->solutions.size()); std::vector times; times.reserve(results.size()); std::transform(results.begin(), @@ -774,6 +778,9 @@ struct compile_manager for(auto& cell : cp.results) { auto [it, inserted] = index.emplace(cell->key, std::make_pair(&cp, cell)); + // The map key must view the string owned by the cell stored alongside it, or it + // dangles when a duplicate cell is dropped. + assert(it->first.data() == it->second.second->key.data()); if(not inserted) cell = it->second.second; } diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index 8066fab4866..36fd8b69572 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -90,7 +90,8 @@ struct MIGRAPHX_GPU_EXPORT binary_cache /// Counts of what the cache did. struct stats { - /// Served from memory, so an earlier compile in this process was shared. + /// Served from memory: the key was already compiled or read off disk earlier in this + /// process. std::size_t reused = 0; /// Served from the cache directory. std::size_t hits = 0; @@ -109,7 +110,7 @@ struct MIGRAPHX_GPU_EXPORT binary_cache void insert(const context& ctx, entry e); /// True when reused results should be checked against a fresh compile. - bool verify() const; + bool verify() const { return settings.verify; } const stats& get_stats() const { return counters; } diff --git a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp index 793494df7d3..330b75b81dc 100644 --- a/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp +++ b/src/targets/gpu/include/migraphx/gpu/compile_hip.hpp @@ -63,8 +63,6 @@ struct hip_compiler_info /// The full __clang_version__ string, which also names the source revision. std::string version = {}; - std::vector flags = {}; - bool empty() const { return version.empty(); } }; diff --git a/src/targets/gpu/include/migraphx/gpu/context.hpp b/src/targets/gpu/include/migraphx/gpu/context.hpp index cab865ee35b..41bf09c3b9c 100644 --- a/src/targets/gpu/include/migraphx/gpu/context.hpp +++ b/src/targets/gpu/include/migraphx/gpu/context.hpp @@ -288,13 +288,10 @@ struct context this->save(); } }; - context(std::size_t device_id = 0, - std::size_t n = value_of(MIGRAPHX_NSTREAMS{}, 1), - std::shared_ptr bcache = make_binary_cache()) + context(std::size_t device_id = 0, std::size_t n = value_of(MIGRAPHX_NSTREAMS{}, 1)) : current_device(std::make_shared(device_id, n)), begin_event(create_event()), - finish_event(create_event()), - bc(std::move(bcache)) + finish_event(create_event()) { // Bind the cache to this context's device (cross-compile safe: the // key is derived from the context, not a live HIP query). @@ -303,9 +300,8 @@ struct context /// Construct a context for a device that is not present, which can only be /// used to compile and not to execute. - explicit context(const device_description& desc, - std::shared_ptr bcache = make_binary_cache()) - : current_device(std::make_shared(desc)), bc(std::move(bcache)) + explicit context(const device_description& desc) + : current_device(std::make_shared(desc)) { } @@ -479,6 +475,10 @@ struct context return *bc; } + /// Install a cache built with the settings the compile options carry, replacing the default + /// one the context was constructed with. + void set_binary_cache(std::shared_ptr bcache) { bc = std::move(bcache); } + private: // TODO: Make this a vector to support multiple devices std::shared_ptr current_device; @@ -492,7 +492,7 @@ struct context shared begin_event = nullptr; shared finish_event = nullptr; std::shared_ptr pc = std::make_shared(); - std::shared_ptr bc; + std::shared_ptr bc = make_binary_cache(); }; inline void migraphx_to_value(value& v, const context& ctx) { v = ctx.to_value(); } diff --git a/src/targets/gpu/target.cpp b/src/targets/gpu/target.cpp index 16edf20ac80..d827906213f 100644 --- a/src/targets/gpu/target.cpp +++ b/src/targets/gpu/target.cpp @@ -299,22 +299,21 @@ struct pipeline_factory }; } // namespace -static migraphx::context make_context(const target& t, std::shared_ptr bc) +static migraphx::context make_context(const target& t) { if(t.is_cross_compile()) - return context(t.desc, std::move(bc)); - return context(gpu::get_device_id(), value_of(MIGRAPHX_NSTREAMS{}, 1), std::move(bc)); + return context(t.desc); + return context(gpu::get_device_id()); } std::vector target::get_passes(migraphx::context& gctx, const compile_options& options) const { auto backend_opts = get_backend_options(options); - // The initial context predates the compile options, so rebuild it around a cache - // constructed with the settings they carry. - gctx = make_context(*this, - std::make_shared(binary_cache_settings{ - backend_opts.binary_cache, backend_opts.binary_cache_verify})); - auto& ctx = any_cast(gctx); + auto& ctx = any_cast(gctx); + // The initial context predates the compile options, so install a cache constructed with the + // settings they carry. + ctx.set_binary_cache(std::make_shared( + binary_cache_settings{backend_opts.binary_cache, backend_opts.binary_cache_verify})); ctx.set_exhaustive_tune_flag(options.exhaustive_tune); ctx.load_problem_cache(); // TODO: update load_problem_cache to include gpu arch @@ -357,7 +356,7 @@ std::vector target::get_passes(migraphx::context& gctx, const compile_opti std::string target::name() const { return "gpu"; } -migraphx::context target::get_context() const { return make_context(*this, make_binary_cache()); } +migraphx::context target::get_context() const { return make_context(*this); } argument target::copy_to(const argument& arg) const { diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index d506d7c745b..27cda498189 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -198,7 +198,8 @@ TEST_CASE(duplicate_kernels_compile_once_without_a_directory) { auto cache = std::make_shared( migraphx::gpu::binary_cache_settings{.path = ""}); - migraphx::gpu::context ctx{0, 1, cache}; + migraphx::gpu::context ctx{0, 1}; + ctx.set_binary_cache(cache); auto p1 = two_identical_pointwise(); migraphx::run_passes(*p1.get_main_module(), From 7ad700a4d650d5b66a83bbe42876ae50de34cf29 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 14:19:53 -0500 Subject: [PATCH 24/49] Use tmp_dir --- src/include/migraphx/tmp_dir.hpp | 2 +- src/targets/gpu/binary_cache.cpp | 24 ++++++++---------------- src/tmp_dir.cpp | 5 ++--- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/include/migraphx/tmp_dir.hpp b/src/include/migraphx/tmp_dir.hpp index 8a851124089..70dfb3b1d80 100644 --- a/src/include/migraphx/tmp_dir.hpp +++ b/src/include/migraphx/tmp_dir.hpp @@ -39,7 +39,7 @@ MIGRAPHX_EXPORT std::string unique_string(const std::string& prefix); struct MIGRAPHX_EXPORT tmp_dir { fs::path path; - tmp_dir(std::string_view prefix = ""); + tmp_dir(std::string_view prefix = "", const fs::path& loc = fs::temp_directory_path()); tmp_dir(tmp_dir&&) = default; void execute(std::string_view cmd, const std::vector& args = {}) const; diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index a88c78d39a0..be217fefd67 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -105,24 +105,16 @@ static fs::path entry_path(const fs::path& root, const context& ctx, const std:: return root / version / device_dir(ctx) / (md5(key) + ".mxr"); } -/// Publish content by writing a temporary and renaming it into place, so a reader never sees a -/// half-written file and concurrent writers cannot tear it. The temporary is removed on -/// failure, since leaving it behind would accumulate in the cache directory. +/// Publish content by writing it in a temporary directory beside the destination and renaming +/// it into place, so a reader never sees a half-written file and concurrent writers cannot +/// tear it. The temporary stays beside the destination because the rename is only atomic +/// within one filesystem; the tmp_dir removes whatever a failed write leaves behind. static void write_atomically(const fs::path& dest, const std::vector& content) { - auto tmp = dest; - tmp += "." + unique_string("tmp"); - try - { - write_buffer(tmp, content); - fs::rename(tmp, dest); - } - catch(...) - { - std::error_code ec; - fs::remove(tmp, ec); - throw; - } + tmp_dir td{"cache", dest.parent_path()}; + auto tmp = td.path / dest.filename(); + write_buffer(tmp, content); + fs::rename(tmp, dest); } /// Record what this build is, so a directory full of hashes can be identified later. diff --git a/src/tmp_dir.cpp b/src/tmp_dir.cpp index c8e9f6b0a44..e326c9d0401 100644 --- a/src/tmp_dir.cpp +++ b/src/tmp_dir.cpp @@ -76,9 +76,8 @@ std::string unique_string(const std::string& prefix) return ss.str(); } -tmp_dir::tmp_dir(std::string_view prefix) - : path(fs::temp_directory_path() / - unique_string(prefix.empty() ? "migraphx" : "migraphx-" + std::string{prefix})) +tmp_dir::tmp_dir(std::string_view prefix, const fs::path& loc) + : path(loc / unique_string(prefix.empty() ? "migraphx" : "migraphx-" + std::string{prefix})) { fs::create_directories(this->path); } From 4f65f3c12bd44b39e63aa8071a80ed22041f1fee Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 14:36:11 -0500 Subject: [PATCH 25/49] Cleanup comments --- src/targets/gpu/CMakeLists.txt | 6 +- src/targets/gpu/binary_cache.cpp | 11 ++-- src/targets/gpu/compile_hip_code_object.cpp | 3 +- src/targets/gpu/compile_ops.cpp | 58 +++++++------------ .../gpu/include/migraphx/gpu/binary_cache.hpp | 3 +- .../gpu/include/migraphx/gpu/context.hpp | 3 +- src/targets/gpu/target.cpp | 12 +--- 7 files changed, 34 insertions(+), 62 deletions(-) diff --git a/src/targets/gpu/CMakeLists.txt b/src/targets/gpu/CMakeLists.txt index 2122090ac75..5924f0e3097 100644 --- a/src/targets/gpu/CMakeLists.txt +++ b/src/targets/gpu/CMakeLists.txt @@ -24,10 +24,8 @@ find_package(hip REQUIRED) -# Produce a short identity for a list of toolchain library files, used to separate binary cache -# entries that different toolchains cannot share. Entries that are not files (e.g. target names) -# are skipped; CMake cannot stream one hash across several files, so hash the per-file content -# hashes instead. +# A short identity for a list of toolchain libraries, used to separate binary cache entries +# that different toolchains cannot share. Non-file entries (e.g. target names) are skipped. function(migraphx_hash_libraries LIB_PATHS OUT_VAR) set(HASH_INPUT "") foreach(LIB_PATH IN LISTS LIB_PATHS) diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index be217fefd67..f07862c6b27 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -105,10 +105,8 @@ static fs::path entry_path(const fs::path& root, const context& ctx, const std:: return root / version / device_dir(ctx) / (md5(key) + ".mxr"); } -/// Publish content by writing it in a temporary directory beside the destination and renaming -/// it into place, so a reader never sees a half-written file and concurrent writers cannot -/// tear it. The temporary stays beside the destination because the rename is only atomic -/// within one filesystem; the tmp_dir removes whatever a failed write leaves behind. +/// Publish by rename so a reader never sees a half-written file. The temporary stays beside +/// the destination since the rename is only atomic within one filesystem. static void write_atomically(const fs::path& dest, const std::vector& content) { tmp_dir td{"cache", dest.parent_path()}; @@ -132,9 +130,8 @@ static void write_stamp(const fs::path& dir) write_atomically(stamp, std::vector(s.begin(), s.end())); } -/// Read the entry for a key off disk, or nullopt when there is no usable one. A damaged or -/// stale entry is only worth a recompile, so every failure is a miss and the result is written -/// over the top. +/// Read the entry for a key off disk. Any failure is just a miss, so a damaged entry costs a +/// recompile and is written over. static optional read_entry(const fs::path& root, const context& ctx, const std::string& key) { diff --git a/src/targets/gpu/compile_hip_code_object.cpp b/src/targets/gpu/compile_hip_code_object.cpp index 66a38f56da2..bf83493fa2b 100644 --- a/src/targets/gpu/compile_hip_code_object.cpp +++ b/src/targets/gpu/compile_hip_code_object.cpp @@ -246,8 +246,7 @@ compile_hip_raw(context& ctx, const std::string& content, hip_compile_options op return cos.front(); } -/// The generated args.hpp for a compile, from the same inputs the compile hands to clang. Shared -/// with hip_compile_key so the key cannot drift from what is compiled. +/// Shared between the compile and hip_compile_key so the key cannot drift from what is compiled. static std::string make_args_hpp(const hip_compile_options& options) { assert(not options.inputs.empty()); diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index e756aa2e724..ad692b3a3c3 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -175,10 +175,9 @@ static void verify_reuse(context& ctx, to_string(fresh.code.fragment)); } -/// Prefix of placeholder keys for cells whose compiler cannot describe its output. Each -/// placeholder is unique, so such cells never group with another cell, and the cache must -/// ignore them: they are numbered by slot, so the same key names different code from one -/// compile batch to the next. +/// Prefix of unique placeholder keys for cells whose compiler cannot describe its output, so +/// they never group with another cell. The cache must ignore them, since a slot number names +/// different code from one compile batch to the next. static constexpr const char* private_key_prefix = "__migraphx_private_unique_key"; static bool is_private_key(const std::string& key) { return starts_with(key, private_key_prefix); } @@ -398,20 +397,15 @@ MIGRAPHX_REGISTER_OP(dynamic_code_object_op); // forward declared since it requires compile_manager static void replace_inserted_device_ops(context& ctx, module& m); -/** - * One compilation and its result, shared by every result slot whose key matches. - * - * Duplicate compiles are grouped by pointing the slots of every matching plan at the same cell, - * so writing the result once puts it in place everywhere. During the parallel compile each cell - * is written by exactly one worker; the plans sharing it read it only after the workers join. - */ +/// One compilation and its result, shared by every result slot whose key matches. Each cell is +/// written by exactly one worker during the parallel compile; sharers read it after the join. struct compile_cell { explicit compile_cell(value s) : solution(std::move(s)) {} value solution = {}; - /// Identifies the code the compile would produce. When the compiler cannot say, a private - /// key is invented instead, which keeps the cell out of the cache and out of the grouping. + /// Identifies the code the compile would produce, or an invented private key when the + /// compiler cannot say. std::string key = {}; optional result = nullopt; }; @@ -487,8 +481,7 @@ struct compile_plan } } - /// A standalone program with just this instruction and the compiled code objects inserted, - /// used for benchmarking a result and for dumping it. + /// A standalone program with just this instruction and its compiled code objects inserted. program make_program(const compiler_replace& cr) const { program bench_prog; @@ -618,8 +611,7 @@ struct compile_plan MIGRAPHX_THROW("Multiple kernels without config for " + preop.name()); if(trace_level > 1) std::cout << "Problem: " << config->problem << std::endl; - // One slot was created per solution, and grouping only redirects slots, never adds or - // removes them. + // One slot per solution; grouping only redirects slots. assert(results.size() == config->solutions.size()); std::vector times; times.reserve(results.size()); @@ -764,12 +756,9 @@ struct compile_manager }); } - // Group the slots that would compile to the same code by pointing them at one shared - // cell; writing that cell's result once then puts it in place for every plan that - // shares it. The plan kept with the cell drives the compile, which any of the sharers - // could do since they compile to the same code. The keys are whole kernel sources, so - // the index views the canonical cell's key instead of copying; it does not outlive - // this function. + // Group slots that would compile to the same code by pointing them at one shared cell, + // so writing the result once puts it in place everywhere; the stored plan drives the + // compile. The keys are whole kernel sources, so the index views them instead of copying. std::unordered_map>> index; @@ -778,26 +767,24 @@ struct compile_manager for(auto& cell : cp.results) { auto [it, inserted] = index.emplace(cell->key, std::make_pair(&cp, cell)); - // The map key must view the string owned by the cell stored alongside it, or it - // dangles when a duplicate cell is dropped. + // The key must view the stored cell's string, or it dangles when a duplicate + // cell is dropped. assert(it->first.data() == it->second.second->key.data()); if(not inserted) cell = it->second.second; } } - // The cache is unguarded, so everything it can answer is looked up before any compile - // starts, and everything compiled is stored after they all finish. Until the compiles - // run below, a cell holding a result is one whose result was reused from the cache. + // The cache is unguarded, so all lookups happen before any compile starts and stores + // after they finish. Until then, a cell holding a result is one reused from the cache. for(const auto& entry : index) { const auto& [cp, cell] = entry.second; cell->result = cp->lookup(cell->key); } - // Only the cells that still need parallel work, a compile or a verify of a reused - // result, become tasks; since every key is known before any compile starts, each cell - // is handed to exactly one task and the compiles never have to coordinate. + // Only cells that still need work, a compile or a verify of a reused result, become + // tasks; each cell gets exactly one task, so the compiles never coordinate. std::vector>> tasks; transform_if( index.begin(), @@ -821,13 +808,12 @@ struct compile_manager { if(not cell->result.has_value()) continue; - // When verifying, this also stores reused results back, which rewrites the same - // bytes under the same key and is harmless. + // When verifying, reused results are stored again, rewriting the same bytes + // harmlessly. cp->store(cell->solution, cell->key, cell->result->code); assert(not cell->result->code.empty()); - // Only the serializable code is used from here on. Dropping the replace function - // releases the code objects and split modules its closure holds, and keeps it from - // being run against an instruction other than the one it was built for. + // Only the serializable code is used from here on; dropping the replace function + // releases what its closure holds and keeps it off other instructions. cell->result->replace_fn = nullptr; } diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index 36fd8b69572..21e3b4464ac 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -90,8 +90,7 @@ struct MIGRAPHX_GPU_EXPORT binary_cache /// Counts of what the cache did. struct stats { - /// Served from memory: the key was already compiled or read off disk earlier in this - /// process. + /// Served from memory, from an earlier compile or disk read in this process. std::size_t reused = 0; /// Served from the cache directory. std::size_t hits = 0; diff --git a/src/targets/gpu/include/migraphx/gpu/context.hpp b/src/targets/gpu/include/migraphx/gpu/context.hpp index d765abea133..a7dafd80363 100644 --- a/src/targets/gpu/include/migraphx/gpu/context.hpp +++ b/src/targets/gpu/include/migraphx/gpu/context.hpp @@ -492,8 +492,7 @@ struct context return *bc; } - /// Install a cache built with the settings the compile options carry, replacing the default - /// one the context was constructed with. + /// Replace the default cache with one built from the compile options. void set_binary_cache(std::shared_ptr bcache) { bc = std::move(bcache); } private: diff --git a/src/targets/gpu/target.cpp b/src/targets/gpu/target.cpp index 89d2a134b4e..b9ed45663cf 100644 --- a/src/targets/gpu/target.cpp +++ b/src/targets/gpu/target.cpp @@ -114,11 +114,9 @@ struct backend_options std::string binary_cache = binary_cache_settings{}.path; /// Compile even when a kernel could be reused, and fail if the two disagree. bool binary_cache_verify = false; - // Read/write problem caches (the common case: a user tuning a model). New - // tuning solutions are saved back to these files. + // Problem caches that new tuning solutions are saved back to. std::vector problem_cache_files = {}; - // Read-only problem caches (system-level, e.g. shipped by gpuep or an ISV), - // searched after the writable caches and never written back. + // System-level problem caches, searched after the writable ones and never written. std::vector read_only_problem_cache_files = {}; // Layout used for convolutions, by name: channels_first, channels_last, or channels_auto. layout_convolution::layout_order convolution_layout = layout_convolution::channels_auto; @@ -323,8 +321,7 @@ std::vector target::get_passes(migraphx::context& gctx, const compile_opti { auto backend_opts = get_backend_options(options); auto& ctx = any_cast(gctx); - // The initial context predates the compile options, so install a cache constructed with the - // settings they carry. + // The context predates the compile options, so the cache they configure is installed here. ctx.set_binary_cache(std::make_shared( binary_cache_settings{backend_opts.binary_cache, backend_opts.binary_cache_verify})); ctx.set_exhaustive_tune_flag(options.exhaustive_tune); @@ -332,9 +329,6 @@ std::vector target::get_passes(migraphx::context& gctx, const compile_opti if(options.compile_mode == compile_modes::max) ctx.set_exhaustive_tune_flag(true); - // Problem cache files arrive as GPU backend options. The writable caches - // (problem_cache_files) save new tuning solutions back; the read-only caches - // (read_only_problem_cache_files) are system-level and never written. ctx.load_problem_caches(backend_opts.read_only_problem_cache_files, backend_opts.problem_cache_files); From 263d34ec2732615c273e14090ac7862ba224c02a Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 27 Aug 2026 14:37:59 -0500 Subject: [PATCH 26/49] Add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d0999d6eab..5972dc67bc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Full documentation for MIGraphX is available at ### Added +* Added a binary cache for compiled GPU kernels: identical kernels within a model compile once, and setting the `MIGRAPHX_BINARY_CACHE` environment variable (or the `binary_cache` backend option) also persists them on disk so later compiles of the same kernels skip compilation entirely; a `binary_cache_verify` backend option recompiles reused kernels and fails if they differ. * Added a layered problem-cache priority list (searched in order, first hit wins), delivered to the GPU target through the `problem_cache_files` backend option. * Added `ArrayFeatureExtractor` ONNX operator support (#4742). * Added support for building against ROCm 7.13 and newer using TheRock (#4952) From 3e6d61b4fec1f395a6d43f67d655ddff14935b77 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Wed, 2 Sep 2026 16:16:35 +0200 Subject: [PATCH 27/49] Sql backend implmementation for binary cache; Added common interface to support mutiple backend in binary cache; Added functionalities to sql wrapper to support blobs --- src/include/migraphx/sqlite.hpp | 65 ++- src/sqlite.cpp | 128 +++++- src/targets/gpu/CMakeLists.txt | 2 + src/targets/gpu/binary_cache.cpp | 123 +++--- src/targets/gpu/file_binary_cache.cpp | 112 +++++ .../gpu/include/migraphx/gpu/binary_cache.hpp | 35 +- .../migraphx/gpu/binary_cache_backend.hpp | 399 ++++++++++++++++++ .../migraphx/gpu/binary_cache_entry.hpp | 68 +++ .../migraphx/gpu/file_binary_cache.hpp | 60 +++ .../migraphx/gpu/sqlite_binary_cache.hpp | 81 ++++ src/targets/gpu/sqlite_binary_cache.cpp | 210 +++++++++ test/gpu/binary_cache.cpp | 333 +++++++++++++-- test/sqlite.cpp | 79 +++- 13 files changed, 1575 insertions(+), 120 deletions(-) create mode 100644 src/targets/gpu/file_binary_cache.cpp create mode 100644 src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp create mode 100644 src/targets/gpu/include/migraphx/gpu/binary_cache_entry.hpp create mode 100644 src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp create mode 100644 src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp create mode 100644 src/targets/gpu/sqlite_binary_cache.cpp diff --git a/src/include/migraphx/sqlite.hpp b/src/include/migraphx/sqlite.hpp index 97f3c0405f7..b11bb6e000b 100644 --- a/src/include/migraphx/sqlite.hpp +++ b/src/include/migraphx/sqlite.hpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,7 +26,11 @@ #include #include +#include +#include #include +#include +#include #include #include @@ -34,14 +38,73 @@ namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { struct sqlite_impl; +struct sqlite_stmt_impl; + +/// A prepared statement, holding a reference to the connection it was prepared on so it can +/// never outlive it. +/// +/// Not thread safe: one statement may be used by one thread at a time, even though the +/// connection itself is serialized. Reuse is the point of preparing -- prepare once, then +/// reset/bind/step per operation. +struct MIGRAPHX_EXPORT sqlite_stmt +{ + sqlite_stmt() = default; + + /// Bind a parameter. Indices are 1-based, matching sqlite's own convention. + sqlite_stmt& bind(int i, std::string_view s); + sqlite_stmt& bind(int i, std::int64_t x); + sqlite_stmt& bind(int i, const std::vector& blob); + + /// Step once. True when a row is available, false when the statement is done. + bool step(); + + /// Clear bindings and rewind, so the statement can be used again. Safe at any point, + /// including after step() has thrown. + void reset() noexcept; + + /// Read a column of the current row. Indices here are 0-based, again matching sqlite. + std::string column_text(int i) const; + std::vector column_blob(int i) const; + + bool valid() const { return impl != nullptr; } + + private: + friend struct sqlite; + std::shared_ptr impl; +}; + +/// Resets a statement on scope exit, so an early return or a thrown exception cannot leave +/// bindings or a half-consumed result set behind for whoever uses the statement next. +struct sqlite_stmt_reset +{ + explicit sqlite_stmt_reset(sqlite_stmt& s) : stmt(&s) {} + sqlite_stmt_reset(const sqlite_stmt_reset&) = delete; + sqlite_stmt_reset& operator=(const sqlite_stmt_reset&) = delete; + ~sqlite_stmt_reset() { stmt->reset(); } + + private: + sqlite_stmt* stmt; +}; struct MIGRAPHX_EXPORT sqlite { sqlite() = default; static sqlite read(const fs::path& p); static sqlite write(const fs::path& p); + + /// Open for writing, or nullopt if the file cannot be opened or created. For callers + /// that treat an unusable database as "no cache" rather than as an error. + static optional try_write(const fs::path& p); + std::vector> execute(const std::string& s); + sqlite_stmt prepare(const std::string& sql); + + /// How long to wait for a lock held by another connection before failing. + void set_busy_timeout(int ms); + + bool valid() const { return impl != nullptr; } + private: std::shared_ptr impl; }; diff --git a/src/sqlite.cpp b/src/sqlite.cpp index 823d74c3a77..5dcdd247150 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,12 +36,21 @@ using sqlite3_ptr = MIGRAPHX_MANAGE_PTR(sqlite3*, sqlite3_close); struct sqlite_impl { sqlite3* get() const { return ptr.get(); } - void open(const fs::path& p, int flags) + + // Returns false rather than throwing, so callers that treat an unusable database as + // "no cache" do not have to catch. sqlite3_open_v2 hands back a handle even on failure + // (that is where the error message lives), so `ptr` takes ownership either way. + bool try_open(const fs::path& p, int flags) { sqlite3* ptr_tmp = nullptr; int rc = sqlite3_open_v2(p.string().c_str(), &ptr_tmp, flags, nullptr); ptr = sqlite3_ptr{ptr_tmp}; - if(rc != 0) + return rc == 0; + } + + void open(const fs::path& p, int flags) + { + if(not try_open(p, flags)) MIGRAPHX_THROW("error opening " + p.string() + ": " + error_message()); } @@ -74,6 +83,19 @@ struct sqlite_impl sqlite3_ptr ptr; }; +using sqlite3_stmt_ptr = MIGRAPHX_MANAGE_PTR(sqlite3_stmt*, sqlite3_finalize); + +struct sqlite_stmt_impl +{ + sqlite3_stmt* get() const { return ptr.get(); } + std::string error_message() const { return db->error_message(); } + + // Holding the connection keeps it alive for as long as any statement prepared on it, + // since finalizing after the connection is closed is undefined. + std::shared_ptr db; + sqlite3_stmt_ptr ptr; +}; + sqlite sqlite::read(const fs::path& p) { sqlite r; @@ -91,6 +113,16 @@ sqlite sqlite::write(const fs::path& p) return r; } +optional sqlite::try_write(const fs::path& p) +{ + sqlite r; + r.impl = std::make_shared(); + // Using '+' instead of bitwise '|' to avoid compilation warning + if(not r.impl->try_open(p, SQLITE_OPEN_READWRITE + SQLITE_OPEN_CREATE)) + return nullopt; + return r; +} + std::vector> sqlite::execute(const std::string& s) { std::vector> result; @@ -108,5 +140,95 @@ std::vector> sqlite::execute(const return result; } +sqlite_stmt sqlite::prepare(const std::string& s) +{ + sqlite3_stmt* stmt_tmp = nullptr; + int rc = sqlite3_prepare_v2(impl->get(), s.c_str(), -1, &stmt_tmp, nullptr); + sqlite_stmt result; + result.impl = std::make_shared(); + result.impl->db = impl; + result.impl->ptr = sqlite3_stmt_ptr{stmt_tmp}; + if(rc != SQLITE_OK) + MIGRAPHX_THROW("error preparing '" + s + "': " + impl->error_message()); + return result; +} + +void sqlite::set_busy_timeout(int ms) { sqlite3_busy_timeout(impl->get(), ms); } + +sqlite_stmt& sqlite_stmt::bind(int i, std::string_view s) +{ + // A null pointer binds SQL NULL rather than an empty string, and a default-constructed + // string_view has null data(), so empty input needs its own case. + int rc = s.empty() ? sqlite3_bind_text64(impl->get(), i, "", 0, SQLITE_STATIC, SQLITE_UTF8) + : sqlite3_bind_text64( + impl->get(), i, s.data(), s.size(), SQLITE_TRANSIENT, SQLITE_UTF8); + if(rc != SQLITE_OK) + MIGRAPHX_THROW(impl->error_message()); + return *this; +} + +sqlite_stmt& sqlite_stmt::bind(int i, std::int64_t x) +{ + int rc = sqlite3_bind_int64(impl->get(), i, x); + if(rc != SQLITE_OK) + MIGRAPHX_THROW(impl->error_message()); + return *this; +} + +sqlite_stmt& sqlite_stmt::bind(int i, const std::vector& blob) +{ + // As with text, an empty vector's data() may be null, which would bind SQL NULL; a + // zero-length zeroblob is an empty BLOB instead. bind_blob64 is used because the + // non-64 form takes the size as an int. + int rc = blob.empty() + ? sqlite3_bind_zeroblob(impl->get(), i, 0) + : sqlite3_bind_blob64( + impl->get(), i, blob.data(), blob.size(), SQLITE_TRANSIENT); + if(rc != SQLITE_OK) + MIGRAPHX_THROW(impl->error_message()); + return *this; +} + +bool sqlite_stmt::step() +{ + int rc = sqlite3_step(impl->get()); + if(rc == SQLITE_ROW) + return true; + if(rc == SQLITE_DONE) + return false; + MIGRAPHX_THROW(impl->error_message()); +} + +void sqlite_stmt::reset() noexcept +{ + if(impl == nullptr) + return; + // The return of sqlite3_reset is the error from the preceding step(), which the caller + // has already seen as a throw. There is nothing new to report, and this must not throw. + (void)sqlite3_reset(impl->get()); + (void)sqlite3_clear_bindings(impl->get()); +} + +std::string sqlite_stmt::column_text(int i) const +{ + const auto* text = sqlite3_column_text(impl->get(), i); + int n = sqlite3_column_bytes(impl->get(), i); + if(text == nullptr or n <= 0) + return {}; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return {reinterpret_cast(text), static_cast(n)}; +} + +std::vector sqlite_stmt::column_blob(int i) const +{ + // sqlite3_column_blob must be called before sqlite3_column_bytes: the other order can + // force a type conversion that invalidates the pointer. + const auto* data = static_cast(sqlite3_column_blob(impl->get(), i)); + int n = sqlite3_column_bytes(impl->get(), i); + if(data == nullptr or n <= 0) + return {}; + return {data, data + n}; +} + } // namespace MIGRAPHX_INLINE_NS } // namespace migraphx diff --git a/src/targets/gpu/CMakeLists.txt b/src/targets/gpu/CMakeLists.txt index 5924f0e3097..0d60bef9584 100644 --- a/src/targets/gpu/CMakeLists.txt +++ b/src/targets/gpu/CMakeLists.txt @@ -235,6 +235,7 @@ add_library(migraphx_gpu device_description.cpp device_name.cpp eliminate_data_type_for_gpu.cpp + file_binary_cache.cpp fixed_pad.cpp fuse_ck.cpp fuse_mlir.cpp @@ -262,6 +263,7 @@ add_library(migraphx_gpu problem_cache.cpp rocblas.cpp schedule_model.cpp + sqlite_binary_cache.cpp sqlite_problem_cache.cpp sync_device.cpp target.cpp diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index f07862c6b27..c896c3ecb9b 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -22,16 +22,15 @@ * THE SOFTWARE. */ #include +#include +#include #include #include -#include -#include #include #include #include #include #include -#include #include #include @@ -95,69 +94,65 @@ static std::string device_dir(const context& ctx) "_wf" + std::to_string(device.get_wavefront_size()); } -/// Where an entry lives, or an empty path when the toolchain cannot be identified and entries -/// from different toolchains would be indistinguishable. -static fs::path entry_path(const fs::path& root, const context& ctx, const std::string& key) +const std::string& binary_cache::version_stamp() { - const auto& version = binary_cache::version_dir(); - if(version.empty()) - return {}; - return root / version / device_dir(ctx) / (md5(key) + ".mxr"); -} - -/// Publish by rename so a reader never sees a half-written file. The temporary stays beside -/// the destination since the rename is only atomic within one filesystem. -static void write_atomically(const fs::path& dest, const std::vector& content) -{ - tmp_dir td{"cache", dest.parent_path()}; - auto tmp = td.path / dest.filename(); - write_buffer(tmp, content); - fs::rename(tmp, dest); -} - -/// Record what this build is, so a directory full of hashes can be identified later. -static void write_stamp(const fs::path& dir) -{ - auto stamp = dir / "cache.info"; - if(fs::exists(stamp)) - return; - std::stringstream ss; - ss << "format: " << binary_cache_format << "\n"; - ss << "hip: " << hip_compiler_version().version << "\n"; - ss << "kernels: " << kernels_digest() << "\n"; - ss << "rocmlir: " << rocmlir_id << "\n"; - auto s = ss.str(); - write_atomically(stamp, std::vector(s.begin(), s.end())); + static const std::string stamp = [] { + std::stringstream ss; + ss << "format: " << binary_cache_format << "\n"; + ss << "hip: " << hip_compiler_version().version << "\n"; + ss << "kernels: " << kernels_digest() << "\n"; + ss << "rocmlir: " << rocmlir_id << "\n"; + return ss.str(); + }(); + return stamp; } -/// Read the entry for a key off disk. Any failure is just a miss, so a damaged entry costs a -/// recompile and is written over. -static optional -read_entry(const fs::path& root, const context& ctx, const std::string& key) +/// Turn a stored blob back into an entry. Any failure is just a miss, so a damaged entry costs +/// a recompile and is written over. Shared by every backend, so they all tolerate corruption +/// and survive a hash collision the same way. +static optional decode_entry(const std::vector& blob, + const std::string& key) { - if(root.empty()) - return nullopt; - auto path = entry_path(root, ctx, key); - if(path.empty() or not fs::exists(path)) - return nullopt; binary_cache::entry e; try { - migraphx::from_value(from_msgpack(read_buffer(path)), e); + migraphx::from_value(from_msgpack(blob), e); } catch(const std::exception& ex) { - log::warn() << "Ignoring unreadable binary cache entry " << path << ": " << ex.what(); + log::warn() << "Ignoring unreadable binary cache entry " << md5(key) << ": " << ex.what(); return nullopt; } + // Entries are addressed by a hash of the key, so the full key is checked here to make a + // collision a miss rather than a wrong kernel. if(e.key != key) { - log::warn() << "Ignoring binary cache entry with mismatched key: " << path; + log::warn() << "Ignoring binary cache entry with mismatched key: " << md5(key); return nullopt; } return e; } +// Select the storage backend by file type, matching make_problem_cache_backend: a ".db"/".sqlite" +// path is a SQLite database, anything else is a directory of entries. +static optional make_binary_cache_backend(const std::string& path) +{ + if(path.empty()) + return nullopt; + if(ends_with(path, ".db") or ends_with(path, ".sqlite")) + return sqlite_binary_cache::open(path); // nullopt when the database is unusable + return binary_cache_backend{file_binary_cache{path}}; +} + +// Nothing can be persisted safely when the compiler cannot be identified, since entries from +// different toolchains would be indistinguishable. That is a property of the cache rather than +// of the storage medium, so it is checked here instead of in each backend. +binary_cache::binary_cache(binary_cache_settings s) : settings(std::move(s)) +{ + if(not version_dir().empty()) + backend = make_binary_cache_backend(settings.path); +} + optional binary_cache::get(const context& ctx, const std::string& key) { if(key.empty()) @@ -168,14 +163,21 @@ optional binary_cache::get(const context& ctx, const std::string& counters.reused++; return it->second; } - auto e = read_entry(settings.path, ctx, key); - if(not e.has_value()) + if(backend.has_value()) { - counters.misses++; - return nullopt; + auto blob = backend->load(version_dir(), device_dir(ctx), md5(key)); + if(blob.has_value()) + { + auto e = decode_entry(*blob, key); + if(e.has_value()) + { + counters.hits++; + return memo.emplace(key, std::move(e->code)).first->second; + } + } } - counters.hits++; - return memo.emplace(key, std::move(e->code)).first->second; + counters.misses++; + return nullopt; } void binary_cache::insert(const context& ctx, entry e) @@ -183,21 +185,18 @@ void binary_cache::insert(const context& ctx, entry e) if(e.key.empty()) return; counters.compiled++; - const auto& root = settings.path; - auto path = root.empty() ? fs::path{} : entry_path(root, ctx, e.key); - if(not path.empty()) + if(backend.has_value()) { - // The content is decided entirely by the key, so a writer that loses the publish race - // replaces the file with the same bytes and no locking is needed. try { - fs::create_directories(path.parent_path()); - write_stamp(fs::path(root) / version_dir()); - write_atomically(path, to_msgpack(migraphx::to_value(e))); + // Serializing inside the guard, rather than in the argument list, keeps a failure + // here a warning like any other storage failure instead of escaping insert(). + auto blob = to_msgpack(migraphx::to_value(e)); + backend->store(version_dir(), device_dir(ctx), md5(e.key), e, blob); } catch(const std::exception& ex) { - log::warn() << "Failed to write binary cache entry " << path << ": " << ex.what(); + log::warn() << "Failed to store binary cache entry " << md5(e.key) << ": " << ex.what(); } } memo[std::move(e.key)] = std::move(e.code); diff --git a/src/targets/gpu/file_binary_cache.cpp b/src/targets/gpu/file_binary_cache.cpp new file mode 100644 index 00000000000..92adb413c62 --- /dev/null +++ b/src/targets/gpu/file_binary_cache.cpp @@ -0,0 +1,112 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +#include +#include +#include +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +static_assert(std::is_constructible{}, + "file_binary_cache must satisfy the binary_cache_backend concept"); + +/// Where an entry lives. The caller guarantees a non-empty version, so entries compiled by +/// different toolchains can never land on the same path. +static fs::path entry_path(const fs::path& root, + const std::string& version, + const std::string& device, + const std::string& key_hash) +{ + return root / version / device / (key_hash + ".mxr"); +} + +/// Publish by rename so a reader never sees a half-written file. The temporary stays beside +/// the destination since the rename is only atomic within one filesystem. +static void write_atomically(const fs::path& dest, const std::vector& content) +{ + tmp_dir td{"cache", dest.parent_path()}; + auto tmp = td.path / dest.filename(); + write_buffer(tmp, content); + fs::rename(tmp, dest); +} + +/// Record what this build is, so a directory full of hashes can be identified later. +static void write_stamp(const fs::path& dir) +{ + auto stamp = dir / "cache.info"; + if(fs::exists(stamp)) + return; + const auto& s = binary_cache::version_stamp(); + write_atomically(stamp, std::vector(s.begin(), s.end())); +} + +optional> file_binary_cache::load(const std::string& version, + const std::string& device, + const std::string& key_hash) +{ + auto path = entry_path(root, version, device, key_hash); + try + { + if(not fs::exists(path)) + return nullopt; + return read_buffer(path); + } + catch(const std::exception& ex) + { + // An unreadable entry is a miss, which costs a recompile and nothing else. + log::warn() << "Failed to read binary cache entry " << path << ": " << ex.what(); + return nullopt; + } +} + +void file_binary_cache::store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry&, + const std::vector& blob) +{ + auto path = entry_path(root, version, device, key_hash); + // The content is decided entirely by the key, so a writer that loses the publish race + // replaces the file with the same bytes and no locking is needed. + try + { + fs::create_directories(path.parent_path()); + write_stamp(root / version); + write_atomically(path, blob); + } + catch(const std::exception& ex) + { + log::warn() << "Failed to write binary cache entry " << path << ": " << ex.what(); + } +} + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index 21e3b4464ac..f928d091aee 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -26,6 +26,8 @@ #include #include +#include +#include #include #include #include @@ -66,33 +68,16 @@ struct binary_cache_settings */ struct MIGRAPHX_GPU_EXPORT binary_cache { - /// What gets written to disk for one compiled kernel. The op name, problem and solution are - /// stored for offline inspection; only the key is checked when an entry is loaded. - struct entry - { - std::string key = {}; - std::string op_name = {}; - value problem = {}; - value solution = {}; - compiled_code code = {}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.key, "key"), - f(self.op_name, "op_name"), - f(self.problem, "problem"), - f(self.solution, "solution"), - f(self.code, "code")); - } - }; + /// What gets stored for one compiled kernel. Defined in binary_cache_entry.hpp so the + /// storage backends can name it; the alias keeps binary_cache::entry working. + using entry = binary_cache_entry; /// Counts of what the cache did. struct stats { /// Served from memory, from an earlier compile or disk read in this process. std::size_t reused = 0; - /// Served from the cache directory. + /// Served from the storage backend. std::size_t hits = 0; /// Not found, so the caller had to compile. std::size_t misses = 0; @@ -100,7 +85,7 @@ struct MIGRAPHX_GPU_EXPORT binary_cache std::size_t compiled = 0; }; - explicit binary_cache(binary_cache_settings s = {}) : settings(std::move(s)) {} + explicit binary_cache(binary_cache_settings s = {}); /// Look up a key, consulting memory first and then the cache directory. optional get(const context& ctx, const std::string& key); @@ -119,9 +104,15 @@ struct MIGRAPHX_GPU_EXPORT binary_cache /// since entries from different compilers could not be told apart. static const std::string& version_dir(); + /// A human-readable description of what version_dir() encodes, for backends that can store + /// a self-describing marker alongside their entries. + static const std::string& version_stamp(); + private: std::unordered_map memo; binary_cache_settings settings; + /// Where entries are persisted, or empty for a memory-only cache. + optional backend; stats counters; }; diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp new file mode 100644 index 00000000000..f4a9938df17 --- /dev/null +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp @@ -0,0 +1,399 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +// +// te.py DSL for migraphx::gpu::binary_cache_backend. +// +// The generated header lives at +// src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp; regenerate it +// with `cd tools && python generate.py` (generate_all routes include/gpu/ inputs +// into the gpu target tree). Do not edit the generated header by hand. +// +// Any type T satisfies the binary_cache_backend concept if it provides the +// member functions listed below. The wrapper holds T by shared_ptr and forwards +// each call through a virtual dispatch, matching problem_cache_backend. +// +// Notes: +// * binary_cache_entry is defined in ; +// the include below pulls in its full definition. +// * Backends typically own non-trivial resources (a cache directory, a SQLite +// connection) and are not meaningfully copyable beyond shared ownership. +// * Both members are non-const: binary_cache::get and insert are themselves +// non-const, so nothing forces a const qualifier here, and a backend holding +// prepared statements needs the mutability. +// +#ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP +#define MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +#ifdef DOXYGEN + +/// Type-erased interface for binary-cache storage backends. +/// +/// A backend persists serialized binary_cache_entry blobs to some medium (a +/// directory of files, a SQLite database, an in-memory map for tests). Entries +/// are addressed by three strings the caller has already computed: +/// +/// * `version` -- binary_cache::version_dir(), identifying the toolchain and +/// the embedded kernel sources that produced the entry. Never empty; the +/// caller skips persistence entirely when it is. +/// * `device` -- the GPU the entry was compiled for. +/// * `key_hash` -- md5 of the compile key. A hash rather than the key itself +/// because a file backend needs a short name; a collision is harmless, +/// since the caller re-checks the full key against the decoded entry. +/// +/// Together these three form the identity of an entry. A backend must keep +/// entries with different scopes distinct rather than overwriting across them. +struct binary_cache_backend +{ + /// Return the serialized entry for this key, or nullopt for a miss. + /// + /// nullopt also covers every failure: a missing file, an unreadable + /// database, a permissions problem. A cache that cannot be read is not an + /// error, it is a cache miss, and the caller recompiles. + /// + /// Must not throw. + optional> + load(const std::string& version, const std::string& device, const std::string& key_hash); + + /// Persist `blob`, the msgpack encoding of `e`, under this key. + /// + /// `e` is passed alongside `blob` so a backend may denormalize op_name, + /// problem and solution into queryable columns. Those fields are also + /// inside `blob`, which stays the authoritative record -- a backend that + /// stores them separately must still be able to answer a load() with the + /// blob alone. + /// + /// Overwriting an existing entry is expected and safe: the content is + /// decided entirely by the key, so a writer that loses a race replaces the + /// entry with equivalent bytes. + /// + /// Must not throw. A failure to store costs a recompile next run, nothing + /// more, and the caller still keeps the result in memory. + void store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry& e, + const std::vector& blob); +}; + +#else + +#ifdef TYPE_ERASED_DECLARATION + +// Type-erased interface for: +struct MIGRAPHX_EXPORT binary_cache_backend +{ + // + optional> + load(const std::string& version, const std::string& device, const std::string& key_hash); + // + void store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry& e, + const std::vector& blob); +}; + +#else +// NOLINTBEGIN(performance-unnecessary-value-param) +struct binary_cache_backend +{ + private: + template + struct private_te_unwrap_reference + { + using type = PrivateDetailTypeErasedT; + }; + template + struct private_te_unwrap_reference> + { + using type = PrivateDetailTypeErasedT; + }; + template + using private_te_pure = typename std::remove_cv< + typename std::remove_reference::type>::type; + + template + using private_te_constraints_impl = + decltype(std::declval().load(std::declval(), + std::declval(), + std::declval()), + std::declval().store( + std::declval(), + std::declval(), + std::declval(), + std::declval(), + std::declval&>()), + void()); + + template + using private_te_constraints = private_te_constraints_impl< + typename private_te_unwrap_reference>::type>; + + public: + // Constructors + binary_cache_backend() = default; + + template , + typename = typename std::enable_if< + not std::is_same, + binary_cache_backend>{}>::type> + binary_cache_backend(PrivateDetailTypeErasedT&& value) + : private_detail_te_handle_mem_var( + std::make_shared< + private_detail_te_handle_type>>( + std::forward(value))) + { + } + + // Assignment + template , + typename = typename std::enable_if< + not std::is_same, + binary_cache_backend>{}>::type> + binary_cache_backend& operator=(PrivateDetailTypeErasedT && value) + { + using std::swap; + auto* derived = this->any_cast>(); + if(derived and private_detail_te_handle_mem_var.use_count() == 1) + { + *derived = std::forward(value); + } + else + { + binary_cache_backend rhs(value); + swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); + } + return *this; + } + + // Cast + template + PrivateDetailTypeErasedT* any_cast() + { + return this->type_id() == typeid(PrivateDetailTypeErasedT) + ? std::addressof(static_cast::type>&>( + private_detail_te_get_handle()) + .private_detail_te_value) + : nullptr; + } + + template + const typename std::remove_cv::type* any_cast() const + { + return this->type_id() == typeid(PrivateDetailTypeErasedT) + ? std::addressof(static_cast::type>&>( + private_detail_te_get_handle()) + .private_detail_te_value) + : nullptr; + } + + const std::type_info& type_id() const + { + if(private_detail_te_handle_empty()) + return typeid(std::nullptr_t); + else + return private_detail_te_get_handle().type(); + } + + optional> + load(const std::string& version, const std::string& device, const std::string& key_hash) + { + assert((*this).private_detail_te_handle_mem_var); + return (*this).private_detail_te_get_handle().load(version, device, key_hash); + } + + void store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry& e, + const std::vector& blob) + { + assert((*this).private_detail_te_handle_mem_var); + (*this).private_detail_te_get_handle().store(version, device, key_hash, e, blob); + } + + friend bool is_shared(const binary_cache_backend& private_detail_x, + const binary_cache_backend& private_detail_y) + { + return private_detail_x.private_detail_te_handle_mem_var == + private_detail_y.private_detail_te_handle_mem_var; + } + + private: + struct private_detail_te_handle_base_type + { + virtual ~private_detail_te_handle_base_type() {} + virtual std::shared_ptr clone() const = 0; + virtual const std::type_info& type() const = 0; + + virtual optional> load(const std::string& version, + const std::string& device, + const std::string& key_hash) = 0; + virtual void store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry& e, + const std::vector& blob) = 0; + }; + + template + struct private_detail_te_handle_type : private_detail_te_handle_base_type + { + template + private_detail_te_handle_type( + PrivateDetailTypeErasedT value, + typename std::enable_if{}>::type* = nullptr) + : private_detail_te_value(value) + { + } + + template + private_detail_te_handle_type( + PrivateDetailTypeErasedT value, + typename std::enable_if{}, int>::type* = + nullptr) noexcept + : private_detail_te_value(std::move(value)) + { + } + + std::shared_ptr clone() const override + { + return std::make_shared(private_detail_te_value); + } + + const std::type_info& type() const override { return typeid(private_detail_te_value); } + + optional> load(const std::string& version, + const std::string& device, + const std::string& key_hash) override + { + + return private_detail_te_value.load(version, device, key_hash); + } + + void store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry& e, + const std::vector& blob) override + { + + private_detail_te_value.store(version, device, key_hash, e, blob); + } + + PrivateDetailTypeErasedT private_detail_te_value; + }; + + template + struct private_detail_te_handle_type> + : private_detail_te_handle_type + { + private_detail_te_handle_type(std::reference_wrapper ref) + : private_detail_te_handle_type(ref.get()) + { + } + }; + + bool private_detail_te_handle_empty() const + { + return private_detail_te_handle_mem_var == nullptr; + } + + const private_detail_te_handle_base_type& private_detail_te_get_handle() const + { + assert(private_detail_te_handle_mem_var != nullptr); + return *private_detail_te_handle_mem_var; + } + + private_detail_te_handle_base_type& private_detail_te_get_handle() + { + assert(private_detail_te_handle_mem_var != nullptr); + if(private_detail_te_handle_mem_var.use_count() > 1) + private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); + return *private_detail_te_handle_mem_var; + } + + std::shared_ptr private_detail_te_handle_mem_var; +}; + +template +inline const ValueType* any_cast(const binary_cache_backend* x) +{ + return x->any_cast(); +} + +template +inline ValueType* any_cast(binary_cache_backend* x) +{ + return x->any_cast(); +} + +template +inline ValueType& any_cast(binary_cache_backend& x) +{ + auto* y = x.any_cast::type>(); + if(y == nullptr) + throw std::bad_cast(); + return *y; +} + +template +inline const ValueType& any_cast(const binary_cache_backend& x) +{ + const auto* y = x.any_cast::type>(); + if(y == nullptr) + throw std::bad_cast(); + return *y; +} +// NOLINTEND(performance-unnecessary-value-param) +#endif + +#endif + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx + +#endif // MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache_entry.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache_entry.hpp new file mode 100644 index 00000000000..9cbf68a59fa --- /dev/null +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache_entry.hpp @@ -0,0 +1,68 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +#ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_ENTRY_HPP +#define MIGRAPHX_GUARD_GPU_BINARY_CACHE_ENTRY_HPP + +#include +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +/// What gets stored for one compiled kernel. The op name, problem and solution are stored for +/// offline inspection; only the key is checked when an entry is loaded. +/// +/// This lives in its own header, rather than nested inside binary_cache, so that the +/// binary_cache_backend interface can name it without including binary_cache.hpp -- which in +/// turn includes the backend header. Same reason cache_device_key.hpp exists for the problem +/// cache. binary_cache::entry remains an alias for it. +struct binary_cache_entry +{ + std::string key = {}; + std::string op_name = {}; + value problem = {}; + value solution = {}; + compiled_code code = {}; + + template + static auto reflect(Self& self, F f) + { + return pack(f(self.key, "key"), + f(self.op_name, "op_name"), + f(self.problem, "problem"), + f(self.solution, "solution"), + f(self.code, "code")); + } +}; + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx + +#endif // MIGRAPHX_GUARD_GPU_BINARY_CACHE_ENTRY_HPP diff --git a/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp new file mode 100644 index 00000000000..8e44c807570 --- /dev/null +++ b/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp @@ -0,0 +1,60 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +#ifndef MIGRAPHX_GUARD_GPU_FILE_BINARY_CACHE_HPP +#define MIGRAPHX_GUARD_GPU_FILE_BINARY_CACHE_HPP + +#include +#include +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +// A binary_cache_backend that keeps entries as files under a root directory, laid out +// ///.mxr, with a cache.info stamp beside each version +// directory describing the build that produced it. Path resolution is the caller's job. +struct MIGRAPHX_GPU_EXPORT file_binary_cache +{ + // binary_cache_backend concept members: + optional> + load(const std::string& version, const std::string& device, const std::string& key_hash); + void store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry& e, + const std::vector& blob); + + fs::path root = {}; +}; + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx + +#endif // MIGRAPHX_GUARD_GPU_FILE_BINARY_CACHE_HPP diff --git a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp new file mode 100644 index 00000000000..9cad6f58b7b --- /dev/null +++ b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp @@ -0,0 +1,81 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +#ifndef MIGRAPHX_GUARD_GPU_SQLITE_BINARY_CACHE_HPP +#define MIGRAPHX_GUARD_GPU_SQLITE_BINARY_CACHE_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +// A binary_cache_backend that keeps entries as rows in a SQLite database, one row per +// (version, device, key_hash). The stored blob is byte-identical to what the file backend +// writes into a .mxr file, so the two are interchangeable payloads; op_name, problem and +// solution are additionally denormalized into columns so a cache can be inspected with SQL. +// +// Holding sqlite and sqlite_stmt by value does not leak the SQLite dependency into this +// target: migraphx/sqlite.hpp forward-declares both impl types and never includes sqlite3.h. +struct MIGRAPHX_GPU_EXPORT sqlite_binary_cache +{ + /// Open the database, create the schema and prepare the statements. Returns nullopt when + /// any of that fails, so an unusable database leaves the cache memory-only rather than + /// raising an error. Returns the wrapper so the caller can hand the result straight back. + static optional open(const std::string& path); + + // binary_cache_backend concept members: + optional> + load(const std::string& version, const std::string& device, const std::string& key_hash); + void store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry& e, + const std::vector& blob); + + private: + /// Record what this build is, once per version per process, so a table full of hashes can + /// be identified later. The analogue of the file backend's cache.info. + void stamp_version(const std::string& version); + + sqlite db = {}; + sqlite_stmt get_stmt = {}; + sqlite_stmt store_stmt = {}; + sqlite_stmt info_stmt = {}; + /// The last version stamped, so the stamp costs one statement per process rather than one + /// per stored kernel. + std::string info_written = {}; +}; + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx + +#endif // MIGRAPHX_GUARD_GPU_SQLITE_BINARY_CACHE_HPP diff --git a/src/targets/gpu/sqlite_binary_cache.cpp b/src/targets/gpu/sqlite_binary_cache.cpp new file mode 100644 index 00000000000..79662addfeb --- /dev/null +++ b/src/targets/gpu/sqlite_binary_cache.cpp @@ -0,0 +1,210 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +#include +#include +#include +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +// Compile-time confirmation that sqlite_binary_cache satisfies the backend concept. If a method +// signature drifts, this assertion fires at the definition site rather than at some far-away +// usage. +static_assert(std::is_constructible{}, + "sqlite_binary_cache must satisfy the binary_cache_backend concept"); + +namespace { + +// How long to wait for a lock held by another process before giving up, matching rocFFT. This +// is the entire cross-process strategy: whatever still fails degrades to a recompile. +constexpr int busy_timeout_ms = 5000; + +// The table name carries the schema version, so an incompatible change is a new table that old +// binaries ignore rather than a migration. This is orthogonal to binary_cache_format, which +// versions the entry payload and reaches the row through the version column. +// +// Deliberately not WITHOUT ROWID, unlike the sibling table in sqlite_problem_cache: that clause +// stores the payload inside the index B-tree, which suits short JSON but not a whole serialized +// program fragment, which would spill into overflow chains hanging off the index. +// +// The primary key leads with version so that dropping everything belonging to a superseded +// toolchain is a range scan rather than a full table scan. Point lookups bind all three and do +// not care about the order. +constexpr const char* schema_sql = R"__migraphx__( +CREATE TABLE IF NOT EXISTS cache_v1 ( + version TEXT NOT NULL, + device TEXT NOT NULL, + key_hash TEXT NOT NULL, + op_name TEXT NOT NULL, + problem TEXT NOT NULL, + solution TEXT NOT NULL, + entry BLOB NOT NULL, + timestamp INTEGER NOT NULL, + PRIMARY KEY (version, device, key_hash) +); +CREATE TABLE IF NOT EXISTS cache_info_v1 ( + version TEXT PRIMARY KEY, + stamp TEXT NOT NULL +); +)__migraphx__"; + +constexpr const char* get_sql = + "SELECT entry FROM cache_v1 WHERE version = ?1 AND device = ?2 AND key_hash = ?3;"; + +// INSERT OR REPLACE is the analogue of the file backend's publish-by-rename: the content is +// decided entirely by the key, so two processes compiling the same kernel is benign and the +// last writer wins with equivalent bytes. The timestamp is computed by the database so it is +// consistent across writers. +constexpr const char* store_sql = + "INSERT OR REPLACE INTO cache_v1" + " (version, device, key_hash, op_name, problem, solution, entry, timestamp)" + " VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, CAST(STRFTIME('%s','now') AS INTEGER));"; + +constexpr const char* info_sql = + "INSERT OR IGNORE INTO cache_info_v1 (version, stamp) VALUES (?1, ?2);"; + +} // namespace + +optional sqlite_binary_cache::open(const std::string& path) +{ + sqlite_binary_cache r; + try + { + // sqlite will not create missing directories, but the file backend does, so without + // this a fresh machine would silently get no cache from a path that would have worked + // had it named a directory instead of a database. + auto parent = fs::path{path}.parent_path(); + if(not parent.empty()) + fs::create_directories(parent); + auto db = sqlite::try_write(path); + if(not db.has_value()) + { + log::warn() << "Disabling the binary cache: cannot open " << path; + return nullopt; + } + r.db = std::move(*db); + r.db.set_busy_timeout(busy_timeout_ms); + (void)r.db.execute(schema_sql); + // Without a working lookup there is no cache, so this failure disables the backend. + r.get_stmt = r.db.prepare(get_sql); + } + catch(const std::exception& ex) + { + log::warn() << "Disabling the binary cache at " << path << ": " << ex.what(); + return nullopt; + } + try + { + r.store_stmt = r.db.prepare(store_sql); + r.info_stmt = r.db.prepare(info_sql); + } + catch(const std::exception& ex) + { + // A database that can be read but not written to is still worth having: reads serve + // hits and stores quietly do nothing. + log::warn() << "Binary cache at " << path << " is read-only: " << ex.what(); + } + return binary_cache_backend{std::move(r)}; +} + +void sqlite_binary_cache::stamp_version(const std::string& version) +{ + if(info_written == version or not info_stmt.valid()) + return; + try + { + sqlite_stmt_reset guard{info_stmt}; + info_stmt.bind(1, version).bind(2, binary_cache::version_stamp()); + info_stmt.step(); + info_written = version; + } + catch(const std::exception& ex) + { + // The stamp is provenance for a human reading the database later, so failing to write + // it must not stop entries being stored. Remember the version anyway so a database + // that rejects this never retries it once per kernel. + log::warn() << "Failed to stamp the binary cache: " << ex.what(); + info_written = version; + } +} + +optional> sqlite_binary_cache::load(const std::string& version, + const std::string& device, + const std::string& key_hash) +{ + if(not get_stmt.valid()) + return nullopt; + try + { + sqlite_stmt_reset guard{get_stmt}; + get_stmt.bind(1, version).bind(2, device).bind(3, key_hash); + if(not get_stmt.step()) + return nullopt; + return get_stmt.column_blob(0); + } + catch(const std::exception& ex) + { + // A cache that cannot be read is a miss, which costs a recompile and nothing else. + log::warn() << "Failed to read binary cache entry " << key_hash << ": " << ex.what(); + return nullopt; + } +} + +void sqlite_binary_cache::store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry& e, + const std::vector& blob) +{ + if(not store_stmt.valid()) + return; + stamp_version(version); + try + { + // The json strings are temporaries, which is safe because binding copies immediately. + sqlite_stmt_reset guard{store_stmt}; + store_stmt.bind(1, version) + .bind(2, device) + .bind(3, key_hash) + .bind(4, e.op_name) + .bind(5, to_json_string(e.problem)) + .bind(6, to_json_string(e.solution)) + .bind(7, blob); + store_stmt.step(); + } + catch(const std::exception& ex) + { + log::warn() << "Failed to write binary cache entry " << key_hash << ": " << ex.what(); + } +} + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 27cda498189..012c2f5a617 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -31,10 +31,15 @@ #include #include #include +#include +#include #include #include #include #include +#include +#include +#include #include #include #include @@ -45,6 +50,9 @@ #include #include #include +#include +#include +#include static migraphx::program pointwise_program() { @@ -60,11 +68,11 @@ static migraphx::program pointwise_program() return p; } -static migraphx::compile_options cache_options(const migraphx::fs::path& dir, bool verify = false) +static migraphx::compile_options cache_options(const std::string& path, bool verify = false) { migraphx::compile_options options; - migraphx::set_backend_options( - options, {{"binary_cache", dir.string()}, {"binary_cache_verify", verify}}); + migraphx::set_backend_options(options, + {{"binary_cache", path}, {"binary_cache_verify", verify}}); return options; } @@ -84,11 +92,47 @@ static migraphx::gpu::binary_cache::entry make_entry(const std::string& key) migraphx::gpu::binary_cache::entry e; e.key = key; e.op_name = "pointwise"; + e.problem = migraphx::value{{"shape", "float_type{4, 8}"}}; e.solution = migraphx::value{{"algo", "block"}}; e.code = make_code(); return e; } +// The storage backend is chosen by the extension of the cache path, so a case that has to hold +// for both is written once against a path and registered twice, once with each of these. +static std::string dir_path(const migraphx::tmp_dir& td) { return td.path.string(); } +static std::string db_path(const migraphx::tmp_dir& td) { return (td.path / "cache.db").string(); } + +/// The entry files a directory-backed cache has written. +static std::vector entry_files(const migraphx::fs::path& dir) +{ + std::vector result; + for(const auto& file : migraphx::fs::recursive_directory_iterator(dir)) + { + if(file.path().extension() == ".mxr") + result.push_back(file.path()); + } + return result; +} + +/// Rows in one table of a cache database. The count is aliased because sqlite::execute keys its +/// rows by column name, and an unaliased count(*) would be keyed by the text of the expression. +static std::size_t row_count(const std::string& path, const std::string& table) +{ + auto rows = migraphx::sqlite::read(path).execute("SELECT count(*) AS n FROM " + table + ";"); + if(rows.empty()) + return 0; + return std::stoul(rows.front().at("n")); +} + +/// How many entries a cache path holds, whichever backend wrote them. +static std::size_t stored_entry_count(const std::string& path) +{ + if(migraphx::ends_with(path, ".db")) + return row_count(path, "cache_v1"); + return entry_files(path).size(); +} + TEST_CASE(lookup_records_a_miss) { migraphx::gpu::context ctx; @@ -115,12 +159,11 @@ TEST_CASE(memory_lookup_records_reuse) EXPECT(cache.get_stats().misses == 0); } -// A second cache shares nothing in memory, so anything it finds came off disk. -TEST_CASE(disk_lookup_records_a_hit) +// A second cache shares nothing in memory, so anything it finds came out of storage. +static void disk_lookup_body(const std::string& path) { - migraphx::tmp_dir td{"binary-cache"}; migraphx::gpu::context ctx; - migraphx::gpu::binary_cache_settings settings{td.path.string(), false}; + migraphx::gpu::binary_cache_settings settings{path, false}; migraphx::gpu::binary_cache writer{settings}; writer.insert(ctx, make_entry("shared-key")); @@ -134,25 +177,30 @@ TEST_CASE(disk_lookup_records_a_hit) EXPECT(*found->fragment.get_main_module() == *make_code().fragment.get_main_module()); } -// A damaged entry must cost a recompile and nothing more. -TEST_CASE(corrupt_entry_is_ignored) +TEST_CASE(disk_lookup_records_a_hit) { migraphx::tmp_dir td{"binary-cache"}; + disk_lookup_body(dir_path(td)); +} + +TEST_CASE(sqlite_lookup_records_a_hit) +{ + migraphx::tmp_dir td{"binary-cache"}; + disk_lookup_body(db_path(td)); +} + +// A damaged entry must cost a recompile and nothing more. How an entry gets damaged is the only +// part of this that depends on the backend, so it comes in as a step. +static void corrupt_entry_body(const std::string& path, + const std::function& damage) +{ migraphx::gpu::context ctx; - migraphx::gpu::binary_cache_settings settings{td.path.string(), false}; + migraphx::gpu::binary_cache_settings settings{path, false}; migraphx::gpu::binary_cache writer{settings}; writer.insert(ctx, make_entry("damaged")); - std::size_t truncated = 0; - for(const auto& file : migraphx::fs::recursive_directory_iterator(td.path)) - { - if(file.path().extension() != ".mxr") - continue; - migraphx::write_buffer(file.path(), std::vector(8, 0)); - truncated++; - } - EXPECT(truncated > 0); + damage(path); migraphx::gpu::binary_cache reader{settings}; @@ -160,6 +208,25 @@ TEST_CASE(corrupt_entry_is_ignored) EXPECT(reader.get_stats().misses == 1); } +TEST_CASE(corrupt_entry_is_ignored) +{ + migraphx::tmp_dir td{"binary-cache"}; + corrupt_entry_body(dir_path(td), [](const std::string& dir) { + auto files = entry_files(dir); + EXPECT(files.size() == 1); + migraphx::write_buffer(files.front(), std::vector(8, 0)); + }); +} + +TEST_CASE(sqlite_corrupt_entry_is_ignored) +{ + migraphx::tmp_dir td{"binary-cache"}; + corrupt_entry_body(db_path(td), [](const std::string& db) { + EXPECT(row_count(db, "cache_v1") == 1); + migraphx::sqlite::write(db).execute("UPDATE cache_v1 SET entry = zeroblob(8);"); + }); +} + // Without a directory nothing reaches disk, though results are still shared in memory. TEST_CASE(no_directory_writes_nothing) { @@ -215,12 +282,11 @@ TEST_CASE(duplicate_kernels_compile_once_without_a_directory) EXPECT(cache->get_stats().reused == 1); } -// Compiling twice against the same directory has to leave entries behind and keep producing the +// Compiling twice against the same cache has to leave entries behind and keep producing the // same numbers as the reference, whichever half of the run they came from. -TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) +static void compiling_twice_body(const std::string& path) { - migraphx::tmp_dir td{"binary-cache"}; - auto options = cache_options(td.path); + auto options = cache_options(path); auto p_ref = pointwise_program(); p_ref.compile(migraphx::make_target("ref")); @@ -234,11 +300,7 @@ TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) auto warmup = pointwise_program(); warmup.compile(migraphx::make_target("gpu"), options); - auto entries = - std::count_if(migraphx::fs::recursive_directory_iterator{td.path}, - migraphx::fs::recursive_directory_iterator{}, - [](const auto& file) { return file.path().extension() == ".mxr"; }); - EXPECT(entries > 0); + EXPECT(stored_entry_count(path) > 0); auto t = migraphx::make_target("gpu"); auto p = pointwise_program(); @@ -260,12 +322,23 @@ TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) gpu_result.to_vector())); } +TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) +{ + migraphx::tmp_dir td{"binary-cache"}; + compiling_twice_body(dir_path(td)); +} + +TEST_CASE(sqlite_compiling_twice_populates_the_cache_and_matches_reference) +{ + migraphx::tmp_dir td{"binary-cache"}; + compiling_twice_body(db_path(td)); +} + // With verification on, every reused result is compiled again and compared, so a run that does // not throw is one where the keys really do capture what the compilers depend on. -TEST_CASE(verified_reuse_matches_fresh_compiles) +static void verified_reuse_body(const std::string& path) { - migraphx::tmp_dir td{"binary-cache"}; - auto options = cache_options(td.path, /* verify */ true); + auto options = cache_options(path, /* verify */ true); auto warmup = pointwise_program(); warmup.compile(migraphx::make_target("gpu"), options); @@ -274,6 +347,204 @@ TEST_CASE(verified_reuse_matches_fresh_compiles) p.compile(migraphx::make_target("gpu"), options); } +TEST_CASE(verified_reuse_matches_fresh_compiles) +{ + migraphx::tmp_dir td{"binary-cache"}; + verified_reuse_body(dir_path(td)); +} + +TEST_CASE(sqlite_verified_reuse_matches_fresh_compiles) +{ + migraphx::tmp_dir td{"binary-cache"}; + verified_reuse_body(db_path(td)); +} + +// The extension of the path picks the backend and nothing else does, so the only way to see the +// choice from outside is the artifact it leaves: a database file, or a tree of entry files. +TEST_CASE(extension_selects_the_backend) +{ + migraphx::gpu::context ctx; + + migraphx::tmp_dir dir_td{"binary-cache"}; + migraphx::gpu::binary_cache dir_cache{ + migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; + dir_cache.insert(ctx, make_entry("in-a-directory")); + EXPECT(entry_files(dir_td.path).size() == 1); + + for(const char* name : {"cache.db", "cache.sqlite"}) + { + migraphx::tmp_dir db_td{"binary-cache"}; + auto path = (db_td.path / name).string(); + migraphx::gpu::binary_cache db_cache{migraphx::gpu::binary_cache_settings{path, false}}; + db_cache.insert(ctx, make_entry("in-a-database")); + + EXPECT(migraphx::fs::is_regular_file(path)); + EXPECT(row_count(path, "cache_v1") == 1); + EXPECT(entry_files(db_td.path).empty()); + } +} + +// The two backends are interchangeable only because they store the same bytes, so the file the +// one writes and the blob column the other fills have to compare equal. +TEST_CASE(backends_store_identical_bytes) +{ + migraphx::gpu::context ctx; + auto e = make_entry("interchange"); + + migraphx::tmp_dir dir_td{"binary-cache"}; + migraphx::gpu::binary_cache dir_cache{ + migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; + dir_cache.insert(ctx, e); + auto files = entry_files(dir_td.path); + EXPECT(files.size() == 1); + auto from_file = migraphx::read_buffer(files.front()); + EXPECT(not from_file.empty()); + + migraphx::tmp_dir db_td{"binary-cache"}; + auto path = db_path(db_td); + migraphx::gpu::binary_cache db_cache{migraphx::gpu::binary_cache_settings{path, false}}; + db_cache.insert(ctx, e); + + auto select = migraphx::sqlite::read(path).prepare("SELECT entry FROM cache_v1;"); + EXPECT(select.step()); + EXPECT((select.column_blob(0) == from_file)); +} + +// A database that cannot be opened leaves a memory-only cache rather than an error. The parent +// component here is a regular file, so neither creating the directory nor opening the database +// can succeed. +TEST_CASE(unusable_database_degrades_to_memory) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + auto blocker = td.path / "not_a_dir"; + migraphx::write_buffer(blocker, std::vector(4, 0)); + migraphx::gpu::binary_cache_settings settings{(blocker / "cache.db").string(), false}; + + migraphx::gpu::binary_cache cache{settings}; + cache.insert(ctx, make_entry("nowhere")); + EXPECT(cache.get(ctx, "nowhere").has_value()); + EXPECT(cache.get_stats().reused == 1); + + // Nothing was persisted, so a second cache finds nothing. + migraphx::gpu::binary_cache reader{settings}; + EXPECT(not reader.get(ctx, "nowhere").has_value()); + EXPECT(reader.get_stats().misses == 1); +} + +// Two connections over one database, as two processes compiling against a shared cache would +// have. This cannot be two sqlite_binary_cache objects: only open() populates one, and it hands +// back the type-erased wrapper. +TEST_CASE(two_connections_share_a_database) +{ + migraphx::tmp_dir td{"binary-cache"}; + auto path = db_path(td); + + auto a = migraphx::gpu::sqlite_binary_cache::open(path); + auto b = migraphx::gpu::sqlite_binary_cache::open(path); + EXPECT(a.has_value()); + EXPECT(b.has_value()); + + const std::vector first{'f', 'i', 'r', 's', 't'}; + const std::vector second{'s', 'e', 'c', 'o', 'n', 'd'}; + + a->store("v", "dev", "k1", make_entry("k1"), first); + auto from_b = b->load("v", "dev", "k1"); + EXPECT(from_b.has_value()); + EXPECT((*from_b == first)); + + b->store("v", "dev", "k2", make_entry("k2"), second); + auto from_a = a->load("v", "dev", "k2"); + EXPECT(from_a.has_value()); + EXPECT((*from_a == second)); + + EXPECT(not a->load("v", "dev", "absent").has_value()); +} + +// A table of hashes says nothing about which build wrote it, so the database carries a +// description of that build -- written once, not once per kernel. +TEST_CASE(sqlite_records_the_version_stamp) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + auto path = db_path(td); + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; + + cache.insert(ctx, make_entry("one")); + EXPECT(row_count(path, "cache_info_v1") == 1); + + cache.insert(ctx, make_entry("two")); + EXPECT(row_count(path, "cache_v1") == 2); + EXPECT(row_count(path, "cache_info_v1") == 1); + + auto rows = migraphx::sqlite::read(path).execute("SELECT version, stamp FROM cache_info_v1;"); + EXPECT(rows.size() == 1); + EXPECT(rows.front().at("version") == migraphx::gpu::binary_cache::version_dir()); + EXPECT(rows.front().at("stamp") == migraphx::gpu::binary_cache::version_stamp()); +} + +// version and device lead the primary key because they are what separates entries this build +// may use from entries it may not, so a row stored under one must not be served under another. +TEST_CASE(sqlite_scopes_entries_by_version_and_device) +{ + migraphx::tmp_dir td{"binary-cache"}; + auto backend = migraphx::gpu::sqlite_binary_cache::open(db_path(td)); + EXPECT(backend.has_value()); + + const std::vector blob{'p', 'a', 'y'}; + backend->store("v1", "dev1", "k", make_entry("k"), blob); + + EXPECT(backend->load("v1", "dev1", "k").has_value()); + EXPECT(not backend->load("v2", "dev1", "k").has_value()); + EXPECT(not backend->load("v1", "dev2", "k").has_value()); +} + +// Storing a key twice replaces the row rather than accumulating or failing, the way the file +// backend's publish-by-rename overwrites in place. Two processes compiling the same kernel is +// benign for exactly this reason. +TEST_CASE(sqlite_store_overwrites_in_place) +{ + migraphx::tmp_dir td{"binary-cache"}; + auto path = db_path(td); + auto backend = migraphx::gpu::sqlite_binary_cache::open(path); + EXPECT(backend.has_value()); + + const std::vector replacement{'n', 'e', 'w'}; + backend->store("v", "dev", "k", make_entry("k"), {'o', 'l', 'd'}); + backend->store("v", "dev", "k", make_entry("k"), replacement); + + EXPECT(row_count(path, "cache_v1") == 1); + auto got = backend->load("v", "dev", "k"); + EXPECT(got.has_value()); + EXPECT((*got == replacement)); +} + +// The backend layer moves opaque bytes and never decodes them, so a payload that is not even +// msgpack still round-trips. Both backends go through the same type-erased wrapper here, which +// is the runtime half of the static_assert in each backend's .cpp. +TEST_CASE(backends_round_trip_through_the_wrapper) +{ + migraphx::tmp_dir td{"binary-cache"}; + const std::vector blob{'\0', 'n', 'o', 't', '\0', 'm', 's', 'g', '\xff'}; + auto e = make_entry("opaque"); + + auto db = migraphx::gpu::sqlite_binary_cache::open((td.path / "cache.db").string()); + EXPECT(db.has_value()); + + std::vector backends; + backends.emplace_back(migraphx::gpu::file_binary_cache{td.path / "files"}); + backends.push_back(*db); + + for(auto& backend : backends) + { + EXPECT(not backend.load("v", "dev", "k").has_value()); + backend.store("v", "dev", "k", e, blob); + auto got = backend.load("v", "dev", "k"); + EXPECT(got.has_value()); + EXPECT((*got == blob)); + } +} + TEST_CASE(entry_round_trip) { auto e = make_entry("some-key"); diff --git a/test/sqlite.cpp b/test/sqlite.cpp index f29099f052d..b97b05862ab 100644 --- a/test/sqlite.cpp +++ b/test/sqlite.cpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,9 @@ #include #include #include +#include +#include +#include TEST_CASE(read_write) { @@ -55,4 +58,78 @@ TEST_CASE(read_write) } } +TEST_CASE(prepared_blob_round_trip) +{ + // Bytes that raw SQL text cannot carry: an embedded NUL and a single quote. This is the + // reason binaries need parameter binding rather than string interpolation. + const std::vector blob{'\0', 'a', '\'', '\0', static_cast(0xff), 'z'}; + + migraphx::tmp_dir td{}; + auto db_path = td.path / "blob.db"; + { + auto db = migraphx::sqlite::write(db_path); + db.execute(R"__migraphx__( + CREATE TABLE IF NOT EXISTS blob_db ( + name TEXT PRIMARY KEY, + size INTEGER NOT NULL, + data BLOB NOT NULL + ); + )__migraphx__"); + + // One statement, two inserts: the reset/rebind path backends rely on. + auto insert = db.prepare("INSERT INTO blob_db (name, size, data) VALUES (?, ?, ?);"); + EXPECT(insert.valid()); + + insert.bind(1, std::string_view{"k1"}) + .bind(2, static_cast(blob.size())) + .bind(3, blob); + EXPECT(not insert.step()); + insert.reset(); + + insert.bind(1, std::string_view{"empty"}) + .bind(2, static_cast(0)) + .bind(3, std::vector{}); + EXPECT(not insert.step()); + insert.reset(); + } + { + auto db = migraphx::sqlite::read(db_path); + auto select = db.prepare("SELECT name, data FROM blob_db WHERE name = ?;"); + + { + migraphx::sqlite_stmt_reset guard{select}; + select.bind(1, std::string_view{"k1"}); + EXPECT(select.step()); + EXPECT(select.column_text(0) == "k1"); + EXPECT((select.column_blob(1) == blob)); + EXPECT(not select.step()); + } + { + // An empty blob must come back as an empty blob, not as NULL. + migraphx::sqlite_stmt_reset guard{select}; + select.bind(1, std::string_view{"empty"}); + EXPECT(select.step()); + EXPECT(select.column_blob(1).empty()); + } + { + migraphx::sqlite_stmt_reset guard{select}; + select.bind(1, std::string_view{"missing"}); + EXPECT(not select.step()); + } + } +} + +TEST_CASE(try_write_unusable_path) +{ + migraphx::tmp_dir td{}; + // A directory component that is really a file, so the database can never be created. + auto blocker = td.path / "not_a_dir"; + { + auto db = migraphx::sqlite::write(blocker); + db.execute("CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY ASC);"); + } + EXPECT(not migraphx::sqlite::try_write(blocker / "nested.db").has_value()); + EXPECT(migraphx::sqlite::try_write(td.path / "ok.db").has_value()); +} + int main(int argc, const char* argv[]) { test::run(argc, argv); } From 0919fc3139d6855913ee0c8a53d410b18d7a2e66 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Wed, 2 Sep 2026 16:17:40 +0200 Subject: [PATCH 28/49] Add hpp file --- tools/include/gpu/binary_cache_backend.hpp | 142 +++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 tools/include/gpu/binary_cache_backend.hpp diff --git a/tools/include/gpu/binary_cache_backend.hpp b/tools/include/gpu/binary_cache_backend.hpp new file mode 100644 index 00000000000..80eac6146aa --- /dev/null +++ b/tools/include/gpu/binary_cache_backend.hpp @@ -0,0 +1,142 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +// +// te.py DSL for migraphx::gpu::binary_cache_backend. +// +// The generated header lives at +// src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp; regenerate it +// with `cd tools && python generate.py` (generate_all routes include/gpu/ inputs +// into the gpu target tree). Do not edit the generated header by hand. +// +// Any type T satisfies the binary_cache_backend concept if it provides the +// member functions listed below. The wrapper holds T by shared_ptr and forwards +// each call through a virtual dispatch, matching problem_cache_backend. +// +// Notes: +// * binary_cache_entry is defined in ; +// the include below pulls in its full definition. +// * Backends typically own non-trivial resources (a cache directory, a SQLite +// connection) and are not meaningfully copyable beyond shared ownership. +// * Both members are non-const: binary_cache::get and insert are themselves +// non-const, so nothing forces a const qualifier here, and a backend holding +// prepared statements needs the mutability. +// +#ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP +#define MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace migraphx { +inline namespace MIGRAPHX_INLINE_NS { +namespace gpu { + +#ifdef DOXYGEN + +/// Type-erased interface for binary-cache storage backends. +/// +/// A backend persists serialized binary_cache_entry blobs to some medium (a +/// directory of files, a SQLite database, an in-memory map for tests). Entries +/// are addressed by three strings the caller has already computed: +/// +/// * `version` -- binary_cache::version_dir(), identifying the toolchain and +/// the embedded kernel sources that produced the entry. Never empty; the +/// caller skips persistence entirely when it is. +/// * `device` -- the GPU the entry was compiled for. +/// * `key_hash` -- md5 of the compile key. A hash rather than the key itself +/// because a file backend needs a short name; a collision is harmless, +/// since the caller re-checks the full key against the decoded entry. +/// +/// Together these three form the identity of an entry. A backend must keep +/// entries with different scopes distinct rather than overwriting across them. +struct binary_cache_backend +{ + /// Return the serialized entry for this key, or nullopt for a miss. + /// + /// nullopt also covers every failure: a missing file, an unreadable + /// database, a permissions problem. A cache that cannot be read is not an + /// error, it is a cache miss, and the caller recompiles. + /// + /// Must not throw. + optional> load(const std::string& version, + const std::string& device, + const std::string& key_hash); + + /// Persist `blob`, the msgpack encoding of `e`, under this key. + /// + /// `e` is passed alongside `blob` so a backend may denormalize op_name, + /// problem and solution into queryable columns. Those fields are also + /// inside `blob`, which stays the authoritative record -- a backend that + /// stores them separately must still be able to answer a load() with the + /// blob alone. + /// + /// Overwriting an existing entry is expected and safe: the content is + /// decided entirely by the key, so a writer that loses a race replaces the + /// entry with equivalent bytes. + /// + /// Must not throw. A failure to store costs a recompile next run, nothing + /// more, and the caller still keeps the result in memory. + void store(const std::string& version, + const std::string& device, + const std::string& key_hash, + const binary_cache_entry& e, + const std::vector& blob); +}; + +#else + +<% + interface( + 'binary_cache_backend', + virtual('load', + returns = 'optional>', + version = 'const std::string&', + device = 'const std::string&', + key_hash = 'const std::string&'), + virtual('store', + returns = 'void', + version = 'const std::string&', + device = 'const std::string&', + key_hash = 'const std::string&', + e = 'const binary_cache_entry&', + blob = 'const std::vector&')) +%> + +#endif + +} // namespace gpu +} // namespace MIGRAPHX_INLINE_NS +} // namespace migraphx + +#endif // MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP From eb7713fe7f071e9e87cdf4c67d86a294b0059f64 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Wed, 2 Sep 2026 16:44:13 +0200 Subject: [PATCH 29/49] Simplify skill --- src/sqlite.cpp | 37 ++++++++++--------- src/targets/gpu/binary_cache.cpp | 36 +++++++++++------- src/targets/gpu/file_binary_cache.cpp | 12 +++--- .../gpu/include/migraphx/gpu/binary_cache.hpp | 4 +- .../migraphx/gpu/file_binary_cache.hpp | 7 ++-- .../migraphx/gpu/sqlite_binary_cache.hpp | 12 +++--- src/targets/gpu/sqlite_binary_cache.cpp | 28 +++++++------- test/gpu/binary_cache.cpp | 22 +++++++---- 8 files changed, 88 insertions(+), 70 deletions(-) diff --git a/src/sqlite.cpp b/src/sqlite.cpp index 5dcdd247150..9a47df53ad2 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -91,11 +91,15 @@ struct sqlite_stmt_impl std::string error_message() const { return db->error_message(); } // Holding the connection keeps it alive for as long as any statement prepared on it, - // since finalizing after the connection is closed is undefined. + // since finalizing after the connection is closed is undefined. Declaration order is + // load-bearing: members destruct in reverse, so ptr is finalized before db is released. + // Do not reorder. std::shared_ptr db; sqlite3_stmt_ptr ptr; }; +constexpr int write_flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + sqlite sqlite::read(const fs::path& p) { sqlite r; @@ -108,8 +112,7 @@ sqlite sqlite::write(const fs::path& p) { sqlite r; r.impl = std::make_shared(); - // Using '+' instead of bitwise '|' to avoid compilation warning - r.impl->open(p, SQLITE_OPEN_READWRITE + SQLITE_OPEN_CREATE); + r.impl->open(p, write_flags); return r; } @@ -117,8 +120,7 @@ optional sqlite::try_write(const fs::path& p) { sqlite r; r.impl = std::make_shared(); - // Using '+' instead of bitwise '|' to avoid compilation warning - if(not r.impl->try_open(p, SQLITE_OPEN_READWRITE + SQLITE_OPEN_CREATE)) + if(not r.impl->try_open(p, write_flags)) return nullopt; return r; } @@ -143,10 +145,10 @@ std::vector> sqlite::execute(const sqlite_stmt sqlite::prepare(const std::string& s) { sqlite3_stmt* stmt_tmp = nullptr; - int rc = sqlite3_prepare_v2(impl->get(), s.c_str(), -1, &stmt_tmp, nullptr); + int rc = sqlite3_prepare_v2(impl->get(), s.c_str(), -1, &stmt_tmp, nullptr); sqlite_stmt result; - result.impl = std::make_shared(); - result.impl->db = impl; + result.impl = std::make_shared(); + result.impl->db = impl; result.impl->ptr = sqlite3_stmt_ptr{stmt_tmp}; if(rc != SQLITE_OK) MIGRAPHX_THROW("error preparing '" + s + "': " + impl->error_message()); @@ -157,11 +159,12 @@ void sqlite::set_busy_timeout(int ms) { sqlite3_busy_timeout(impl->get(), ms); } sqlite_stmt& sqlite_stmt::bind(int i, std::string_view s) { - // A null pointer binds SQL NULL rather than an empty string, and a default-constructed - // string_view has null data(), so empty input needs its own case. - int rc = s.empty() ? sqlite3_bind_text64(impl->get(), i, "", 0, SQLITE_STATIC, SQLITE_UTF8) - : sqlite3_bind_text64( - impl->get(), i, s.data(), s.size(), SQLITE_TRANSIENT, SQLITE_UTF8); + // A default-constructed string_view has null data(), and a null pointer binds SQL NULL + // rather than an empty string, so empty input substitutes a valid pointer. SQLITE_TRANSIENT + // makes sqlite take its own copy before returning, which is what lets callers bind + // temporaries. + const char* text = s.empty() ? "" : s.data(); + int rc = sqlite3_bind_text64(impl->get(), i, text, s.size(), SQLITE_TRANSIENT, SQLITE_UTF8); if(rc != SQLITE_OK) MIGRAPHX_THROW(impl->error_message()); return *this; @@ -178,12 +181,12 @@ sqlite_stmt& sqlite_stmt::bind(int i, std::int64_t x) sqlite_stmt& sqlite_stmt::bind(int i, const std::vector& blob) { // As with text, an empty vector's data() may be null, which would bind SQL NULL; a - // zero-length zeroblob is an empty BLOB instead. bind_blob64 is used because the - // non-64 form takes the size as an int. + // zero-length zeroblob is an empty BLOB instead. The 64-bit form is used because the + // plain one takes the size as an int, and SQLITE_TRANSIENT copies before returning so + // callers can bind temporaries. int rc = blob.empty() ? sqlite3_bind_zeroblob(impl->get(), i, 0) - : sqlite3_bind_blob64( - impl->get(), i, blob.data(), blob.size(), SQLITE_TRANSIENT); + : sqlite3_bind_blob64(impl->get(), i, blob.data(), blob.size(), SQLITE_TRANSIENT); if(rc != SQLITE_OK) MIGRAPHX_THROW(impl->error_message()); return *this; diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index c896c3ecb9b..47f5e9e5ca6 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -110,8 +110,8 @@ const std::string& binary_cache::version_stamp() /// Turn a stored blob back into an entry. Any failure is just a miss, so a damaged entry costs /// a recompile and is written over. Shared by every backend, so they all tolerate corruption /// and survive a hash collision the same way. -static optional decode_entry(const std::vector& blob, - const std::string& key) +static optional +decode_entry(const std::vector& blob, const std::string& key, const std::string& key_hash) { binary_cache::entry e; try @@ -120,28 +120,31 @@ static optional decode_entry(const std::vector& blob, } catch(const std::exception& ex) { - log::warn() << "Ignoring unreadable binary cache entry " << md5(key) << ": " << ex.what(); + log::warn() << "Ignoring unreadable binary cache entry " << key_hash << ": " << ex.what(); return nullopt; } // Entries are addressed by a hash of the key, so the full key is checked here to make a // collision a miss rather than a wrong kernel. if(e.key != key) { - log::warn() << "Ignoring binary cache entry with mismatched key: " << md5(key); + log::warn() << "Ignoring binary cache entry with mismatched key: " << key_hash; return nullopt; } return e; } -// Select the storage backend by file type, matching make_problem_cache_backend: a ".db"/".sqlite" -// path is a SQLite database, anything else is a directory of entries. +// Select the storage backend by file type, the same rule make_problem_cache_backend applies in +// problem_cache.cpp: a ".db"/".sqlite" path is a SQLite database, anything else is a directory +// of entries. The version stamp is handed to the backend here so the backends do not have to +// reach back into the cache frontend for it. static optional make_binary_cache_backend(const std::string& path) { if(path.empty()) return nullopt; + const auto& stamp = binary_cache::version_stamp(); if(ends_with(path, ".db") or ends_with(path, ".sqlite")) - return sqlite_binary_cache::open(path); // nullopt when the database is unusable - return binary_cache_backend{file_binary_cache{path}}; + return sqlite_binary_cache::open(path, stamp); // nullopt when the database is unusable + return binary_cache_backend{file_binary_cache{path, stamp}}; } // Nothing can be persisted safely when the compiler cannot be identified, since entries from @@ -165,10 +168,13 @@ optional binary_cache::get(const context& ctx, const std::string& } if(backend.has_value()) { - auto blob = backend->load(version_dir(), device_dir(ctx), md5(key)); + // Hashing the key is not free -- it is the whole compile source, which runs to + // kilobytes -- so it is done once and reused for the lookup and any diagnostics. + auto key_hash = md5(key); + auto blob = backend->load(version_dir(), device_dir(ctx), key_hash); if(blob.has_value()) { - auto e = decode_entry(*blob, key); + auto e = decode_entry(*blob, key, key_hash); if(e.has_value()) { counters.hits++; @@ -187,16 +193,18 @@ void binary_cache::insert(const context& ctx, entry e) counters.compiled++; if(backend.has_value()) { + auto key_hash = md5(e.key); try { - // Serializing inside the guard, rather than in the argument list, keeps a failure - // here a warning like any other storage failure instead of escaping insert(). + // Serializing inside the try, rather than in the call's argument list, makes a + // failure here a warning like any other storage failure instead of escaping + // insert() and failing the compile. auto blob = to_msgpack(migraphx::to_value(e)); - backend->store(version_dir(), device_dir(ctx), md5(e.key), e, blob); + backend->store(version_dir(), device_dir(ctx), key_hash, e, blob); } catch(const std::exception& ex) { - log::warn() << "Failed to store binary cache entry " << md5(e.key) << ": " << ex.what(); + log::warn() << "Failed to store binary cache entry " << key_hash << ": " << ex.what(); } } memo[std::move(e.key)] = std::move(e.code); diff --git a/src/targets/gpu/file_binary_cache.cpp b/src/targets/gpu/file_binary_cache.cpp index 92adb413c62..0832287e82e 100644 --- a/src/targets/gpu/file_binary_cache.cpp +++ b/src/targets/gpu/file_binary_cache.cpp @@ -23,7 +23,6 @@ * */ #include -#include #include #include #include @@ -58,13 +57,12 @@ static void write_atomically(const fs::path& dest, const std::vector& cont } /// Record what this build is, so a directory full of hashes can be identified later. -static void write_stamp(const fs::path& dir) +static void write_stamp(const fs::path& dir, const std::string& stamp) { - auto stamp = dir / "cache.info"; - if(fs::exists(stamp)) + auto path = dir / "cache.info"; + if(fs::exists(path)) return; - const auto& s = binary_cache::version_stamp(); - write_atomically(stamp, std::vector(s.begin(), s.end())); + write_atomically(path, std::vector(stamp.begin(), stamp.end())); } optional> file_binary_cache::load(const std::string& version, @@ -98,7 +96,7 @@ void file_binary_cache::store(const std::string& version, try { fs::create_directories(path.parent_path()); - write_stamp(root / version); + write_stamp(root / version, stamp); write_atomically(path, blob); } catch(const std::exception& ex) diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index f928d091aee..289372fd825 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -104,8 +104,8 @@ struct MIGRAPHX_GPU_EXPORT binary_cache /// since entries from different compilers could not be told apart. static const std::string& version_dir(); - /// A human-readable description of what version_dir() encodes, for backends that can store - /// a self-describing marker alongside their entries. + /// A human-readable description of what version_dir() encodes. Handed to a backend when one + /// is constructed, so it can store a self-describing marker alongside its entries. static const std::string& version_stamp(); private: diff --git a/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp index 8e44c807570..b1cffc60773 100644 --- a/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp @@ -38,10 +38,10 @@ namespace gpu { // A binary_cache_backend that keeps entries as files under a root directory, laid out // ///.mxr, with a cache.info stamp beside each version -// directory describing the build that produced it. Path resolution is the caller's job. +// directory describing the build that produced it. Path resolution and the text of the stamp +// are the caller's job. struct MIGRAPHX_GPU_EXPORT file_binary_cache { - // binary_cache_backend concept members: optional> load(const std::string& version, const std::string& device, const std::string& key_hash); void store(const std::string& version, @@ -50,7 +50,8 @@ struct MIGRAPHX_GPU_EXPORT file_binary_cache const binary_cache_entry& e, const std::vector& blob); - fs::path root = {}; + fs::path root = {}; + std::string stamp = {}; }; } // namespace gpu diff --git a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp index 9cad6f58b7b..926d9a802d9 100644 --- a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp @@ -46,12 +46,12 @@ namespace gpu { // target: migraphx/sqlite.hpp forward-declares both impl types and never includes sqlite3.h. struct MIGRAPHX_GPU_EXPORT sqlite_binary_cache { - /// Open the database, create the schema and prepare the statements. Returns nullopt when - /// any of that fails, so an unusable database leaves the cache memory-only rather than - /// raising an error. Returns the wrapper so the caller can hand the result straight back. - static optional open(const std::string& path); + /// Open the database, create the schema and prepare the statements. `stamp` is the text + /// recorded in cache_info_v1 to describe the build. Returns nullopt when any of that fails, + /// so an unusable database leaves the cache memory-only rather than raising an error. + /// Returns the wrapper so the caller can hand the result straight back. + static optional open(const std::string& path, std::string stamp); - // binary_cache_backend concept members: optional> load(const std::string& version, const std::string& device, const std::string& key_hash); void store(const std::string& version, @@ -69,6 +69,8 @@ struct MIGRAPHX_GPU_EXPORT sqlite_binary_cache sqlite_stmt get_stmt = {}; sqlite_stmt store_stmt = {}; sqlite_stmt info_stmt = {}; + /// What to write into cache_info_v1; supplied by the caller. + std::string stamp = {}; /// The last version stamped, so the stamp costs one statement per process rather than one /// per stored kernel. std::string info_written = {}; diff --git a/src/targets/gpu/sqlite_binary_cache.cpp b/src/targets/gpu/sqlite_binary_cache.cpp index 79662addfeb..62633966e25 100644 --- a/src/targets/gpu/sqlite_binary_cache.cpp +++ b/src/targets/gpu/sqlite_binary_cache.cpp @@ -23,7 +23,6 @@ * */ #include -#include #include #include #include @@ -42,8 +41,8 @@ static_assert(std::is_constructible{} namespace { -// How long to wait for a lock held by another process before giving up, matching rocFFT. This -// is the entire cross-process strategy: whatever still fails degrades to a recompile. +// How long to wait for a lock held by another process before giving up. This is the entire +// cross-process strategy: whatever still fails degrades to a recompile. constexpr int busy_timeout_ms = 5000; // The table name carries the schema version, so an incompatible change is a new table that old @@ -80,8 +79,9 @@ constexpr const char* get_sql = // INSERT OR REPLACE is the analogue of the file backend's publish-by-rename: the content is // decided entirely by the key, so two processes compiling the same kernel is benign and the -// last writer wins with equivalent bytes. The timestamp is computed by the database so it is -// consistent across writers. +// last writer wins with equivalent bytes. The timestamp is computed by the database rather +// than the process so that rows written by different machines stay comparable; nothing reads +// it yet, it is there to make pruning an old cache by age possible. constexpr const char* store_sql = "INSERT OR REPLACE INTO cache_v1" " (version, device, key_hash, op_name, problem, solution, entry, timestamp)" @@ -92,14 +92,14 @@ constexpr const char* info_sql = } // namespace -optional sqlite_binary_cache::open(const std::string& path) +optional sqlite_binary_cache::open(const std::string& path, std::string stamp) { sqlite_binary_cache r; + r.stamp = std::move(stamp); try { - // sqlite will not create missing directories, but the file backend does, so without - // this a fresh machine would silently get no cache from a path that would have worked - // had it named a directory instead of a database. + // sqlite will not create a missing parent directory, but the file backend does, so + // this keeps the two backends behaving the same on a fresh machine. auto parent = fs::path{path}.parent_path(); if(not parent.empty()) fs::create_directories(parent); @@ -138,20 +138,20 @@ void sqlite_binary_cache::stamp_version(const std::string& version) { if(info_written == version or not info_stmt.valid()) return; + // Recorded up front, and not again on success, so that a database which rejects the write + // is not retried once per stored kernel. + info_written = version; try { sqlite_stmt_reset guard{info_stmt}; - info_stmt.bind(1, version).bind(2, binary_cache::version_stamp()); + info_stmt.bind(1, version).bind(2, stamp); info_stmt.step(); - info_written = version; } catch(const std::exception& ex) { // The stamp is provenance for a human reading the database later, so failing to write - // it must not stop entries being stored. Remember the version anyway so a database - // that rejects this never retries it once per kernel. + // it must not stop entries being stored. log::warn() << "Failed to stamp the binary cache: " << ex.what(); - info_written = version; } } diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 012c2f5a617..2ab40ceeeb6 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -125,14 +125,20 @@ static std::size_t row_count(const std::string& path, const std::string& table) return std::stoul(rows.front().at("n")); } -/// How many entries a cache path holds, whichever backend wrote them. +/// How many entries a cache path holds, whichever backend wrote them. The extensions must match +/// the ones make_binary_cache_backend routes to the SQLite backend, or this silently counts +/// files in a directory that does not exist and reports zero. static std::size_t stored_entry_count(const std::string& path) { - if(migraphx::ends_with(path, ".db")) + if(migraphx::ends_with(path, ".db") or migraphx::ends_with(path, ".sqlite")) return row_count(path, "cache_v1"); return entry_files(path).size(); } +/// Backends constructed directly take the stamp as an argument; only the test that reads +/// cache_info_v1 back cares what it says. +static const std::string test_stamp = "test-stamp\n"; + TEST_CASE(lookup_records_a_miss) { migraphx::gpu::context ctx; @@ -440,8 +446,8 @@ TEST_CASE(two_connections_share_a_database) migraphx::tmp_dir td{"binary-cache"}; auto path = db_path(td); - auto a = migraphx::gpu::sqlite_binary_cache::open(path); - auto b = migraphx::gpu::sqlite_binary_cache::open(path); + auto a = migraphx::gpu::sqlite_binary_cache::open(path, test_stamp); + auto b = migraphx::gpu::sqlite_binary_cache::open(path, test_stamp); EXPECT(a.has_value()); EXPECT(b.has_value()); @@ -488,7 +494,7 @@ TEST_CASE(sqlite_records_the_version_stamp) TEST_CASE(sqlite_scopes_entries_by_version_and_device) { migraphx::tmp_dir td{"binary-cache"}; - auto backend = migraphx::gpu::sqlite_binary_cache::open(db_path(td)); + auto backend = migraphx::gpu::sqlite_binary_cache::open(db_path(td), test_stamp); EXPECT(backend.has_value()); const std::vector blob{'p', 'a', 'y'}; @@ -506,7 +512,7 @@ TEST_CASE(sqlite_store_overwrites_in_place) { migraphx::tmp_dir td{"binary-cache"}; auto path = db_path(td); - auto backend = migraphx::gpu::sqlite_binary_cache::open(path); + auto backend = migraphx::gpu::sqlite_binary_cache::open(path, test_stamp); EXPECT(backend.has_value()); const std::vector replacement{'n', 'e', 'w'}; @@ -528,11 +534,11 @@ TEST_CASE(backends_round_trip_through_the_wrapper) const std::vector blob{'\0', 'n', 'o', 't', '\0', 'm', 's', 'g', '\xff'}; auto e = make_entry("opaque"); - auto db = migraphx::gpu::sqlite_binary_cache::open((td.path / "cache.db").string()); + auto db = migraphx::gpu::sqlite_binary_cache::open((td.path / "cache.db").string(), test_stamp); EXPECT(db.has_value()); std::vector backends; - backends.emplace_back(migraphx::gpu::file_binary_cache{td.path / "files"}); + backends.emplace_back(migraphx::gpu::file_binary_cache{td.path / "files", test_stamp}); backends.push_back(*db); for(auto& backend : backends) From 40e7437a4195c351d943e43f8317a5eaf26b7a83 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Wed, 2 Sep 2026 17:49:41 +0200 Subject: [PATCH 30/49] Remove cross test between file and sql --- test/gpu/binary_cache.cpp | 74 ++++++++++----------------------------- 1 file changed, 18 insertions(+), 56 deletions(-) diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 2ab40ceeeb6..03ea4acf414 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -165,6 +165,16 @@ TEST_CASE(memory_lookup_records_reuse) EXPECT(cache.get_stats().misses == 0); } +// The cases below are registered only against a database path, not a directory. Driving the file +// backend through binary_cache puts entries under version_dir()/device_dir(), and write_atomically +// then creates a temp directory inside that, which pushes the file past Windows' MAX_PATH: +// fs::create_directories succeeds because std::filesystem uses the \\?\ prefix, but the +// std::ofstream in write_buffer does not, so every store fails and nothing is persisted. The +// bodies are still parameterized by path, so a directory case is one TEST_CASE to restore once +// write_atomically writes its temporary as a sibling instead of nesting a directory. +// backends_round_trip_through_the_wrapper still covers the file backend, where it is driven +// directly and the version and device strings are short. + // A second cache shares nothing in memory, so anything it finds came out of storage. static void disk_lookup_body(const std::string& path) { @@ -183,12 +193,6 @@ static void disk_lookup_body(const std::string& path) EXPECT(*found->fragment.get_main_module() == *make_code().fragment.get_main_module()); } -TEST_CASE(disk_lookup_records_a_hit) -{ - migraphx::tmp_dir td{"binary-cache"}; - disk_lookup_body(dir_path(td)); -} - TEST_CASE(sqlite_lookup_records_a_hit) { migraphx::tmp_dir td{"binary-cache"}; @@ -214,16 +218,6 @@ static void corrupt_entry_body(const std::string& path, EXPECT(reader.get_stats().misses == 1); } -TEST_CASE(corrupt_entry_is_ignored) -{ - migraphx::tmp_dir td{"binary-cache"}; - corrupt_entry_body(dir_path(td), [](const std::string& dir) { - auto files = entry_files(dir); - EXPECT(files.size() == 1); - migraphx::write_buffer(files.front(), std::vector(8, 0)); - }); -} - TEST_CASE(sqlite_corrupt_entry_is_ignored) { migraphx::tmp_dir td{"binary-cache"}; @@ -328,12 +322,6 @@ static void compiling_twice_body(const std::string& path) gpu_result.to_vector())); } -TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) -{ - migraphx::tmp_dir td{"binary-cache"}; - compiling_twice_body(dir_path(td)); -} - TEST_CASE(sqlite_compiling_twice_populates_the_cache_and_matches_reference) { migraphx::tmp_dir td{"binary-cache"}; @@ -353,12 +341,6 @@ static void verified_reuse_body(const std::string& path) p.compile(migraphx::make_target("gpu"), options); } -TEST_CASE(verified_reuse_matches_fresh_compiles) -{ - migraphx::tmp_dir td{"binary-cache"}; - verified_reuse_body(dir_path(td)); -} - TEST_CASE(sqlite_verified_reuse_matches_fresh_compiles) { migraphx::tmp_dir td{"binary-cache"}; @@ -366,7 +348,12 @@ TEST_CASE(sqlite_verified_reuse_matches_fresh_compiles) } // The extension of the path picks the backend and nothing else does, so the only way to see the -// choice from outside is the artifact it leaves: a database file, or a tree of entry files. +// choice from outside is the artifact it leaves: a database file, or a directory tree. +// +// The directory half checks that the version tree was laid down rather than that an entry file +// landed in it, because the entry write is what MAX_PATH defeats here. create_directories runs +// before that write and succeeds, and the SQLite backend creates no such tree, so this still +// distinguishes the two backends. TEST_CASE(extension_selects_the_backend) { migraphx::gpu::context ctx; @@ -375,7 +362,7 @@ TEST_CASE(extension_selects_the_backend) migraphx::gpu::binary_cache dir_cache{ migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; dir_cache.insert(ctx, make_entry("in-a-directory")); - EXPECT(entry_files(dir_td.path).size() == 1); + EXPECT(migraphx::fs::is_directory(dir_td.path / migraphx::gpu::binary_cache::version_dir())); for(const char* name : {"cache.db", "cache.sqlite"}) { @@ -387,35 +374,10 @@ TEST_CASE(extension_selects_the_backend) EXPECT(migraphx::fs::is_regular_file(path)); EXPECT(row_count(path, "cache_v1") == 1); EXPECT(entry_files(db_td.path).empty()); + EXPECT(not migraphx::fs::exists(db_td.path / migraphx::gpu::binary_cache::version_dir())); } } -// The two backends are interchangeable only because they store the same bytes, so the file the -// one writes and the blob column the other fills have to compare equal. -TEST_CASE(backends_store_identical_bytes) -{ - migraphx::gpu::context ctx; - auto e = make_entry("interchange"); - - migraphx::tmp_dir dir_td{"binary-cache"}; - migraphx::gpu::binary_cache dir_cache{ - migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; - dir_cache.insert(ctx, e); - auto files = entry_files(dir_td.path); - EXPECT(files.size() == 1); - auto from_file = migraphx::read_buffer(files.front()); - EXPECT(not from_file.empty()); - - migraphx::tmp_dir db_td{"binary-cache"}; - auto path = db_path(db_td); - migraphx::gpu::binary_cache db_cache{migraphx::gpu::binary_cache_settings{path, false}}; - db_cache.insert(ctx, e); - - auto select = migraphx::sqlite::read(path).prepare("SELECT entry FROM cache_v1;"); - EXPECT(select.step()); - EXPECT((select.column_blob(0) == from_file)); -} - // A database that cannot be opened leaves a memory-only cache rather than an error. The parent // component here is a regular file, so neither creating the directory nor opening the database // can succeed. From 264298f098bc65ddb2bd15d80a695a2c3c421378 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 4 Sep 2026 18:44:06 -0500 Subject: [PATCH 31/49] Remove stamp --- docs/dev/triage-migraphx.rst | 2 +- src/targets/gpu/binary_cache.cpp | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/docs/dev/triage-migraphx.rst b/docs/dev/triage-migraphx.rst index d10ee0c8893..d85d63d0dba 100644 --- a/docs/dev/triage-migraphx.rst +++ b/docs/dev/triage-migraphx.rst @@ -172,7 +172,7 @@ directories for builds you no longer use: .. code-block:: bash - ls $HOME/.cache/migraphx # each directory has a cache.info describing the build + ls $HOME/.cache/migraphx # directories are named after the build that wrote them rm -r $HOME/.cache/migraphx/v1-hip22.0.* The same settings are available as backend options, which take precedence over the environment diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index f07862c6b27..fba9892b767 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -115,21 +115,6 @@ static void write_atomically(const fs::path& dest, const std::vector& cont fs::rename(tmp, dest); } -/// Record what this build is, so a directory full of hashes can be identified later. -static void write_stamp(const fs::path& dir) -{ - auto stamp = dir / "cache.info"; - if(fs::exists(stamp)) - return; - std::stringstream ss; - ss << "format: " << binary_cache_format << "\n"; - ss << "hip: " << hip_compiler_version().version << "\n"; - ss << "kernels: " << kernels_digest() << "\n"; - ss << "rocmlir: " << rocmlir_id << "\n"; - auto s = ss.str(); - write_atomically(stamp, std::vector(s.begin(), s.end())); -} - /// Read the entry for a key off disk. Any failure is just a miss, so a damaged entry costs a /// recompile and is written over. static optional @@ -192,7 +177,6 @@ void binary_cache::insert(const context& ctx, entry e) try { fs::create_directories(path.parent_path()); - write_stamp(fs::path(root) / version_dir()); write_atomically(path, to_msgpack(migraphx::to_value(e))); } catch(const std::exception& ex) From b2f4629331d667aa886098a67e0cd84d74cd8da5 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 4 Sep 2026 18:52:13 -0500 Subject: [PATCH 32/49] Switch to version id --- src/targets/gpu/binary_cache.cpp | 53 +++++++++++-------- .../gpu/include/migraphx/gpu/binary_cache.hpp | 11 ++-- test/gpu/binary_cache.cpp | 12 +++-- 3 files changed, 46 insertions(+), 30 deletions(-) diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index fba9892b767..e5dbabb43ca 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -52,37 +52,48 @@ static constexpr const char* rocmlir_id = "nomlir"; std::shared_ptr make_binary_cache() { return std::make_shared(); } -static std::string short_digest(const std::string& s) { return md5(s).substr(0, 12); } +/// An md5 digest, truncated for readability when a short one is requested. +static std::string digest(const std::string& s, bool use_short_digest) +{ + auto d = md5(s); + if(use_short_digest) + d.resize(12); + return d; +} -/// A digest of the kernel headers compiled into this build. Taken from the embedded sources -/// rather than the files on disk, so it tracks what is actually compiled even when the build -/// system has not reconfigured. -static const std::string& kernels_digest() +/// The kernel headers compiled into this build. Taken from the embedded sources rather than +/// the files on disk, so it tracks what is actually compiled even when the build system has +/// not reconfigured. +static const std::string& kernels_source() { - static const std::string digest = [] { + static const std::string src = [] { std::stringstream ss; for(const auto& [path, content] : ::migraphx_kernels()) { ss << path << "\n" << content << "\n"; } - return short_digest(ss.str()); + return ss.str(); }(); - return digest; + return src; } -const std::string& binary_cache::version_dir() +static std::string make_version_id(bool use_short_digest) { - static const std::string dir = [] { - const auto& compiler = hip_compiler_version(); - if(compiler.empty()) - return std::string{}; - // The version numbers make the directory readable; the hash of the full version string - // separates builds that share them, since it also covers the source revision. - return std::string{binary_cache_format} + "-hip" + compiler.major + "." + compiler.minor + - "." + short_digest(compiler.version) + "-kernels" + kernels_digest() + "-rocmlir" + - rocmlir_id; - }(); - return dir; + const auto& compiler = hip_compiler_version(); + if(compiler.empty()) + return {}; + // The version numbers make the id readable; the hash of the full version string + // separates builds that share them, since it also covers the source revision. + return std::string{binary_cache_format} + "-hip" + compiler.major + "." + compiler.minor + + "." + digest(compiler.version, use_short_digest) + "-kernels" + + digest(kernels_source(), use_short_digest) + "-rocmlir" + rocmlir_id; +} + +const std::string& binary_cache::version_id(bool use_short_digest) +{ + static const std::string short_id = make_version_id(true); + static const std::string long_id = make_version_id(false); + return use_short_digest ? short_id : long_id; } /// Entries are grouped by the device they were compiled for. This keeps the directory @@ -99,7 +110,7 @@ static std::string device_dir(const context& ctx) /// from different toolchains would be indistinguishable. static fs::path entry_path(const fs::path& root, const context& ctx, const std::string& key) { - const auto& version = binary_cache::version_dir(); + const auto& version = binary_cache::version_id(true); if(version.empty()) return {}; return root / version / device_dir(ctx) / (md5(key) + ".mxr"); diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index 21e3b4464ac..e1a90d49b06 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -113,11 +113,12 @@ struct MIGRAPHX_GPU_EXPORT binary_cache const stats& get_stats() const { return counters; } - /// Names the directory holding entries that this build can use: the entry format, the - /// compiler, a digest of the embedded kernel headers, and the rocMLIR build. Empty when the - /// compiler cannot be identified, in which case nothing is read from or written to disk, - /// since entries from different compilers could not be told apart. - static const std::string& version_dir(); + /// Identifies what this build can use: the entry format, the compiler, a digest of the + /// embedded kernel headers, and the rocMLIR build. Digests are truncated when a short id is + /// requested; the short id names the cache directory. Empty when the compiler cannot be + /// identified, in which case nothing is read from or written to disk, since entries from + /// different compilers could not be told apart. + static const std::string& version_id(bool use_short_digest); private: std::unordered_map memo; diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 27cda498189..5a9332d33c2 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -334,11 +334,15 @@ TEST_CASE(key_covers_more_than_the_source) EXPECT(migraphx::gpu::hip_compile_key(ctx, changed_views) != base); } -TEST_CASE(version_dir_is_stable) +TEST_CASE(version_id_is_stable) { - EXPECT(migraphx::gpu::binary_cache::version_dir() == - migraphx::gpu::binary_cache::version_dir()); - EXPECT(not migraphx::gpu::binary_cache::version_dir().empty()); + EXPECT(migraphx::gpu::binary_cache::version_id(true) == + migraphx::gpu::binary_cache::version_id(true)); + EXPECT(migraphx::gpu::binary_cache::version_id(false) == + migraphx::gpu::binary_cache::version_id(false)); + EXPECT(not migraphx::gpu::binary_cache::version_id(true).empty()); + EXPECT(migraphx::gpu::binary_cache::version_id(true) != + migraphx::gpu::binary_cache::version_id(false)); } int main(int argc, const char* argv[]) { test::run(argc, argv); } From cdb51eaaf13c92e103b021e7ef6f423d4fd0127f Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Fri, 25 Sep 2026 15:14:01 +0200 Subject: [PATCH 33/49] Address review feedback on SQLite binary cache backend - sqlite_stmt: make bind/step/reset private; calling it binds its args and returns a range of rows as values; drop sqlite_stmt_reset - Reset the statement when its rows go away so reads don't keep a lock - Fall back to read-only use of a database that can't be written - Batch stores in one transaction (optional begin/end_batch on backends) - Open cache storage on first use instead of at construction - Restore directory-backend tests on non-Windows; add SQLite tests - Document .db/.sqlite selection and database maintenance Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- docs/dev/triage-migraphx.rst | 23 +++ src/include/migraphx/sqlite.hpp | 115 +++++++++++--- src/sqlite.cpp | 70 ++++++--- src/targets/gpu/binary_cache.cpp | 46 ++++-- src/targets/gpu/compile_ops.cpp | 26 ++-- .../gpu/include/migraphx/gpu/binary_cache.hpp | 23 +++ .../migraphx/gpu/binary_cache_backend.hpp | 86 ++++++++++- .../migraphx/gpu/sqlite_binary_cache.hpp | 30 +++- src/targets/gpu/sqlite_binary_cache.cpp | 122 +++++++++++---- test/gpu/binary_cache.cpp | 146 ++++++++++++++++-- test/sqlite.cpp | 92 ++++++----- tools/include/gpu/binary_cache_backend.hpp | 30 +++- 13 files changed, 650 insertions(+), 161 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5972dc67bc2..2da44a782eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Full documentation for MIGraphX is available at ### Added -* Added a binary cache for compiled GPU kernels: identical kernels within a model compile once, and setting the `MIGRAPHX_BINARY_CACHE` environment variable (or the `binary_cache` backend option) also persists them on disk so later compiles of the same kernels skip compilation entirely; a `binary_cache_verify` backend option recompiles reused kernels and fails if they differ. +* Added a binary cache for compiled GPU kernels: identical kernels within a model compile once, and setting the `MIGRAPHX_BINARY_CACHE` environment variable (or the `binary_cache` backend option) also persists them on disk so later compiles of the same kernels skip compilation entirely, in a directory or, for a path ending in `.db` or `.sqlite`, in a single SQLite database; a `binary_cache_verify` backend option recompiles reused kernels and fails if they differ. * Added a layered problem-cache priority list (searched in order, first hit wins), delivered to the GPU target through the `problem_cache_files` backend option. * Added `ArrayFeatureExtractor` ONNX operator support (#4742). * Added support for building against ROCm 7.13 and newer using TheRock (#4952) diff --git a/docs/dev/triage-migraphx.rst b/docs/dev/triage-migraphx.rst index d85d63d0dba..efdd3019081 100644 --- a/docs/dev/triage-migraphx.rst +++ b/docs/dev/triage-migraphx.rst @@ -175,6 +175,29 @@ directories for builds you no longer use: ls $HOME/.cache/migraphx # directories are named after the build that wrote them rm -r $HOME/.cache/migraphx/v1-hip22.0.* +A path ending in ``.db`` or ``.sqlite`` keeps the cache in a single SQLite database instead of a +directory, which is easier to copy between machines and can be shared by processes compiling at +the same time. Missing parent directories are created, as for a directory cache: + +.. code-block:: bash + + export MIGRAPHX_BINARY_CACHE=$HOME/.cache/migraphx/kernels.db + +Every row records the full version id of the build that wrote it in the ``version`` column, +alongside the operator name, problem and solution, so the database can be inspected and pruned +with the ``sqlite3`` shell. Deleting rows does not shrink the file until it is vacuumed: + +.. code-block:: bash + + sqlite3 $HOME/.cache/migraphx/kernels.db \ + "SELECT version, op_name, count(*) FROM cache_v1 GROUP BY version, op_name;" + sqlite3 $HOME/.cache/migraphx/kernels.db \ + "DELETE FROM cache_v1 WHERE version LIKE 'v1-hip22.0.%'; VACUUM;" + +A database that cannot be written to, such as a shared cache installed read-only, is still used +for lookups, and newly compiled kernels are kept in memory only. A database that cannot be opened +at all is skipped with a warning and the compile proceeds without a disk cache. + The same settings are available as backend options, which take precedence over the environment and are how tests configure the cache: diff --git a/src/include/migraphx/sqlite.hpp b/src/include/migraphx/sqlite.hpp index b11bb6e000b..f65217ae2e0 100644 --- a/src/include/migraphx/sqlite.hpp +++ b/src/include/migraphx/sqlite.hpp @@ -25,12 +25,18 @@ #define MIGRAPHX_GUARD_MIGRAPHX_SQLITE_HPP #include +#include #include +#include +#include #include +#include #include +#include #include #include #include +#include #include #include @@ -41,49 +47,108 @@ struct sqlite_impl; struct sqlite_stmt_impl; /// A prepared statement, holding a reference to the connection it was prepared on so it can -/// never outlive it. +/// never outlive it. Copies share the same statement. +/// +/// Calling it with arguments runs it: the arguments are bound to the parameters in order and the +/// result comes back as a range of rows. Preparing once and calling many times is the point. /// /// Not thread safe: one statement may be used by one thread at a time, even though the -/// connection itself is serialized. Reuse is the point of preparing -- prepare once, then -/// reset/bind/step per operation. +/// connection itself is serialized. struct MIGRAPHX_EXPORT sqlite_stmt { + struct rows; + sqlite_stmt() = default; - /// Bind a parameter. Indices are 1-based, matching sqlite's own convention. - sqlite_stmt& bind(int i, std::string_view s); - sqlite_stmt& bind(int i, std::int64_t x); - sqlite_stmt& bind(int i, const std::vector& blob); + /// Run the statement with xs bound to its parameters in order, and return its rows. + template + rows operator()(const Ts&... xs) const + { + if(not valid()) + MIGRAPHX_THROW("sqlite: calling a statement that was never prepared"); + // Anything left from the previous call, bindings or an unfinished result, goes first. + reset(); + sequence_c([&](auto... is) { swallow{(bind(int{is + 1}, xs), 0)...}; }); + return rows{*this}; + } + + bool valid() const { return impl != nullptr; } + + private: + // Parameter indices are 1-based, matching sqlite's own convention. + void bind(int i, std::string_view s) const; + void bind(int i, std::int64_t x) const; + void bind(int i, const std::vector& blob) const; /// Step once. True when a row is available, false when the statement is done. - bool step(); + bool step() const; /// Clear bindings and rewind, so the statement can be used again. Safe at any point, /// including after step() has thrown. - void reset() noexcept; + void reset() const noexcept; - /// Read a column of the current row. Indices here are 0-based, again matching sqlite. - std::string column_text(int i) const; - std::vector column_blob(int i) const; + /// The current row as an object keyed by column name. Blobs become value::binary and SQL + /// NULL becomes a null value. + value to_value() const; - bool valid() const { return impl != nullptr; } - - private: friend struct sqlite; std::shared_ptr impl; }; -/// Resets a statement on scope exit, so an early return or a thrown exception cannot leave -/// bindings or a half-consumed result set behind for whoever uses the statement next. -struct sqlite_stmt_reset +/// The rows produced by one call of a statement, as an input range of values. +/// +/// The first row is fetched when the call is made, so a statement that returns nothing, such as +/// an insert, has already run by the time the call returns, whether or not the range is +/// iterated. The statement is reset when the range is destroyed: an unfinished select holds a +/// read lock on the database until then, which would stall writers in other processes. +struct sqlite_stmt::rows { - explicit sqlite_stmt_reset(sqlite_stmt& s) : stmt(&s) {} - sqlite_stmt_reset(const sqlite_stmt_reset&) = delete; - sqlite_stmt_reset& operator=(const sqlite_stmt_reset&) = delete; - ~sqlite_stmt_reset() { stmt->reset(); } + explicit rows(sqlite_stmt s) : stmt(std::move(s)), first(stmt.step()) {} + // Only ever a prvalue returned from a call, so it never needs copying or moving, and a copy + // would reset the statement out from under the original. + rows(const rows&) = delete; + rows(rows&&) = delete; + rows& operator=(const rows&) = delete; + rows& operator=(rows&&) = delete; + ~rows() { stmt.reset(); } + + struct iterator : iterator_operators + { + using value_type = value; + using reference = value_type; + using difference_type = std::ptrdiff_t; + using iterator_category = std::input_iterator_tag; + using pointer = std::add_pointer_t>; + + iterator() = default; + + iterator(const rows* pparent, bool pavailable) : parent(pparent), available(pavailable) {} + + reference operator*() const { return parent->stmt.to_value(); } + + template + static void increment(U& x) + { + x.available = x.parent->stmt.step(); + } + + template + static auto equal(const U& x, const V& y) + { + return x.parent == y.parent and x.available == y.available; + } + + private: + const rows* parent = nullptr; + bool available = false; + }; + + iterator begin() const { return {this, first}; } + iterator end() const { return {this, false}; } private: - sqlite_stmt* stmt; + sqlite_stmt stmt; + bool first = false; }; struct MIGRAPHX_EXPORT sqlite @@ -103,6 +168,10 @@ struct MIGRAPHX_EXPORT sqlite /// How long to wait for a lock held by another connection before failing. void set_busy_timeout(int ms); + /// True when writes will be refused. Opening for writing still succeeds on a file the OS + /// has write-protected, in which case sqlite quietly opens it read-only; this is how to tell. + bool read_only() const; + bool valid() const { return impl != nullptr; } private: diff --git a/src/sqlite.cpp b/src/sqlite.cpp index 61bda7e5652..619089aced5 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -25,8 +25,10 @@ #include #include #include +#include #include #include +#include namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { @@ -157,7 +159,9 @@ sqlite_stmt sqlite::prepare(const std::string& sql) void sqlite::set_busy_timeout(int ms) { sqlite3_busy_timeout(impl->get(), ms); } -sqlite_stmt& sqlite_stmt::bind(int i, std::string_view s) +bool sqlite::read_only() const { return sqlite3_db_readonly(impl->get(), "main") == 1; } + +void sqlite_stmt::bind(int i, std::string_view s) const { // A default-constructed string_view has null data(), and a null pointer binds SQL NULL // rather than an empty string, so empty input substitutes a valid pointer. SQLITE_TRANSIENT @@ -167,18 +171,16 @@ sqlite_stmt& sqlite_stmt::bind(int i, std::string_view s) int rc = sqlite3_bind_text64(impl->get(), i, text, s.size(), SQLITE_TRANSIENT, SQLITE_UTF8); if(rc != SQLITE_OK) MIGRAPHX_THROW(impl->error_message()); - return *this; } -sqlite_stmt& sqlite_stmt::bind(int i, std::int64_t x) +void sqlite_stmt::bind(int i, std::int64_t x) const { int rc = sqlite3_bind_int64(impl->get(), i, x); if(rc != SQLITE_OK) MIGRAPHX_THROW(impl->error_message()); - return *this; } -sqlite_stmt& sqlite_stmt::bind(int i, const std::vector& blob) +void sqlite_stmt::bind(int i, const std::vector& blob) const { // As with text, an empty vector's data() may be null, which would bind SQL NULL; a // zero-length zeroblob is an empty BLOB instead. The 64-bit form is used because the @@ -189,10 +191,9 @@ sqlite_stmt& sqlite_stmt::bind(int i, const std::vector& blob) : sqlite3_bind_blob64(impl->get(), i, blob.data(), blob.size(), SQLITE_TRANSIENT); if(rc != SQLITE_OK) MIGRAPHX_THROW(impl->error_message()); - return *this; } -bool sqlite_stmt::step() +bool sqlite_stmt::step() const { int rc = sqlite3_step(impl->get()); if(rc == SQLITE_ROW) @@ -202,7 +203,7 @@ bool sqlite_stmt::step() MIGRAPHX_THROW(impl->error_message()); } -void sqlite_stmt::reset() noexcept +void sqlite_stmt::reset() const noexcept { if(impl == nullptr) return; @@ -212,25 +213,48 @@ void sqlite_stmt::reset() noexcept (void)sqlite3_clear_bindings(impl->get()); } -std::string sqlite_stmt::column_text(int i) const +static value column_value(sqlite3_stmt* stmt, int i) { - const auto* text = sqlite3_column_text(impl->get(), i); - int n = sqlite3_column_bytes(impl->get(), i); - if(text == nullptr or n <= 0) - return {}; - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - return {reinterpret_cast(text), static_cast(n)}; + // For text and blobs the data must be fetched before sqlite3_column_bytes: the other order + // can force a type conversion that invalidates the pointer. A zero-length value comes back + // as a null pointer, which still means empty rather than NULL. + switch(sqlite3_column_type(stmt, i)) + { + case SQLITE_INTEGER: return std::int64_t{sqlite3_column_int64(stmt, i)}; + case SQLITE_FLOAT: return sqlite3_column_double(stmt, i); + case SQLITE_TEXT: { + const auto* text = sqlite3_column_text(stmt, i); + auto n = sqlite3_column_bytes(stmt, i); + if(text == nullptr) + return std::string{}; + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return std::string(reinterpret_cast(text), n); + } + case SQLITE_BLOB: { + const auto* data = static_cast(sqlite3_column_blob(stmt, i)); + auto n = sqlite3_column_bytes(stmt, i); + if(data == nullptr) + return value::binary{}; + return value::binary{data, static_cast(n)}; + } + default: return {}; + } } -std::vector sqlite_stmt::column_blob(int i) const +value sqlite_stmt::to_value() const { - // sqlite3_column_blob must be called before sqlite3_column_bytes: the other order can - // force a type conversion that invalidates the pointer. - const auto* data = static_cast(sqlite3_column_blob(impl->get(), i)); - int n = sqlite3_column_bytes(impl->get(), i); - if(data == nullptr or n <= 0) - return {}; - return {data, data + n}; + auto* stmt = impl->get(); + value::object row; + auto columns = range(sqlite3_column_count(stmt)); + std::transform(columns.begin(), + columns.end(), + std::inserter(row, row.end()), + [&](std::ptrdiff_t i) { + auto col = static_cast(i); + return std::make_pair(std::string{sqlite3_column_name(stmt, col)}, + column_value(stmt, col)); + }); + return row; } } // namespace MIGRAPHX_INLINE_NS diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index f56c0a86304..55fa75fb83d 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -146,19 +146,39 @@ static optional make_binary_cache_backend(const std::strin return binary_cache_backend{file_binary_cache{path}}; } +binary_cache::binary_cache(binary_cache_settings s) : settings(std::move(s)) {} + // Nothing can be persisted safely when the compiler cannot be identified, since entries from // different toolchains would be indistinguishable. That is a property of the cache rather than // of the storage medium, so it is checked here instead of in each backend. -binary_cache::binary_cache(binary_cache_settings s) : settings(std::move(s)) +binary_cache_backend* binary_cache::get_backend() { - // Checked first so that a memory-only cache never compiles the version probe. - if(settings.path.empty()) - return; - // The version names a directory for the file backend, so it is kept short there. A database - // has no such limit and records the full id, which is self-describing. - version = version_id(not is_database_path(settings.path)); - if(not version.empty()) - backend = make_binary_cache_backend(settings.path); + if(not backend_opened) + { + backend_opened = true; + // Checked first so that a memory-only cache never compiles the version probe. + if(not settings.path.empty()) + { + // The version names a directory for the file backend, so it is kept short there. A + // database has no such limit and records the full id, which is self-describing. + version = version_id(not is_database_path(settings.path)); + if(not version.empty()) + backend = make_binary_cache_backend(settings.path); + } + } + return backend.has_value() ? &*backend : nullptr; +} + +binary_cache::store_batch::store_batch(binary_cache& c) : cache(&c) +{ + if(auto* b = cache->get_backend()) + b->begin_batch(); +} + +binary_cache::store_batch::~store_batch() +{ + if(auto* b = cache->get_backend()) + b->end_batch(); } optional binary_cache::get(const context& ctx, const std::string& key) @@ -171,12 +191,12 @@ optional binary_cache::get(const context& ctx, const std::string& counters.reused++; return it->second; } - if(backend.has_value()) + if(auto* b = get_backend()) { // Hashing the key is not free -- it is the whole compile source, which runs to // kilobytes -- so it is done once and reused for the lookup and any diagnostics. auto key_hash = md5(key); - auto blob = backend->load(version, device_dir(ctx), key_hash); + auto blob = b->load(version, device_dir(ctx), key_hash); if(blob.has_value()) { auto e = decode_entry(*blob, key, key_hash); @@ -196,7 +216,7 @@ void binary_cache::insert(const context& ctx, entry e) if(e.key.empty()) return; counters.compiled++; - if(backend.has_value()) + if(auto* b = get_backend()) { auto key_hash = md5(e.key); try @@ -205,7 +225,7 @@ void binary_cache::insert(const context& ctx, entry e) // failure here a warning like any other storage failure instead of escaping // insert() and failing the compile. auto blob = to_msgpack(migraphx::to_value(e)); - backend->store(version, device_dir(ctx), key_hash, e, blob); + b->store(version, device_dir(ctx), key_hash, e, blob); } catch(const std::exception& ex) { diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index ad692b3a3c3..8ea8289523e 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -804,17 +804,23 @@ struct compile_manager cell->result = cp->run_compile(cell->solution); }); - for(const auto& [cp, cell] : tasks) + if(not tasks.empty()) { - if(not cell->result.has_value()) - continue; - // When verifying, reused results are stored again, rewriting the same bytes - // harmlessly. - cp->store(cell->solution, cell->key, cell->result->code); - assert(not cell->result->code.empty()); - // Only the serializable code is used from here on; dropping the replace function - // releases what its closure holds and keeps it off other instructions. - cell->result->replace_fn = nullptr; + // Every plan compiles with the same context, so the stores all go to one cache, and + // batching them lets its storage commit them together rather than one at a time. + binary_cache::store_batch batch{tasks.front().first->ctx->get_binary_cache()}; + for(const auto& [cp, cell] : tasks) + { + if(not cell->result.has_value()) + continue; + // When verifying, reused results are stored again, rewriting the same bytes + // harmlessly. + cp->store(cell->solution, cell->key, cell->result->code); + assert(not cell->result->code.empty()); + // Only the serializable code is used from here on; dropping the replace function + // releases what its closure holds and keeps it off other instructions. + cell->result->replace_fn = nullptr; + } } static const auto mxr_path = string_value_of(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR{}); diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index 770f7efee0a..887915135e2 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -85,6 +85,24 @@ struct MIGRAPHX_GPU_EXPORT binary_cache std::size_t compiled = 0; }; + /// Groups the inserts made while it lives, so the storage backend can commit them together + /// rather than one at a time. Scope it tightly around a run of inserts: a database holds a + /// write lock against other processes until it ends. + struct MIGRAPHX_GPU_EXPORT store_batch + { + explicit store_batch(binary_cache& c); + store_batch(const store_batch&) = delete; + store_batch(store_batch&&) = delete; + store_batch& operator=(const store_batch&) = delete; + store_batch& operator=(store_batch&&) = delete; + ~store_batch(); + + private: + binary_cache* cache; + }; + + /// Nothing is opened here; storage is set up by the first lookup or insert, so a context + /// that never compiles never touches the disk or probes the compiler. explicit binary_cache(binary_cache_settings s = {}); /// Look up a key, consulting memory first and then the cache directory. @@ -106,12 +124,17 @@ struct MIGRAPHX_GPU_EXPORT binary_cache static const std::string& version_id(bool use_short_digest); private: + /// The storage backend, opened on first use, or null for a memory-only cache. + binary_cache_backend* get_backend(); + std::unordered_map memo; binary_cache_settings settings; /// The version_id entries are stored under, in the form the backend uses. std::string version; /// Where entries are persisted, or empty for a memory-only cache. optional backend; + /// Whether get_backend has already tried to open the backend, successfully or not. + bool backend_opened = false; stats counters; }; diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp index e57c5021c5c..106cf627d91 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp @@ -30,7 +30,8 @@ // into the gpu target tree). Do not edit the generated header by hand. // // Any type T satisfies the binary_cache_backend concept if it provides the -// member functions listed below. The wrapper holds T by shared_ptr and forwards +// member functions listed below; begin_batch and end_batch are optional and +// default to doing nothing. The wrapper holds T by shared_ptr and forwards // each call through a virtual dispatch, matching problem_cache_backend. // // Notes: @@ -38,9 +39,10 @@ // the include below pulls in its full definition. // * Backends typically own non-trivial resources (a cache directory, a SQLite // connection) and are not meaningfully copyable beyond shared ownership. -// * Both members are non-const: binary_cache::get and insert are themselves -// non-const, so nothing forces a const qualifier here, and a backend holding -// prepared statements needs the mutability. +// * The members are non-const: binary_cache::get and insert are themselves +// non-const, so nothing forces a const qualifier here, and a backend that +// tracks an open batch needs the mutability. A backend may still declare +// them const. // #ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP #define MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP @@ -54,6 +56,7 @@ #include #include +#include #include #include #include @@ -112,6 +115,21 @@ struct binary_cache_backend const std::string& key_hash, const binary_cache_entry& e, const std::vector& blob); + + /// Mark the start of a run of stores that may be committed together, such as + /// a database transaction, rather than one at a time. Every begin_batch is + /// followed by an end_batch, and the two are never nested. Optional: a + /// backend without them stores each entry as it comes. + /// + /// Must not throw. A backend that cannot start a batch stores entries one + /// at a time instead. + void begin_batch(); + + /// Commit the stores made since begin_batch. + /// + /// Must not throw. A failed commit costs those entries a recompile next + /// run, nothing more, and must not leave anything locked. + void end_batch(); }; #else @@ -130,6 +148,10 @@ struct MIGRAPHX_EXPORT binary_cache_backend const std::string& key_hash, const binary_cache_entry& e, const std::vector& blob); + // (optional) + void begin_batch(); + // (optional) + void end_batch(); }; #else @@ -137,6 +159,32 @@ struct MIGRAPHX_EXPORT binary_cache_backend struct binary_cache_backend { private: + template + static auto private_detail_te_default_begin_batch(char, T&& private_detail_te_self) + -> decltype(private_detail_te_self.begin_batch()) + { + private_detail_te_self.begin_batch(); + } + + template + static void private_detail_te_default_begin_batch(float, T&& private_detail_te_self) + { + migraphx::nop(private_detail_te_self); + } + + template + static auto private_detail_te_default_end_batch(char, T&& private_detail_te_self) + -> decltype(private_detail_te_self.end_batch()) + { + private_detail_te_self.end_batch(); + } + + template + static void private_detail_te_default_end_batch(float, T&& private_detail_te_self) + { + migraphx::nop(private_detail_te_self); + } + template struct private_te_unwrap_reference { @@ -162,6 +210,10 @@ struct binary_cache_backend std::declval(), std::declval(), std::declval&>()), + private_detail_te_default_begin_batch(char(0), + std::declval()), + private_detail_te_default_end_batch(char(0), + std::declval()), void()); template @@ -255,6 +307,18 @@ struct binary_cache_backend (*this).private_detail_te_get_handle().store(version, device, key_hash, e, blob); } + void begin_batch() + { + assert((*this).private_detail_te_handle_mem_var); + (*this).private_detail_te_get_handle().begin_batch(); + } + + void end_batch() + { + assert((*this).private_detail_te_handle_mem_var); + (*this).private_detail_te_get_handle().end_batch(); + } + friend bool is_shared(const binary_cache_backend& private_detail_x, const binary_cache_backend& private_detail_y) { @@ -277,6 +341,8 @@ struct binary_cache_backend const std::string& key_hash, const binary_cache_entry& e, const std::vector& blob) = 0; + virtual void begin_batch() = 0; + virtual void end_batch() = 0; }; template @@ -324,6 +390,18 @@ struct binary_cache_backend private_detail_te_value.store(version, device, key_hash, e, blob); } + void begin_batch() override + { + + private_detail_te_default_begin_batch(char(0), private_detail_te_value); + } + + void end_batch() override + { + + private_detail_te_default_end_batch(char(0), private_detail_te_value); + } + PrivateDetailTypeErasedT private_detail_te_value; }; diff --git a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp index 1ab8411edd3..0efd4e289e5 100644 --- a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp @@ -46,23 +46,37 @@ namespace gpu { // target: migraphx/sqlite.hpp forward-declares both impl types and never includes sqlite3.h. struct MIGRAPHX_GPU_EXPORT sqlite_binary_cache { - /// Open the database, create the schema and prepare the statements. Returns nullopt when any - /// of that fails, so an unusable database leaves the cache memory-only rather than raising an - /// error. Returns the wrapper so the caller can hand the result straight back. + /// Open the database, create the schema and prepare the statements, and return the wrapper + /// so the caller can hand the result straight back. + /// + /// A database that can only be read gives a backend that serves lookups and ignores stores; + /// it is used as it stands, without creating the schema. Returns nullopt when the database + /// cannot be opened at all or entries cannot be looked up in it, so an unusable database + /// leaves the cache memory-only rather than raising an error. static optional open(const std::string& path); optional> - load(const std::string& version, const std::string& device, const std::string& key_hash); + load(const std::string& version, const std::string& device, const std::string& key_hash) const; void store(const std::string& version, const std::string& device, const std::string& key_hash, const binary_cache_entry& e, - const std::vector& blob); + const std::vector& blob) const; + + /// Open a transaction, so the stores that follow cost one commit rather than one each. + void begin_batch(); + /// Commit the transaction begin_batch opened, or roll it back if the commit fails. + void end_batch(); private: - sqlite db = {}; - sqlite_stmt get_stmt = {}; - sqlite_stmt store_stmt = {}; + sqlite db = {}; + sqlite_stmt get_stmt = {}; + sqlite_stmt store_stmt = {}; + sqlite_stmt begin_stmt = {}; + sqlite_stmt commit_stmt = {}; + sqlite_stmt rollback_stmt = {}; + /// Whether begin_batch opened a transaction that end_batch still has to close. + bool in_batch = false; }; } // namespace gpu diff --git a/src/targets/gpu/sqlite_binary_cache.cpp b/src/targets/gpu/sqlite_binary_cache.cpp index cd7504048f1..de0f1ae229b 100644 --- a/src/targets/gpu/sqlite_binary_cache.cpp +++ b/src/targets/gpu/sqlite_binary_cache.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -83,6 +84,14 @@ constexpr const char* store_sql = " (version, device, key_hash, op_name, problem, solution, entry, timestamp)" " VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, CAST(STRFTIME('%s','now') AS INTEGER));"; +// Stores come in a burst after each round of compiles, and outside a transaction every one of +// them is its own commit, each waiting for the disk. IMMEDIATE takes the write lock up front, so +// a busy database is found out here, once, rather than partway through the stores; readers are +// not blocked until the commit itself. +constexpr const char* begin_sql = "BEGIN IMMEDIATE;"; +constexpr const char* commit_sql = "COMMIT;"; +constexpr const char* rollback_sql = "ROLLBACK;"; + } // namespace optional sqlite_binary_cache::open(const std::string& path) @@ -91,11 +100,20 @@ optional sqlite_binary_cache::open(const std::string& path try { // sqlite will not create a missing parent directory, but the file backend does, so - // this keeps the two backends behaving the same on a fresh machine. + // this keeps the two backends behaving the same on a fresh machine. A failure here is + // left to the open below, since an existing database may still be readable. auto parent = fs::path{path}.parent_path(); + std::error_code ec; if(not parent.empty()) - fs::create_directories(parent); + fs::create_directories(parent, ec); + + // A database that can be read but not written to is still worth having: reads serve + // hits and nothing is stored. Opening for writing already falls back to reading when the + // file itself is write-protected; reading is tried here for anything else that refuses + // a writer, such as a read-only mount, as long as there is a database to read. auto db = sqlite::try_write(path); + if(not db.has_value() and fs::exists(path)) + db = sqlite::read(path); if(not db.has_value()) { log::warn() << "Disabling the binary cache: cannot open " << path; @@ -103,8 +121,20 @@ optional sqlite_binary_cache::open(const std::string& path } r.db = std::move(*db); r.db.set_busy_timeout(busy_timeout_ms); - (void)r.db.execute(schema_sql); - // Without a working lookup there is no cache, so this failure disables the backend. + if(r.db.read_only()) + { + log::warn() << "Binary cache at " << path << " is read-only"; + } + else + { + (void)r.db.execute(schema_sql); + r.store_stmt = r.db.prepare(store_sql); + r.begin_stmt = r.db.prepare(begin_sql); + r.commit_stmt = r.db.prepare(commit_sql); + r.rollback_stmt = r.db.prepare(rollback_sql); + } + // Without a working lookup there is no cache, so this failure disables the backend. That + // includes a read-only database that was never given the schema. r.get_stmt = r.db.prepare(get_sql); } catch(const std::exception& ex) @@ -112,32 +142,25 @@ optional sqlite_binary_cache::open(const std::string& path log::warn() << "Disabling the binary cache at " << path << ": " << ex.what(); return nullopt; } - try - { - r.store_stmt = r.db.prepare(store_sql); - } - catch(const std::exception& ex) - { - // A database that can be read but not written to is still worth having: reads serve - // hits and stores quietly do nothing. - log::warn() << "Binary cache at " << path << " is read-only: " << ex.what(); - } return binary_cache_backend{std::move(r)}; } optional> sqlite_binary_cache::load(const std::string& version, const std::string& device, - const std::string& key_hash) + const std::string& key_hash) const { if(not get_stmt.valid()) return nullopt; try { - sqlite_stmt_reset guard{get_stmt}; - get_stmt.bind(1, version).bind(2, device).bind(3, key_hash); - if(not get_stmt.step()) + // The primary key makes this at most one row. + auto rows = get_stmt(version, device, key_hash); + auto it = rows.begin(); + if(it == rows.end()) return nullopt; - return get_stmt.column_blob(0); + auto row = *it; + const auto& entry = row.at("entry").get_binary(); + return std::vector(entry.begin(), entry.end()); } catch(const std::exception& ex) { @@ -151,22 +174,21 @@ void sqlite_binary_cache::store(const std::string& version, const std::string& device, const std::string& key_hash, const binary_cache_entry& e, - const std::vector& blob) + const std::vector& blob) const { if(not store_stmt.valid()) return; try { // The json strings are temporaries, which is safe because binding copies immediately. - sqlite_stmt_reset guard{store_stmt}; - store_stmt.bind(1, version) - .bind(2, device) - .bind(3, key_hash) - .bind(4, e.op_name) - .bind(5, to_json_string(e.problem)) - .bind(6, to_json_string(e.solution)) - .bind(7, blob); - store_stmt.step(); + // An insert returns no rows, and it has run by the time the call returns. + store_stmt(version, + device, + key_hash, + e.op_name, + to_json_string(e.problem), + to_json_string(e.solution), + blob); } catch(const std::exception& ex) { @@ -174,6 +196,48 @@ void sqlite_binary_cache::store(const std::string& version, } } +void sqlite_binary_cache::begin_batch() +{ + assert(not in_batch); + if(not begin_stmt.valid()) + return; + try + { + begin_stmt(); + in_batch = true; + } + catch(const std::exception& ex) + { + // Without a transaction each store commits on its own, which is slower but still works. + log::warn() << "Binary cache stores will be committed one at a time: " << ex.what(); + } +} + +void sqlite_binary_cache::end_batch() +{ + if(not in_batch) + return; + in_batch = false; + try + { + commit_stmt(); + } + catch(const std::exception& ex) + { + // A commit that fails leaves the transaction open, holding the write lock against every + // other process, so it is rolled back and the batch's entries are lost instead. + log::warn() << "Failed to commit binary cache entries: " << ex.what(); + try + { + rollback_stmt(); + } + catch(const std::exception& rex) + { + log::warn() << "Failed to roll back binary cache entries: " << rex.what(); + } + } +} + } // namespace gpu } // namespace MIGRAPHX_INLINE_NS } // namespace migraphx diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 16a93c08a51..4dea3f94034 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include #include @@ -161,15 +163,13 @@ TEST_CASE(memory_lookup_records_reuse) EXPECT(cache.get_stats().misses == 0); } -// The cases below are registered only against a database path, not a directory. Driving the file -// backend through binary_cache puts entries under version_id()/device_dir(), and write_atomically -// then creates a temp directory inside that, which pushes the file past Windows' MAX_PATH: -// fs::create_directories succeeds because std::filesystem uses the \\?\ prefix, but the -// std::ofstream in write_buffer does not, so every store fails and nothing is persisted. The -// bodies are still parameterized by path, so a directory case is one TEST_CASE to restore once -// write_atomically writes its temporary as a sibling instead of nesting a directory. -// backends_round_trip_through_the_wrapper still covers the file backend, where it is driven -// directly and the version and device strings are short. +// The cases below are written once against a path and registered for both backends. The +// directory registrations are skipped on Windows for now: driving the file backend through +// binary_cache puts entries under version_id()/device_dir(), and write_atomically then creates a +// temp directory inside that, which pushes the file past Windows' MAX_PATH. fs::create_directories +// succeeds because std::filesystem uses the \\?\ prefix, but the std::ofstream in write_buffer +// does not, so every store fails and nothing is persisted. backends_round_trip_through_the_wrapper +// still covers the file backend there, where it is driven directly and the paths are short. // A second cache shares nothing in memory, so anything it finds came out of storage. static void disk_lookup_body(const std::string& path) @@ -189,6 +189,14 @@ static void disk_lookup_body(const std::string& path) EXPECT(*found->fragment.get_main_module() == *make_code().fragment.get_main_module()); } +#ifndef _WIN32 +TEST_CASE(disk_lookup_records_a_hit) +{ + migraphx::tmp_dir td{"binary-cache"}; + disk_lookup_body(dir_path(td)); +} +#endif + TEST_CASE(sqlite_lookup_records_a_hit) { migraphx::tmp_dir td{"binary-cache"}; @@ -214,6 +222,19 @@ static void corrupt_entry_body(const std::string& path, EXPECT(reader.get_stats().misses == 1); } +#ifndef _WIN32 +TEST_CASE(corrupt_entry_is_ignored) +{ + migraphx::tmp_dir td{"binary-cache"}; + corrupt_entry_body(dir_path(td), [](const std::string& dir) { + auto files = entry_files(dir); + EXPECT(files.size() == 1); + for(const auto& file : files) + migraphx::write_buffer(file, std::vector(8, 0)); + }); +} +#endif + TEST_CASE(sqlite_corrupt_entry_is_ignored) { migraphx::tmp_dir td{"binary-cache"}; @@ -318,6 +339,14 @@ static void compiling_twice_body(const std::string& path) gpu_result.to_vector())); } +#ifndef _WIN32 +TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) +{ + migraphx::tmp_dir td{"binary-cache"}; + compiling_twice_body(dir_path(td)); +} +#endif + TEST_CASE(sqlite_compiling_twice_populates_the_cache_and_matches_reference) { migraphx::tmp_dir td{"binary-cache"}; @@ -337,6 +366,14 @@ static void verified_reuse_body(const std::string& path) p.compile(migraphx::make_target("gpu"), options); } +#ifndef _WIN32 +TEST_CASE(verified_reuse_matches_fresh_compiles) +{ + migraphx::tmp_dir td{"binary-cache"}; + verified_reuse_body(dir_path(td)); +} +#endif + TEST_CASE(sqlite_verified_reuse_matches_fresh_compiles) { migraphx::tmp_dir td{"binary-cache"}; @@ -445,6 +482,63 @@ TEST_CASE(sqlite_records_the_full_version_id) EXPECT(rows.front().at("version") == migraphx::gpu::binary_cache::version_id(false)); } +// The op name, problem and solution are copied into columns only so a cache can be inspected +// with SQL; loads never read them, so nothing else would notice them being wrong. +TEST_CASE(sqlite_records_what_each_entry_was_compiled_for) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + auto path = db_path(td); + auto e = make_entry("described"); + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; + cache.insert(ctx, e); + + auto rows = migraphx::sqlite::read(path).execute( + "SELECT key_hash, op_name, problem, solution FROM cache_v1;"); + EXPECT(rows.size() == 1); + const auto& row = rows.front(); + EXPECT(row.at("key_hash") == migraphx::md5(e.key)); + EXPECT(row.at("op_name") == e.op_name); + EXPECT(migraphx::from_json_string(row.at("problem")) == e.problem); + EXPECT(migraphx::from_json_string(row.at("solution")) == e.solution); +} + +// A database that cannot be written to, such as a shared cache installed read-only, still +// serves the entries already in it, and storing into it is quietly skipped. +TEST_CASE(sqlite_read_only_database_still_serves_hits) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + auto path = db_path(td); + migraphx::gpu::binary_cache_settings settings{path, false}; + { + migraphx::gpu::binary_cache writer{settings}; + writer.insert(ctx, make_entry("existing")); + } + + const auto writable = migraphx::fs::perms::owner_write | migraphx::fs::perms::group_write | + migraphx::fs::perms::others_write; + migraphx::fs::permissions(path, writable, migraphx::fs::perm_options::remove); + // Permissions do not stop root, so what can be checked about stores depends on whether the + // write protection actually took. + const bool protected_file = migraphx::sqlite::write(path).read_only(); + + migraphx::gpu::binary_cache reader{settings}; + EXPECT(reader.get(ctx, "existing").has_value()); + EXPECT(reader.get_stats().hits == 1); + + reader.insert(ctx, make_entry("new")); + EXPECT(reader.get(ctx, "new").has_value()); + if(protected_file) + { + EXPECT(row_count(path, "cache_v1") == 1); + } + + // Restored so the temporary directory can be removed, which Windows refuses otherwise. + migraphx::fs::permissions( + path, migraphx::fs::perms::owner_write, migraphx::fs::perm_options::add); +} + // version and device lead the primary key because they are what separates entries this build // may use from entries it may not, so a row stored under one must not be served under another. TEST_CASE(sqlite_scopes_entries_by_version_and_device) @@ -464,6 +558,40 @@ TEST_CASE(sqlite_scopes_entries_by_version_and_device) // Storing a key twice replaces the row rather than accumulating or failing, the way the file // backend's publish-by-rename overwrites in place. Two processes compiling the same kernel is // benign for exactly this reason. +// Storage is opened by the first lookup or insert, not by constructing the cache, since every +// context makes one whether or not it ever compiles anything. +TEST_CASE(storage_is_opened_on_first_use) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + auto path = db_path(td); + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; + EXPECT(not migraphx::fs::exists(path)); + + EXPECT(not cache.get(ctx, "absent").has_value()); + EXPECT(migraphx::fs::exists(path)); +} + +// Inserts made inside a batch are committed together when it ends, and are all there after. +TEST_CASE(sqlite_batched_inserts_are_all_committed) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + auto path = db_path(td); + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; + { + migraphx::gpu::binary_cache::store_batch batch{cache}; + cache.insert(ctx, make_entry("first")); + cache.insert(ctx, make_entry("second")); + } + EXPECT(row_count(path, "cache_v1") == 2); + + migraphx::gpu::binary_cache reader{migraphx::gpu::binary_cache_settings{path, false}}; + EXPECT(reader.get(ctx, "first").has_value()); + EXPECT(reader.get(ctx, "second").has_value()); + EXPECT(reader.get_stats().hits == 2); +} + TEST_CASE(sqlite_store_overwrites_in_place) { migraphx::tmp_dir td{"binary-cache"}; diff --git a/test/sqlite.cpp b/test/sqlite.cpp index b97b05862ab..71e5bdba20f 100644 --- a/test/sqlite.cpp +++ b/test/sqlite.cpp @@ -25,9 +25,14 @@ #include #include #include -#include #include +/// Every row a call produced, so a test can count and inspect them. +static std::vector collect(const migraphx::sqlite_stmt::rows& r) +{ + return std::vector(r.begin(), r.end()); +} + TEST_CASE(read_write) { const std::string create_table = R"__migraphx__( @@ -76,47 +81,62 @@ TEST_CASE(prepared_blob_round_trip) ); )__migraphx__"); - // One statement, two inserts: the reset/rebind path backends rely on. + // One statement, two inserts: calling it again rebinds, which is what backends rely on. + // An insert produces no rows, and runs whether or not they are iterated. auto insert = db.prepare("INSERT INTO blob_db (name, size, data) VALUES (?, ?, ?);"); EXPECT(insert.valid()); - - insert.bind(1, std::string_view{"k1"}) - .bind(2, static_cast(blob.size())) - .bind(3, blob); - EXPECT(not insert.step()); - insert.reset(); - - insert.bind(1, std::string_view{"empty"}) - .bind(2, static_cast(0)) - .bind(3, std::vector{}); - EXPECT(not insert.step()); - insert.reset(); + EXPECT(collect(insert("k1", static_cast(blob.size()), blob)).empty()); + insert("empty", std::int64_t{0}, std::vector{}); } { auto db = migraphx::sqlite::read(db_path); - auto select = db.prepare("SELECT name, data FROM blob_db WHERE name = ?;"); - - { - migraphx::sqlite_stmt_reset guard{select}; - select.bind(1, std::string_view{"k1"}); - EXPECT(select.step()); - EXPECT(select.column_text(0) == "k1"); - EXPECT((select.column_blob(1) == blob)); - EXPECT(not select.step()); - } - { - // An empty blob must come back as an empty blob, not as NULL. - migraphx::sqlite_stmt_reset guard{select}; - select.bind(1, std::string_view{"empty"}); - EXPECT(select.step()); - EXPECT(select.column_blob(1).empty()); - } - { - migraphx::sqlite_stmt_reset guard{select}; - select.bind(1, std::string_view{"missing"}); - EXPECT(not select.step()); - } + auto select = db.prepare("SELECT name, size, data FROM blob_db WHERE name = ?;"); + + auto found = collect(select("k1")); + EXPECT(found.size() == 1); + EXPECT(found.front().at("name").get_string() == "k1"); + EXPECT(found.front().at("size").to() == blob.size()); + EXPECT(found.front().at("data").get_binary() == migraphx::value::binary{blob}); + + // An empty blob must come back as an empty blob, not as NULL. + auto empty = collect(select("empty")); + EXPECT(empty.size() == 1); + EXPECT(empty.front().at("data").is_binary()); + EXPECT(empty.front().at("data").get_binary().empty()); + + EXPECT(collect(select("missing")).empty()); + } +} + +// A select abandoned after its first row must not keep holding the database. Until the +// statement is reset it holds a read lock, and a writer on another connection would wait out +// its busy timeout and then fail. +TEST_CASE(abandoned_rows_release_the_database) +{ + migraphx::tmp_dir td{}; + auto db_path = td.path / "lock.db"; + auto writer = migraphx::sqlite::write(db_path); + writer.execute("CREATE TABLE t (id INTEGER PRIMARY KEY);" + "INSERT INTO t (id) VALUES (1), (2);"); + + auto reader = migraphx::sqlite::read(db_path); + auto select = reader.prepare("SELECT id FROM t;"); + { + // Read one of the two rows and stop. + auto rows = select(); + EXPECT(rows.begin() != rows.end()); } + + auto insert = writer.prepare("INSERT INTO t (id) VALUES (?);"); + insert(std::int64_t{3}); + EXPECT(writer.execute("SELECT id FROM t;").size() == 3); +} + +TEST_CASE(unprepared_statement_throws) +{ + migraphx::sqlite_stmt stmt; + EXPECT(not stmt.valid()); + EXPECT(test::throws([&] { stmt(); })); } TEST_CASE(try_write_unusable_path) diff --git a/tools/include/gpu/binary_cache_backend.hpp b/tools/include/gpu/binary_cache_backend.hpp index cc6ddb49a63..cb31fb5d5c4 100644 --- a/tools/include/gpu/binary_cache_backend.hpp +++ b/tools/include/gpu/binary_cache_backend.hpp @@ -30,7 +30,8 @@ // into the gpu target tree). Do not edit the generated header by hand. // // Any type T satisfies the binary_cache_backend concept if it provides the -// member functions listed below. The wrapper holds T by shared_ptr and forwards +// member functions listed below; begin_batch and end_batch are optional and +// default to doing nothing. The wrapper holds T by shared_ptr and forwards // each call through a virtual dispatch, matching problem_cache_backend. // // Notes: @@ -38,9 +39,10 @@ // the include below pulls in its full definition. // * Backends typically own non-trivial resources (a cache directory, a SQLite // connection) and are not meaningfully copyable beyond shared ownership. -// * Both members are non-const: binary_cache::get and insert are themselves -// non-const, so nothing forces a const qualifier here, and a backend holding -// prepared statements needs the mutability. +// * The members are non-const: binary_cache::get and insert are themselves +// non-const, so nothing forces a const qualifier here, and a backend that +// tracks an open batch needs the mutability. A backend may still declare +// them const. // #ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP #define MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP @@ -54,6 +56,7 @@ #include #include +#include #include #include #include @@ -113,6 +116,21 @@ struct binary_cache_backend const std::string& key_hash, const binary_cache_entry& e, const std::vector& blob); + + /// Mark the start of a run of stores that may be committed together, such as + /// a database transaction, rather than one at a time. Every begin_batch is + /// followed by an end_batch, and the two are never nested. Optional: a + /// backend without them stores each entry as it comes. + /// + /// Must not throw. A backend that cannot start a batch stores entries one + /// at a time instead. + void begin_batch(); + + /// Commit the stores made since begin_batch. + /// + /// Must not throw. A failed commit costs those entries a recompile next + /// run, nothing more, and must not leave anything locked. + void end_batch(); }; #else @@ -131,7 +149,9 @@ struct binary_cache_backend device = 'const std::string&', key_hash = 'const std::string&', e = 'const binary_cache_entry&', - blob = 'const std::vector&')) + blob = 'const std::vector&'), + virtual('begin_batch', returns = 'void', default = 'migraphx::nop'), + virtual('end_batch', returns = 'void', default = 'migraphx::nop')) %> #endif From 58ab7b1d7520d7a8453fc8335c18e37ff8a88dca Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Fri, 25 Sep 2026 15:42:16 +0200 Subject: [PATCH 34/49] Add tests --- src/targets/gpu/file_binary_cache.cpp | 24 +- test/gpu/binary_cache.cpp | 351 ++++++++++++++++++++++---- test/sqlite.cpp | 56 ++++ 3 files changed, 371 insertions(+), 60 deletions(-) diff --git a/src/targets/gpu/file_binary_cache.cpp b/src/targets/gpu/file_binary_cache.cpp index c5de351836e..9e121dc242c 100644 --- a/src/targets/gpu/file_binary_cache.cpp +++ b/src/targets/gpu/file_binary_cache.cpp @@ -26,7 +26,9 @@ #include #include #include +#include #include +#include #include namespace migraphx { @@ -48,12 +50,26 @@ static fs::path entry_path(const fs::path& root, /// Publish by rename so a reader never sees a half-written file. The temporary stays beside /// the destination since the rename is only atomic within one filesystem. +/// +/// It is a sibling file with a short unique suffix rather than a file inside a temporary +/// directory: entries already sit several directories deep, and a nested directory with a +/// fully unique name pushed the path past Windows' MAX_PATH, which std::ofstream cannot open. +/// The suffix only has to keep concurrent writers of the same entry apart. static void write_atomically(const fs::path& dest, const std::vector& content) { - tmp_dir td{"cache", dest.parent_path()}; - auto tmp = td.path / dest.filename(); - write_buffer(tmp, content); - fs::rename(tmp, dest); + auto suffix = md5(unique_string("cache")).substr(0, 16); + auto tmp = dest.parent_path() / (dest.stem().string() + "." + suffix + ".tmp"); + try + { + write_buffer(tmp, content); + fs::rename(tmp, dest); + } + catch(...) + { + std::error_code ec; + fs::remove(tmp, ec); + throw; + } } optional> file_binary_cache::load(const std::string& version, diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 4dea3f94034..1967581d4d6 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -53,6 +53,8 @@ #include #include #include +#include +#include #include #include @@ -137,6 +139,52 @@ static std::size_t stored_entry_count(const std::string& path) return entry_files(path).size(); } +/// Every row a statement call produced. +static std::vector collect_rows(const migraphx::sqlite_stmt::rows& r) +{ + return std::vector(r.begin(), r.end()); +} + +using stored_entries = std::map>; + +/// Every entry a cache directory holds, keyed by key hash, which names each file. +static stored_entries dir_entries(const migraphx::fs::path& dir) +{ + stored_entries result; + auto files = entry_files(dir); + std::transform(files.begin(), files.end(), std::inserter(result, result.end()), [](auto f) { + return std::make_pair(f.stem().string(), migraphx::read_buffer(f)); + }); + return result; +} + +/// Every entry a cache database holds, keyed by key hash. +static stored_entries db_entries(const std::string& path) +{ + stored_entries result; + auto select = migraphx::sqlite::read(path).prepare("SELECT key_hash, entry FROM cache_v1;"); + auto rows = collect_rows(select()); + std::transform(rows.begin(), rows.end(), std::inserter(result, result.end()), [](auto row) { + const auto& blob = row.at("entry").get_binary(); + return std::make_pair(row.at("key_hash").get_string(), + std::vector(blob.begin(), blob.end())); + }); + return result; +} + +/// One of each backend over fresh storage in td: a directory at td/files and a database at +/// db_path(td). Driven directly, these skip binary_cache and its version and device strings. +static std::vector both_backends(const migraphx::tmp_dir& td) +{ + std::vector result; + result.emplace_back(migraphx::gpu::file_binary_cache{td.path / "files"}); + auto db = migraphx::gpu::sqlite_binary_cache::open(db_path(td)); + EXPECT(db.has_value()); + if(db.has_value()) + result.push_back(*db); + return result; +} + TEST_CASE(lookup_records_a_miss) { migraphx::gpu::context ctx; @@ -164,12 +212,9 @@ TEST_CASE(memory_lookup_records_reuse) } // The cases below are written once against a path and registered for both backends. The -// directory registrations are skipped on Windows for now: driving the file backend through -// binary_cache puts entries under version_id()/device_dir(), and write_atomically then creates a -// temp directory inside that, which pushes the file past Windows' MAX_PATH. fs::create_directories -// succeeds because std::filesystem uses the \\?\ prefix, but the std::ofstream in write_buffer -// does not, so every store fails and nothing is persisted. backends_round_trip_through_the_wrapper -// still covers the file backend there, where it is driven directly and the paths are short. +// directory registrations use real temporary paths on purpose: entries sit under +// version_id()/device_dir(), and on Windows that depth once pushed the entry write past +// MAX_PATH, so running them there is what keeps it from coming back. // A second cache shares nothing in memory, so anything it finds came out of storage. static void disk_lookup_body(const std::string& path) @@ -189,13 +234,11 @@ static void disk_lookup_body(const std::string& path) EXPECT(*found->fragment.get_main_module() == *make_code().fragment.get_main_module()); } -#ifndef _WIN32 TEST_CASE(disk_lookup_records_a_hit) { migraphx::tmp_dir td{"binary-cache"}; disk_lookup_body(dir_path(td)); } -#endif TEST_CASE(sqlite_lookup_records_a_hit) { @@ -222,7 +265,6 @@ static void corrupt_entry_body(const std::string& path, EXPECT(reader.get_stats().misses == 1); } -#ifndef _WIN32 TEST_CASE(corrupt_entry_is_ignored) { migraphx::tmp_dir td{"binary-cache"}; @@ -233,7 +275,6 @@ TEST_CASE(corrupt_entry_is_ignored) migraphx::write_buffer(file, std::vector(8, 0)); }); } -#endif TEST_CASE(sqlite_corrupt_entry_is_ignored) { @@ -339,13 +380,11 @@ static void compiling_twice_body(const std::string& path) gpu_result.to_vector())); } -#ifndef _WIN32 TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) { migraphx::tmp_dir td{"binary-cache"}; compiling_twice_body(dir_path(td)); } -#endif TEST_CASE(sqlite_compiling_twice_populates_the_cache_and_matches_reference) { @@ -366,13 +405,11 @@ static void verified_reuse_body(const std::string& path) p.compile(migraphx::make_target("gpu"), options); } -#ifndef _WIN32 TEST_CASE(verified_reuse_matches_fresh_compiles) { migraphx::tmp_dir td{"binary-cache"}; verified_reuse_body(dir_path(td)); } -#endif TEST_CASE(sqlite_verified_reuse_matches_fresh_compiles) { @@ -382,11 +419,6 @@ TEST_CASE(sqlite_verified_reuse_matches_fresh_compiles) // The extension of the path picks the backend and nothing else does, so the only way to see the // choice from outside is the artifact it leaves: a database file, or a directory tree. -// -// The directory half checks that the version tree was laid down rather than that an entry file -// landed in it, because the entry write is what MAX_PATH defeats here. create_directories runs -// before that write and succeeds, and the SQLite backend creates no such tree, so this still -// distinguishes the two backends. TEST_CASE(extension_selects_the_backend) { migraphx::gpu::context ctx; @@ -396,7 +428,12 @@ TEST_CASE(extension_selects_the_backend) migraphx::gpu::binary_cache dir_cache{ migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; dir_cache.insert(ctx, make_entry("in-a-directory")); + auto files = entry_files(dir_td.path); + EXPECT(files.size() == 1); EXPECT(migraphx::fs::is_directory(dir_td.path / version_dir)); + EXPECT(std::all_of(files.begin(), files.end(), [&](const auto& f) { + return f.parent_path().parent_path() == dir_td.path / version_dir; + })); for(const char* name : {"cache.db", "cache.sqlite"}) { @@ -434,6 +471,155 @@ TEST_CASE(unusable_database_degrades_to_memory) EXPECT(reader.get_stats().misses == 1); } +// A cache path that already holds something other than a cache database is left alone: the +// cache runs from memory, and the file is not overwritten. +TEST_CASE(not_a_database_degrades_to_memory) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + auto path = db_path(td); + const std::vector garbage(64, 'x'); + migraphx::write_buffer(path, garbage); + + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; + cache.insert(ctx, make_entry("in-memory")); + EXPECT(cache.get(ctx, "in-memory").has_value()); + EXPECT(cache.get_stats().reused == 1); + EXPECT((migraphx::read_buffer(path) == garbage)); +} + +// A database whose cache table has a different shape, as a future or foreign version might +// leave, cannot be stored into or looked up in, so it is skipped rather than half used. +TEST_CASE(incompatible_schema_degrades_to_memory) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + auto path = db_path(td); + migraphx::sqlite::write(path).execute("CREATE TABLE cache_v1 (unrelated INTEGER);"); + + EXPECT(not migraphx::gpu::sqlite_binary_cache::open(path).has_value()); + + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; + cache.insert(ctx, make_entry("in-memory")); + EXPECT(cache.get(ctx, "in-memory").has_value()); + EXPECT(row_count(path, "cache_v1") == 0); +} + +// Both backends store the same serialized entry, so a cache can move between them. +TEST_CASE(backends_store_identical_bytes) +{ + migraphx::gpu::context ctx; + auto e = make_entry("interchange"); + + migraphx::tmp_dir dir_td{"binary-cache"}; + migraphx::gpu::binary_cache dir_cache{ + migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; + dir_cache.insert(ctx, e); + auto files = entry_files(dir_td.path); + EXPECT(files.size() == 1); + auto from_file = migraphx::read_buffer(files.front()); + EXPECT(not from_file.empty()); + + migraphx::tmp_dir db_td{"binary-cache"}; + auto path = db_path(db_td); + migraphx::gpu::binary_cache db_cache{migraphx::gpu::binary_cache_settings{path, false}}; + db_cache.insert(ctx, e); + + auto select = migraphx::sqlite::read(path).prepare("SELECT entry FROM cache_v1;"); + auto rows = select(); + auto it = rows.begin(); + EXPECT(it != rows.end()); + auto row = *it; + const auto& from_db = row.at("entry").get_binary(); + EXPECT((std::vector(from_db.begin(), from_db.end()) == from_file)); +} + +// A whole compile against each backend has to leave the same kernels behind: the same key +// hashes, each with the same bytes. That makes the choice of backend purely a storage decision. +TEST_CASE(backends_hold_the_same_entries_after_a_compile) +{ + migraphx::tmp_dir dir_td{"binary-cache"}; + migraphx::tmp_dir db_td{"binary-cache"}; + auto path = db_path(db_td); + + auto p_dir = pointwise_program(); + p_dir.compile(migraphx::make_target("gpu"), cache_options(dir_path(dir_td))); + auto p_db = pointwise_program(); + p_db.compile(migraphx::make_target("gpu"), cache_options(path)); + + auto from_dir = dir_entries(dir_td.path); + auto from_db = db_entries(path); + EXPECT(not from_dir.empty()); + EXPECT(from_dir.size() == from_db.size()); + EXPECT((from_dir == from_db)); +} + +// An entry copied from one backend into the other is a hit there and decodes to the same code, +// so an existing cache can be converted rather than rebuilt. The only translation is the +// version, which a directory names with the short id and a database records in full. +TEST_CASE(entries_move_between_backends) +{ + migraphx::gpu::context ctx; + const auto& short_version = migraphx::gpu::binary_cache::version_id(true); + const auto& long_version = migraphx::gpu::binary_cache::version_id(false); + auto e = make_entry("moving"); + auto expected = make_code(); + + // Directory to database. + { + migraphx::tmp_dir dir_td{"binary-cache"}; + migraphx::tmp_dir db_td{"binary-cache"}; + migraphx::gpu::binary_cache writer{ + migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; + writer.insert(ctx, e); + auto files = entry_files(dir_td.path); + EXPECT(files.size() == 1); + + const auto& file = files.front(); + auto device = file.parent_path().filename().string(); + auto db = migraphx::gpu::sqlite_binary_cache::open(db_path(db_td)); + EXPECT(db.has_value()); + db->store(long_version, device, file.stem().string(), e, migraphx::read_buffer(file)); + + migraphx::gpu::binary_cache reader{ + migraphx::gpu::binary_cache_settings{db_path(db_td), false}}; + auto found = reader.get(ctx, e.key); + EXPECT(found.has_value()); + EXPECT(reader.get_stats().hits == 1); + EXPECT(*found->fragment.get_main_module() == *expected.fragment.get_main_module()); + } + + // Database to directory. + { + migraphx::tmp_dir db_td{"binary-cache"}; + migraphx::tmp_dir dir_td{"binary-cache"}; + migraphx::gpu::binary_cache writer{ + migraphx::gpu::binary_cache_settings{db_path(db_td), false}}; + writer.insert(ctx, e); + + auto select = migraphx::sqlite::read(db_path(db_td)) + .prepare("SELECT version, device, key_hash, entry FROM cache_v1;"); + auto rows = collect_rows(select()); + EXPECT(rows.size() == 1); + const auto& row = rows.front(); + EXPECT(row.at("version").get_string() == long_version); + const auto& blob = row.at("entry").get_binary(); + migraphx::gpu::file_binary_cache files{dir_td.path}; + files.store(short_version, + row.at("device").get_string(), + row.at("key_hash").get_string(), + e, + std::vector(blob.begin(), blob.end())); + + migraphx::gpu::binary_cache reader{ + migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; + auto found = reader.get(ctx, e.key); + EXPECT(found.has_value()); + EXPECT(reader.get_stats().hits == 1); + EXPECT(*found->fragment.get_main_module() == *expected.fragment.get_main_module()); + } +} + // Two connections over one database, as two processes compiling against a shared cache would // have. This cannot be two sqlite_binary_cache objects: only open() populates one, and it hands // back the type-erased wrapper. @@ -539,25 +725,72 @@ TEST_CASE(sqlite_read_only_database_still_serves_hits) path, migraphx::fs::perms::owner_write, migraphx::fs::perm_options::add); } -// version and device lead the primary key because they are what separates entries this build -// may use from entries it may not, so a row stored under one must not be served under another. -TEST_CASE(sqlite_scopes_entries_by_version_and_device) +// version and device separate entries this build may use from entries it may not, so an entry +// stored under one must not be served under another, whichever backend holds it. +TEST_CASE(backends_scope_entries_by_version_and_device) { migraphx::tmp_dir td{"binary-cache"}; - auto backend = migraphx::gpu::sqlite_binary_cache::open(db_path(td)); - EXPECT(backend.has_value()); - const std::vector blob{'p', 'a', 'y'}; - backend->store("v1", "dev1", "k", make_entry("k"), blob); + for(auto& backend : both_backends(td)) + { + backend.store("v1", "dev1", "k", make_entry("k"), blob); - EXPECT(backend->load("v1", "dev1", "k").has_value()); - EXPECT(not backend->load("v2", "dev1", "k").has_value()); - EXPECT(not backend->load("v1", "dev2", "k").has_value()); + EXPECT(backend.load("v1", "dev1", "k").has_value()); + EXPECT(not backend.load("v2", "dev1", "k").has_value()); + EXPECT(not backend.load("v1", "dev2", "k").has_value()); + } +} + +// Storing a key twice replaces the entry rather than accumulating or failing. Two processes +// compiling the same kernel is benign for exactly this reason. +TEST_CASE(backends_store_overwrites_in_place) +{ + migraphx::tmp_dir td{"binary-cache"}; + const std::vector replacement{'n', 'e', 'w'}; + for(auto& backend : both_backends(td)) + { + backend.store("v", "dev", "k", make_entry("k"), {'o', 'l', 'd'}); + backend.store("v", "dev", "k", make_entry("k"), replacement); + + auto got = backend.load("v", "dev", "k"); + EXPECT(got.has_value()); + EXPECT((*got == replacement)); + } + EXPECT(row_count(db_path(td), "cache_v1") == 1); + EXPECT(entry_files(td.path / "files").size() == 1); +} + +// Publishing an entry goes through a temporary beside it, and nothing of that may be left once +// the entry is in place, including when an existing entry is replaced. +TEST_CASE(file_store_leaves_only_entries_behind) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + migraphx::gpu::binary_cache_settings settings{dir_path(td), false}; + + migraphx::gpu::binary_cache first{settings}; + first.insert(ctx, make_entry("one")); + first.insert(ctx, make_entry("two")); + // A second cache stores the same key again, over the file the first one published. + migraphx::gpu::binary_cache second{settings}; + second.insert(ctx, make_entry("one")); + + std::vector files; + std::vector dirs; + for(const auto& item : migraphx::fs::recursive_directory_iterator(td.path)) + { + if(item.is_directory()) + dirs.push_back(item.path()); + else + files.push_back(item.path()); + } + EXPECT(files.size() == 2); + EXPECT(std::all_of( + files.begin(), files.end(), [](const auto& f) { return f.extension() == ".mxr"; })); + // Just the version directory and the device directory inside it. + EXPECT(dirs.size() == 2); } -// Storing a key twice replaces the row rather than accumulating or failing, the way the file -// backend's publish-by-rename overwrites in place. Two processes compiling the same kernel is -// benign for exactly this reason. // Storage is opened by the first lookup or insert, not by constructing the cache, since every // context makes one whether or not it ever compiles anything. TEST_CASE(storage_is_opened_on_first_use) @@ -572,19 +805,18 @@ TEST_CASE(storage_is_opened_on_first_use) EXPECT(migraphx::fs::exists(path)); } -// Inserts made inside a batch are committed together when it ends, and are all there after. -TEST_CASE(sqlite_batched_inserts_are_all_committed) +// Inserts made inside a batch are all there once it ends. The database commits them together; +// the directory backend has no batching and stores each one as it comes. +static void batched_inserts_body(const std::string& path) { - migraphx::tmp_dir td{"binary-cache"}; migraphx::gpu::context ctx; - auto path = db_path(td); migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; { migraphx::gpu::binary_cache::store_batch batch{cache}; cache.insert(ctx, make_entry("first")); cache.insert(ctx, make_entry("second")); } - EXPECT(row_count(path, "cache_v1") == 2); + EXPECT(stored_entry_count(path) == 2); migraphx::gpu::binary_cache reader{migraphx::gpu::binary_cache_settings{path, false}}; EXPECT(reader.get(ctx, "first").has_value()); @@ -592,21 +824,35 @@ TEST_CASE(sqlite_batched_inserts_are_all_committed) EXPECT(reader.get_stats().hits == 2); } -TEST_CASE(sqlite_store_overwrites_in_place) +TEST_CASE(batched_inserts_are_all_stored) { migraphx::tmp_dir td{"binary-cache"}; - auto path = db_path(td); - auto backend = migraphx::gpu::sqlite_binary_cache::open(path); - EXPECT(backend.has_value()); + batched_inserts_body(dir_path(td)); +} - const std::vector replacement{'n', 'e', 'w'}; - backend->store("v", "dev", "k", make_entry("k"), {'o', 'l', 'd'}); - backend->store("v", "dev", "k", make_entry("k"), replacement); +TEST_CASE(sqlite_batched_inserts_are_all_committed) +{ + migraphx::tmp_dir td{"binary-cache"}; + batched_inserts_body(db_path(td)); +} - EXPECT(row_count(path, "cache_v1") == 1); - auto got = backend->load("v", "dev", "k"); - EXPECT(got.has_value()); - EXPECT((*got == replacement)); +// A batch holds the database's write lock, so another connection must be able to write again as +// soon as it ends. +TEST_CASE(sqlite_batch_releases_the_database) +{ + migraphx::tmp_dir td{"binary-cache"}; + migraphx::gpu::context ctx; + auto path = db_path(td); + migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; + { + migraphx::gpu::binary_cache::store_batch batch{cache}; + cache.insert(ctx, make_entry("batched")); + } + + auto other = migraphx::gpu::sqlite_binary_cache::open(path); + EXPECT(other.has_value()); + other->store("v", "dev", "k", make_entry("k"), {'x'}); + EXPECT(row_count(path, "cache_v1") == 2); } // The backend layer moves opaque bytes and never decodes them, so a payload that is not even @@ -618,14 +864,7 @@ TEST_CASE(backends_round_trip_through_the_wrapper) const std::vector blob{'\0', 'n', 'o', 't', '\0', 'm', 's', 'g', '\xff'}; auto e = make_entry("opaque"); - auto db = migraphx::gpu::sqlite_binary_cache::open((td.path / "cache.db").string()); - EXPECT(db.has_value()); - - std::vector backends; - backends.emplace_back(migraphx::gpu::file_binary_cache{td.path / "files"}); - backends.push_back(*db); - - for(auto& backend : backends) + for(auto& backend : both_backends(td)) { EXPECT(not backend.load("v", "dev", "k").has_value()); backend.store("v", "dev", "k", e, blob); diff --git a/test/sqlite.cpp b/test/sqlite.cpp index 71e5bdba20f..8f84e111c6c 100644 --- a/test/sqlite.cpp +++ b/test/sqlite.cpp @@ -21,10 +21,13 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ +#include #include #include #include +#include #include +#include #include /// Every row a call produced, so a test can count and inspect them. @@ -132,6 +135,59 @@ TEST_CASE(abandoned_rows_release_the_database) EXPECT(writer.execute("SELECT id FROM t;").size() == 3); } +// Each column comes back as the value type matching what sqlite stored, keyed by its name. +TEST_CASE(rows_convert_column_types) +{ + migraphx::tmp_dir td{}; + auto db = migraphx::sqlite::write(td.path / "types.db"); + auto select = + db.prepare("SELECT 42 AS i, 2.5 AS f, 'text' AS t, x'00ff' AS b, NULL AS n, ?1 AS p;"); + + auto rows = collect(select(std::int64_t{-7})); + EXPECT(rows.size() == 1); + const auto& row = rows.front(); + EXPECT(row.size() == 6); + EXPECT(row.at("i").is_int64()); + EXPECT(row.at("i").get_int64() == 42); + EXPECT(row.at("f").is_float()); + EXPECT(migraphx::float_equal(row.at("f").get_float(), 2.5)); + EXPECT(row.at("t").get_string() == "text"); + EXPECT(row.at("b").get_binary() == migraphx::value::binary{std::vector{0, 255}}); + EXPECT(row.at("n").is_null()); + EXPECT(row.at("p").get_int64() == -7); +} + +// A statement returning many rows yields each in turn, and calling it again starts over. +TEST_CASE(rows_iterate_in_order_and_restart) +{ + migraphx::tmp_dir td{}; + auto db = migraphx::sqlite::write(td.path / "many.db"); + db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY);" + "INSERT INTO t (id) VALUES (1), (2), (3);"); + auto select = db.prepare("SELECT id FROM t WHERE id >= ?1 ORDER BY id;"); + + auto ids = [&](std::int64_t from) { + std::vector result; + auto rows = select(from); + std::transform(rows.begin(), rows.end(), std::back_inserter(result), [](const auto& row) { + return row.at("id").get_int64(); + }); + return result; + }; + EXPECT((ids(1) == std::vector{1, 2, 3})); + EXPECT((ids(2) == std::vector{2, 3})); + EXPECT(ids(4).empty()); +} + +TEST_CASE(read_only_matches_how_it_was_opened) +{ + migraphx::tmp_dir td{}; + auto path = td.path / "mode.db"; + migraphx::sqlite::write(path).execute("CREATE TABLE t (id INTEGER PRIMARY KEY);"); + EXPECT(not migraphx::sqlite::write(path).read_only()); + EXPECT(migraphx::sqlite::read(path).read_only()); +} + TEST_CASE(unprepared_statement_throws) { migraphx::sqlite_stmt stmt; From d708b88d2d4b0dbebc32c276977b8823df7cc121 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Fri, 25 Sep 2026 15:55:34 +0200 Subject: [PATCH 35/49] migraphx simplify Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/dev/triage-migraphx.rst | 11 +- src/include/migraphx/sqlite.hpp | 29 ++- src/sqlite.cpp | 79 +++--- src/targets/gpu/binary_cache.cpp | 74 +++--- src/targets/gpu/compile_ops.cpp | 6 +- src/targets/gpu/file_binary_cache.cpp | 20 +- .../gpu/include/migraphx/gpu/binary_cache.hpp | 19 +- .../migraphx/gpu/binary_cache_backend.hpp | 23 +- .../migraphx/gpu/binary_cache_entry.hpp | 6 +- .../migraphx/gpu/file_binary_cache.hpp | 2 +- .../migraphx/gpu/sqlite_binary_cache.hpp | 23 +- src/targets/gpu/sqlite_binary_cache.cpp | 53 ++--- test/gpu/binary_cache.cpp | 225 +++++++----------- tools/include/gpu/binary_cache_backend.hpp | 23 +- 14 files changed, 254 insertions(+), 339 deletions(-) diff --git a/docs/dev/triage-migraphx.rst b/docs/dev/triage-migraphx.rst index efdd3019081..6f1dcac722b 100644 --- a/docs/dev/triage-migraphx.rst +++ b/docs/dev/triage-migraphx.rst @@ -176,8 +176,8 @@ directories for builds you no longer use: rm -r $HOME/.cache/migraphx/v1-hip22.0.* A path ending in ``.db`` or ``.sqlite`` keeps the cache in a single SQLite database instead of a -directory, which is easier to copy between machines and can be shared by processes compiling at -the same time. Missing parent directories are created, as for a directory cache: +directory, which is easier to copy between machines. Like a directory cache it can be used by +several processes at once, and missing parent directories are created: .. code-block:: bash @@ -192,11 +192,12 @@ with the ``sqlite3`` shell. Deleting rows does not shrink the file until it is v sqlite3 $HOME/.cache/migraphx/kernels.db \ "SELECT version, op_name, count(*) FROM cache_v1 GROUP BY version, op_name;" sqlite3 $HOME/.cache/migraphx/kernels.db \ - "DELETE FROM cache_v1 WHERE version LIKE 'v1-hip22.0.%'; VACUUM;" + "DELETE FROM cache_v1 WHERE version GLOB 'v1-hip22.0.*'; VACUUM;" A database that cannot be written to, such as a shared cache installed read-only, is still used -for lookups, and newly compiled kernels are kept in memory only. A database that cannot be opened -at all is skipped with a warning and the compile proceeds without a disk cache. +for lookups, and newly compiled kernels are kept in memory only. A database that cannot be +opened, or whose cache table has an unexpected layout, is skipped with a warning and the compile +proceeds without a disk cache. The same settings are available as backend options, which take precedence over the environment and are how tests configure the cache: diff --git a/src/include/migraphx/sqlite.hpp b/src/include/migraphx/sqlite.hpp index f65217ae2e0..1a9dc6a5355 100644 --- a/src/include/migraphx/sqlite.hpp +++ b/src/include/migraphx/sqlite.hpp @@ -31,12 +31,12 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -50,10 +50,10 @@ struct sqlite_stmt_impl; /// never outlive it. Copies share the same statement. /// /// Calling it with arguments runs it: the arguments are bound to the parameters in order and the -/// result comes back as a range of rows. Preparing once and calling many times is the point. +/// result comes back as a range of rows. Since copies share one statement, only the rows of one +/// call may be alive at a time. /// -/// Not thread safe: one statement may be used by one thread at a time, even though the -/// connection itself is serialized. +/// Not thread safe: use a statement from one thread at a time. struct MIGRAPHX_EXPORT sqlite_stmt { struct rows; @@ -66,9 +66,11 @@ struct MIGRAPHX_EXPORT sqlite_stmt { if(not valid()) MIGRAPHX_THROW("sqlite: calling a statement that was never prepared"); + assert(sizeof...(Ts) == parameter_count()); // Anything left from the previous call, bindings or an unfinished result, goes first. reset(); - sequence_c([&](auto... is) { swallow{(bind(int{is + 1}, xs), 0)...}; }); + int i = 0; + each_args([&](const auto& x) { bind(++i, x); }, xs...); return rows{*this}; } @@ -80,6 +82,8 @@ struct MIGRAPHX_EXPORT sqlite_stmt void bind(int i, std::int64_t x) const; void bind(int i, const std::vector& blob) const; + std::size_t parameter_count() const; + /// Step once. True when a row is available, false when the statement is done. bool step() const; @@ -103,7 +107,6 @@ struct MIGRAPHX_EXPORT sqlite_stmt /// read lock on the database until then, which would stall writers in other processes. struct sqlite_stmt::rows { - explicit rows(sqlite_stmt s) : stmt(std::move(s)), first(stmt.step()) {} // Only ever a prvalue returned from a call, so it never needs copying or moving, and a copy // would reset the statement out from under the original. rows(const rows&) = delete; @@ -118,17 +121,22 @@ struct sqlite_stmt::rows using reference = value_type; using difference_type = std::ptrdiff_t; using iterator_category = std::input_iterator_tag; - using pointer = std::add_pointer_t>; + using pointer = value*; iterator() = default; iterator(const rows* pparent, bool pavailable) : parent(pparent), available(pavailable) {} - reference operator*() const { return parent->stmt.to_value(); } + reference operator*() const + { + assert(parent != nullptr and available); + return parent->stmt.to_value(); + } template static void increment(U& x) { + assert(x.parent != nullptr and x.available); x.available = x.parent->stmt.step(); } @@ -147,6 +155,9 @@ struct sqlite_stmt::rows iterator end() const { return {this, false}; } private: + friend struct sqlite_stmt; + explicit rows(sqlite_stmt s) : stmt(std::move(s)), first(stmt.step()) {} + sqlite_stmt stmt; bool first = false; }; @@ -172,8 +183,6 @@ struct MIGRAPHX_EXPORT sqlite /// has write-protected, in which case sqlite quietly opens it read-only; this is how to tell. bool read_only() const; - bool valid() const { return impl != nullptr; } - private: std::shared_ptr impl; }; diff --git a/src/sqlite.cpp b/src/sqlite.cpp index 619089aced5..fb15a844481 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include namespace migraphx { @@ -39,9 +40,8 @@ struct sqlite_impl { sqlite3* get() const { return ptr.get(); } - // Returns false rather than throwing, so callers that treat an unusable database as - // "no cache" do not have to catch. sqlite3_open_v2 hands back a handle even on failure - // (that is where the error message lives), so `ptr` takes ownership either way. + // sqlite3_open_v2 returns a handle even on failure (it carries the error message), so ptr + // takes ownership either way. bool try_open(const fs::path& p, int flags) { sqlite3* ptr_tmp = nullptr; @@ -92,10 +92,8 @@ struct sqlite_stmt_impl sqlite3_stmt* get() const { return ptr.get(); } std::string error_message() const { return db->error_message(); } - // Holding the connection keeps it alive for as long as any statement prepared on it, - // since finalizing after the connection is closed is undefined. Declaration order is - // load-bearing: members destruct in reverse, so ptr is finalized before db is released. - // Do not reorder. + // Holding the connection keeps it alive while any statement on it exists. ptr is declared + // after db so it is finalized first; finalizing after the connection closes is undefined. std::shared_ptr db; sqlite3_stmt_ptr ptr; }; @@ -154,6 +152,8 @@ sqlite_stmt sqlite::prepare(const std::string& sql) result.impl->ptr = sqlite3_stmt_ptr{stmt_tmp}; if(rc != SQLITE_OK) MIGRAPHX_THROW("error preparing '" + sql + "': " + impl->error_message()); + // sqlite succeeds without a statement for text that holds none, such as only a comment. + assert(stmt_tmp != nullptr); return result; } @@ -193,6 +193,11 @@ void sqlite_stmt::bind(int i, const std::vector& blob) const MIGRAPHX_THROW(impl->error_message()); } +std::size_t sqlite_stmt::parameter_count() const +{ + return sqlite3_bind_parameter_count(impl->get()); +} + bool sqlite_stmt::step() const { int rc = sqlite3_step(impl->get()); @@ -205,56 +210,50 @@ bool sqlite_stmt::step() const void sqlite_stmt::reset() const noexcept { - if(impl == nullptr) - return; + assert(impl != nullptr); // The return of sqlite3_reset is the error from the preceding step(), which the caller // has already seen as a throw. There is nothing new to report, and this must not throw. (void)sqlite3_reset(impl->get()); (void)sqlite3_clear_bindings(impl->get()); } +/// Column i of the current row, keyed by its name. static value column_value(sqlite3_stmt* stmt, int i) { - // For text and blobs the data must be fetched before sqlite3_column_bytes: the other order - // can force a type conversion that invalidates the pointer. A zero-length value comes back - // as a null pointer, which still means empty rather than NULL. - switch(sqlite3_column_type(stmt, i)) + std::string name = sqlite3_column_name(stmt, i); + auto type = sqlite3_column_type(stmt, i); + switch(type) { - case SQLITE_INTEGER: return std::int64_t{sqlite3_column_int64(stmt, i)}; - case SQLITE_FLOAT: return sqlite3_column_double(stmt, i); - case SQLITE_TEXT: { - const auto* text = sqlite3_column_text(stmt, i); - auto n = sqlite3_column_bytes(stmt, i); - if(text == nullptr) - return std::string{}; - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - return std::string(reinterpret_cast(text), n); - } + case SQLITE_INTEGER: return value(name, std::int64_t{sqlite3_column_int64(stmt, i)}); + case SQLITE_FLOAT: return value(name, sqlite3_column_double(stmt, i)); + case SQLITE_TEXT: case SQLITE_BLOB: { - const auto* data = static_cast(sqlite3_column_blob(stmt, i)); - auto n = sqlite3_column_bytes(stmt, i); - if(data == nullptr) - return value::binary{}; - return value::binary{data, static_cast(n)}; + // The data must be fetched before sqlite3_column_bytes: the other order can force a + // conversion that invalidates the pointer. Text comes back unchanged through + // sqlite3_column_blob, and a zero-length value as a null pointer, which means empty. + const auto* data = static_cast(sqlite3_column_blob(stmt, i)); + auto bytes = sqlite3_column_bytes(stmt, i); + assert(bytes >= 0); + auto size = data == nullptr ? 0 : static_cast(bytes); + if(type == SQLITE_TEXT) + return value(name, size == 0 ? std::string{} : std::string(data, size)); + return value(name, value::binary{data, size}); } - default: return {}; + default: return value(name, nullptr); } } value sqlite_stmt::to_value() const { auto* stmt = impl->get(); - value::object row; - auto columns = range(sqlite3_column_count(stmt)); - std::transform(columns.begin(), - columns.end(), - std::inserter(row, row.end()), - [&](std::ptrdiff_t i) { - auto col = static_cast(i); - return std::make_pair(std::string{sqlite3_column_name(stmt, col)}, - column_value(stmt, col)); - }); - return row; + // Built as keyed values rather than a map, so a blob is moved into place instead of copied. + std::vector columns; + auto indices = range(sqlite3_column_count(stmt)); + std::transform(indices.begin(), + indices.end(), + std::back_inserter(columns), + [&](std::ptrdiff_t i) { return column_value(stmt, static_cast(i)); }); + return value(columns, /* array_on_empty */ false); } } // namespace MIGRAPHX_INLINE_NS diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index 55fa75fb83d..f7366f48f23 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -105,9 +105,8 @@ static std::string device_dir(const context& ctx) "_wf" + std::to_string(device.get_wavefront_size()); } -/// Turn a stored blob back into an entry. Any failure is just a miss, so a damaged entry costs -/// a recompile and is written over. Shared by every backend, so they all tolerate corruption -/// and survive a hash collision the same way. +/// Turn a stored blob back into an entry. Any failure is a miss, so a damaged entry costs a +/// recompile. static optional decode_entry(const std::vector& blob, const std::string& key, const std::string& key_hash) { @@ -131,54 +130,43 @@ decode_entry(const std::vector& blob, const std::string& key, const std::s return e; } -// Select the storage backend by file type, the same rule make_problem_cache_backend applies in -// problem_cache.cpp: a ".db"/".sqlite" path is a SQLite database, anything else is a directory -// of entries. -static bool is_database_path(const std::string& path) -{ - return ends_with(path, ".db") or ends_with(path, ".sqlite"); -} - -static optional make_binary_cache_backend(const std::string& path) -{ - if(is_database_path(path)) - return sqlite_binary_cache::open(path); // nullopt when the database is unusable - return binary_cache_backend{file_binary_cache{path}}; -} - binary_cache::binary_cache(binary_cache_settings s) : settings(std::move(s)) {} -// Nothing can be persisted safely when the compiler cannot be identified, since entries from -// different toolchains would be indistinguishable. That is a property of the cache rather than -// of the storage medium, so it is checked here instead of in each backend. +// The storage backend is selected by file type, the same rule make_problem_cache_backend applies +// in problem_cache.cpp: a ".db"/".sqlite" path is a SQLite database, anything else is a +// directory of entries. A directory is named with the short version id to keep paths short; a +// database records the full id, which is self-describing. Nothing is persisted when the compiler +// cannot be identified, since entries from different toolchains would be indistinguishable. binary_cache_backend* binary_cache::get_backend() { - if(not backend_opened) - { - backend_opened = true; - // Checked first so that a memory-only cache never compiles the version probe. - if(not settings.path.empty()) - { - // The version names a directory for the file backend, so it is kept short there. A - // database has no such limit and records the full id, which is self-describing. - version = version_id(not is_database_path(settings.path)); - if(not version.empty()) - backend = make_binary_cache_backend(settings.path); - } - } + if(backend_opened) + return backend.has_value() ? &*backend : nullptr; + backend_opened = true; + const auto& path = settings.path; + // Checked first so that a memory-only cache never compiles the version probe. + if(path.empty()) + return nullptr; + const bool database = ends_with(path, ".db") or ends_with(path, ".sqlite"); + version = version_id(not database); + if(version.empty()) + return nullptr; + if(not database) + backend = binary_cache_backend{file_binary_cache{path}}; + else if(auto db = sqlite_binary_cache::open(path)) + backend = binary_cache_backend{std::move(*db)}; return backend.has_value() ? &*backend : nullptr; } -binary_cache::store_batch::store_batch(binary_cache& c) : cache(&c) +binary_cache::store_batch::store_batch(binary_cache& c) : backend(c.get_backend()) { - if(auto* b = cache->get_backend()) - b->begin_batch(); + if(backend != nullptr) + backend->begin_batch(); } binary_cache::store_batch::~store_batch() { - if(auto* b = cache->get_backend()) - b->end_batch(); + if(backend != nullptr) + backend->end_batch(); } optional binary_cache::get(const context& ctx, const std::string& key) @@ -193,8 +181,8 @@ optional binary_cache::get(const context& ctx, const std::string& } if(auto* b = get_backend()) { - // Hashing the key is not free -- it is the whole compile source, which runs to - // kilobytes -- so it is done once and reused for the lookup and any diagnostics. + // The key is the whole compile source, so it is hashed once for the lookup and any + // diagnostics. auto key_hash = md5(key); auto blob = b->load(version, device_dir(ctx), key_hash); if(blob.has_value()) @@ -221,9 +209,7 @@ void binary_cache::insert(const context& ctx, entry e) auto key_hash = md5(e.key); try { - // Serializing inside the try, rather than in the call's argument list, makes a - // failure here a warning like any other storage failure instead of escaping - // insert() and failing the compile. + // A failure to serialize or store is a warning, not a failed compile. auto blob = to_msgpack(migraphx::to_value(e)); b->store(version, device_dir(ctx), key_hash, e, blob); } diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 8ea8289523e..b2e20598c3e 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -808,7 +808,11 @@ struct compile_manager { // Every plan compiles with the same context, so the stores all go to one cache, and // batching them lets its storage commit them together rather than one at a time. - binary_cache::store_batch batch{tasks.front().first->ctx->get_binary_cache()}; + auto* ctx = tasks.front().first->ctx; + assert(std::all_of(tasks.begin(), tasks.end(), [&](const auto& task) { + return task.first->ctx == ctx; + })); + binary_cache::store_batch batch{ctx->get_binary_cache()}; for(const auto& [cp, cell] : tasks) { if(not cell->result.has_value()) diff --git a/src/targets/gpu/file_binary_cache.cpp b/src/targets/gpu/file_binary_cache.cpp index 9e121dc242c..2ccd6f0d32b 100644 --- a/src/targets/gpu/file_binary_cache.cpp +++ b/src/targets/gpu/file_binary_cache.cpp @@ -49,12 +49,9 @@ static fs::path entry_path(const fs::path& root, } /// Publish by rename so a reader never sees a half-written file. The temporary stays beside -/// the destination since the rename is only atomic within one filesystem. -/// -/// It is a sibling file with a short unique suffix rather than a file inside a temporary -/// directory: entries already sit several directories deep, and a nested directory with a -/// fully unique name pushed the path past Windows' MAX_PATH, which std::ofstream cannot open. -/// The suffix only has to keep concurrent writers of the same entry apart. +/// the destination since the rename is only atomic within one filesystem. Its short random +/// suffix keeps concurrent writers of the same entry apart without lengthening an already deep +/// path, which on Windows must stay under MAX_PATH for std::ofstream to open it. static void write_atomically(const fs::path& dest, const std::vector& content) { auto suffix = md5(unique_string("cache")).substr(0, 16); @@ -100,15 +97,8 @@ void file_binary_cache::store(const std::string& version, auto path = entry_path(root, version, device, key_hash); // The content is decided entirely by the key, so a writer that loses the publish race // replaces the file with the same bytes and no locking is needed. - try - { - fs::create_directories(path.parent_path()); - write_atomically(path, blob); - } - catch(const std::exception& ex) - { - log::warn() << "Failed to write binary cache entry " << path << ": " << ex.what(); - } + fs::create_directories(path.parent_path()); + write_atomically(path, blob); } } // namespace gpu diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index 887915135e2..f136b1b5dc3 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -30,8 +30,6 @@ #include #include #include -#include -#include #include #include #include @@ -58,9 +56,9 @@ struct binary_cache_settings /** * Compiled kernels, keyed by a string describing what the compiler was given. * - * Results are held in memory for the life of the context and, when a cache directory is - * configured, written to disk so later runs can reuse them. Things outside the key, such as the - * compiler and the embedded kernel headers, are separated by the directory the entries live in. + * Results are held in memory for the life of the context and, when a cache path is configured, + * written to disk so later runs can reuse them. Things outside the key, such as the compiler and + * the embedded kernel headers, are separated by the version the entries are stored under. * * This is not thread safe, and deliberately so. Every key is known before any compile begins, so * the compile pass looks results up and stores them in serial passes on either side of its @@ -68,8 +66,7 @@ struct binary_cache_settings */ struct MIGRAPHX_GPU_EXPORT binary_cache { - /// What gets stored for one compiled kernel. Defined in binary_cache_entry.hpp so the - /// storage backends can name it; the alias keeps binary_cache::entry working. + /// What gets stored for one compiled kernel; see binary_cache_entry.hpp. using entry = binary_cache_entry; /// Counts of what the cache did. @@ -98,14 +95,14 @@ struct MIGRAPHX_GPU_EXPORT binary_cache ~store_batch(); private: - binary_cache* cache; + binary_cache_backend* backend; }; - /// Nothing is opened here; storage is set up by the first lookup or insert, so a context - /// that never compiles never touches the disk or probes the compiler. + /// Nothing is opened here; storage is set up by the first lookup, insert or store_batch, so + /// a context that never compiles never touches the disk or probes the compiler. explicit binary_cache(binary_cache_settings s = {}); - /// Look up a key, consulting memory first and then the cache directory. + /// Look up a key, consulting memory first and then the storage backend. optional get(const context& ctx, const std::string& key); /// Record a compiled result under its key. diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp index 106cf627d91..2c9d166bde4 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp @@ -37,12 +37,11 @@ // Notes: // * binary_cache_entry is defined in ; // the include below pulls in its full definition. -// * Backends typically own non-trivial resources (a cache directory, a SQLite -// connection) and are not meaningfully copyable beyond shared ownership. -// * The members are non-const: binary_cache::get and insert are themselves -// non-const, so nothing forces a const qualifier here, and a backend that -// tracks an open batch needs the mutability. A backend may still declare -// them const. +// * Backends must be copyable: the wrapper shares T and clones it on a +// non-const call while the handle is shared. sqlite_binary_cache shares its +// connection across copies. +// * The members are non-const so a backend can track an open batch; a backend +// may still declare them const. // #ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP #define MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP @@ -70,13 +69,12 @@ namespace gpu { /// Type-erased interface for binary-cache storage backends. /// /// A backend persists serialized binary_cache_entry blobs to some medium (a -/// directory of files, a SQLite database, an in-memory map for tests). Entries -/// are addressed by three strings the caller has already computed: +/// directory of files or a SQLite database). Entries are addressed by three +/// strings the caller has already computed: /// /// * `version` -- binary_cache::version_id(), identifying the toolchain and /// the embedded kernel sources that produced the entry. Never empty; the -/// caller skips persistence entirely when it is. The short form is used -/// for directories and the full form for databases. +/// caller skips persistence entirely when it is. /// * `device` -- the GPU the entry was compiled for. /// * `key_hash` -- md5 of the compile key. A hash rather than the key itself /// because a file backend needs a short name; a collision is harmless, @@ -108,8 +106,9 @@ struct binary_cache_backend /// decided entirely by the key, so a writer that loses a race replaces the /// entry with equivalent bytes. /// - /// Must not throw. A failure to store costs a recompile next run, nothing - /// more, and the caller still keeps the result in memory. + /// May throw: the caller reports a failed store as a warning. It costs a + /// recompile next run, nothing more, and the caller still keeps the result + /// in memory. void store(const std::string& version, const std::string& device, const std::string& key_hash, diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache_entry.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache_entry.hpp index 9cbf68a59fa..13d54117264 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache_entry.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache_entry.hpp @@ -37,11 +37,7 @@ namespace gpu { /// What gets stored for one compiled kernel. The op name, problem and solution are stored for /// offline inspection; only the key is checked when an entry is loaded. -/// -/// This lives in its own header, rather than nested inside binary_cache, so that the -/// binary_cache_backend interface can name it without including binary_cache.hpp -- which in -/// turn includes the backend header. Same reason cache_device_key.hpp exists for the problem -/// cache. binary_cache::entry remains an alias for it. +/// In its own header so binary_cache_backend.hpp can use it without a circular include. struct binary_cache_entry { std::string key = {}; diff --git a/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp index 10de1884176..ffc37708600 100644 --- a/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp @@ -38,7 +38,7 @@ namespace gpu { // A binary_cache_backend that keeps entries as files under a root directory, laid out // ///.mxr. The version directory is named after the build that -// wrote it, so the tree is self-describing. Path resolution is the caller's job. +// wrote it, so the tree is self-describing. struct MIGRAPHX_GPU_EXPORT file_binary_cache { optional> diff --git a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp index 0efd4e289e5..118e15c1e89 100644 --- a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp @@ -27,7 +27,6 @@ #include #include -#include #include #include #include @@ -46,14 +45,13 @@ namespace gpu { // target: migraphx/sqlite.hpp forward-declares both impl types and never includes sqlite3.h. struct MIGRAPHX_GPU_EXPORT sqlite_binary_cache { - /// Open the database, create the schema and prepare the statements, and return the wrapper - /// so the caller can hand the result straight back. + /// Open the database at path, creating the schema if it is writable. /// - /// A database that can only be read gives a backend that serves lookups and ignores stores; - /// it is used as it stands, without creating the schema. Returns nullopt when the database - /// cannot be opened at all or entries cannot be looked up in it, so an unusable database - /// leaves the cache memory-only rather than raising an error. - static optional open(const std::string& path); + /// A database that can only be read serves lookups and ignores stores; it is used as it + /// stands, without creating the schema. Returns nullopt when the database cannot be opened + /// at all or entries cannot be looked up in it, so an unusable database leaves the cache + /// memory-only rather than raising an error. + static optional open(const std::string& path); optional> load(const std::string& version, const std::string& device, const std::string& key_hash) const; @@ -69,12 +67,9 @@ struct MIGRAPHX_GPU_EXPORT sqlite_binary_cache void end_batch(); private: - sqlite db = {}; - sqlite_stmt get_stmt = {}; - sqlite_stmt store_stmt = {}; - sqlite_stmt begin_stmt = {}; - sqlite_stmt commit_stmt = {}; - sqlite_stmt rollback_stmt = {}; + sqlite db = {}; + sqlite_stmt get_stmt = {}; + sqlite_stmt store_stmt = {}; /// Whether begin_batch opened a transaction that end_batch still has to close. bool in_batch = false; }; diff --git a/src/targets/gpu/sqlite_binary_cache.cpp b/src/targets/gpu/sqlite_binary_cache.cpp index de0f1ae229b..2140d550013 100644 --- a/src/targets/gpu/sqlite_binary_cache.cpp +++ b/src/targets/gpu/sqlite_binary_cache.cpp @@ -23,6 +23,7 @@ * */ #include +#include #include #include #include @@ -34,9 +35,6 @@ namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { namespace gpu { -// Compile-time confirmation that sqlite_binary_cache satisfies the backend concept. If a method -// signature drifts, this assertion fires at the definition site rather than at some far-away -// usage. static_assert(std::is_constructible{}, "sqlite_binary_cache must satisfy the binary_cache_backend concept"); @@ -77,8 +75,8 @@ constexpr const char* get_sql = // INSERT OR REPLACE is the analogue of the file backend's publish-by-rename: the content is // decided entirely by the key, so two processes compiling the same kernel is benign and the // last writer wins with equivalent bytes. The timestamp is computed by the database rather -// than the process so that rows written by different machines stay comparable; nothing reads -// it yet, it is there to make pruning an old cache by age possible. +// than the process so that rows written by different machines stay comparable. MIGraphX never +// reads it; it lets a cache be pruned by age. constexpr const char* store_sql = "INSERT OR REPLACE INTO cache_v1" " (version, device, key_hash, op_name, problem, solution, entry, timestamp)" @@ -94,7 +92,7 @@ constexpr const char* rollback_sql = "ROLLBACK;"; } // namespace -optional sqlite_binary_cache::open(const std::string& path) +optional sqlite_binary_cache::open(const std::string& path) { sqlite_binary_cache r; try @@ -127,11 +125,8 @@ optional sqlite_binary_cache::open(const std::string& path } else { - (void)r.db.execute(schema_sql); - r.store_stmt = r.db.prepare(store_sql); - r.begin_stmt = r.db.prepare(begin_sql); - r.commit_stmt = r.db.prepare(commit_sql); - r.rollback_stmt = r.db.prepare(rollback_sql); + r.db.execute(schema_sql); + r.store_stmt = r.db.prepare(store_sql); } // Without a working lookup there is no cache, so this failure disables the backend. That // includes a read-only database that was never given the schema. @@ -142,15 +137,13 @@ optional sqlite_binary_cache::open(const std::string& path log::warn() << "Disabling the binary cache at " << path << ": " << ex.what(); return nullopt; } - return binary_cache_backend{std::move(r)}; + return r; } optional> sqlite_binary_cache::load(const std::string& version, const std::string& device, const std::string& key_hash) const { - if(not get_stmt.valid()) - return nullopt; try { // The primary key makes this at most one row. @@ -176,34 +169,26 @@ void sqlite_binary_cache::store(const std::string& version, const binary_cache_entry& e, const std::vector& blob) const { + // Not prepared for a read-only database, whose stores are skipped. if(not store_stmt.valid()) return; - try - { - // The json strings are temporaries, which is safe because binding copies immediately. - // An insert returns no rows, and it has run by the time the call returns. - store_stmt(version, - device, - key_hash, - e.op_name, - to_json_string(e.problem), - to_json_string(e.solution), - blob); - } - catch(const std::exception& ex) - { - log::warn() << "Failed to write binary cache entry " << key_hash << ": " << ex.what(); - } + store_stmt(version, + device, + key_hash, + e.op_name, + to_json_string(e.problem), + to_json_string(e.solution), + blob); } void sqlite_binary_cache::begin_batch() { assert(not in_batch); - if(not begin_stmt.valid()) + if(not store_stmt.valid()) return; try { - begin_stmt(); + db.execute(begin_sql); in_batch = true; } catch(const std::exception& ex) @@ -220,7 +205,7 @@ void sqlite_binary_cache::end_batch() in_batch = false; try { - commit_stmt(); + db.execute(commit_sql); } catch(const std::exception& ex) { @@ -229,7 +214,7 @@ void sqlite_binary_cache::end_batch() log::warn() << "Failed to commit binary cache entries: " << ex.what(); try { - rollback_stmt(); + db.execute(rollback_sql); } catch(const std::exception& rex) { diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 1967581d4d6..dcd083eefc8 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -30,11 +30,11 @@ #include #include #include +#include #include #include #include #include -#include #include #include #include @@ -52,7 +52,6 @@ #include #include #include -#include #include #include #include @@ -102,8 +101,7 @@ static migraphx::gpu::binary_cache::entry make_entry(const std::string& key) return e; } -// The storage backend is chosen by the extension of the cache path, so a case that has to hold -// for both is written once against a path and registered twice, once with each of these. +// The storage backend is chosen by the extension of the cache path. static std::string dir_path(const migraphx::tmp_dir& td) { return td.path.string(); } static std::string db_path(const migraphx::tmp_dir& td) { return (td.path / "cache.db").string(); } @@ -111,11 +109,12 @@ static std::string db_path(const migraphx::tmp_dir& td) { return (td.path / "cac static std::vector entry_files(const migraphx::fs::path& dir) { std::vector result; - for(const auto& file : migraphx::fs::recursive_directory_iterator(dir)) - { - if(file.path().extension() == ".mxr") - result.push_back(file.path()); - } + migraphx::transform_if( + migraphx::fs::recursive_directory_iterator{dir}, + migraphx::fs::recursive_directory_iterator{}, + std::back_inserter(result), + [](const auto& file) { return file.path().extension() == ".mxr"; }, + [](const auto& file) { return file.path(); }); return result; } @@ -129,22 +128,6 @@ static std::size_t row_count(const std::string& path, const std::string& table) return std::stoul(rows.front().at("n")); } -/// How many entries a cache path holds, whichever backend wrote them. The extensions must match -/// the ones make_binary_cache_backend routes to the SQLite backend, or this silently counts -/// files in a directory that does not exist and reports zero. -static std::size_t stored_entry_count(const std::string& path) -{ - if(migraphx::ends_with(path, ".db") or migraphx::ends_with(path, ".sqlite")) - return row_count(path, "cache_v1"); - return entry_files(path).size(); -} - -/// Every row a statement call produced. -static std::vector collect_rows(const migraphx::sqlite_stmt::rows& r) -{ - return std::vector(r.begin(), r.end()); -} - using stored_entries = std::map>; /// Every entry a cache directory holds, keyed by key hash, which names each file. @@ -163,7 +146,7 @@ static stored_entries db_entries(const std::string& path) { stored_entries result; auto select = migraphx::sqlite::read(path).prepare("SELECT key_hash, entry FROM cache_v1;"); - auto rows = collect_rows(select()); + auto rows = select(); std::transform(rows.begin(), rows.end(), std::inserter(result, result.end()), [](auto row) { const auto& blob = row.at("entry").get_binary(); return std::make_pair(row.at("key_hash").get_string(), @@ -172,6 +155,33 @@ static stored_entries db_entries(const std::string& path) return result; } +// What a case that must hold for both backends needs to know about each, so the case can be +// written once as a template and registered for both. +struct directory_backend +{ + static std::string path(const migraphx::tmp_dir& td) { return dir_path(td); } + static std::size_t stored(const std::string& p) { return entry_files(p).size(); } + /// Overwrite every stored entry with bytes that do not decode. + static void damage(const std::string& p) + { + auto files = entry_files(p); + std::for_each(files.begin(), files.end(), [](const auto& file) { + migraphx::write_buffer(file, std::vector(8, 0)); + }); + } +}; + +struct database_backend +{ + static std::string path(const migraphx::tmp_dir& td) { return db_path(td); } + static std::size_t stored(const std::string& p) { return row_count(p, "cache_v1"); } + /// Overwrite every stored entry with bytes that do not decode. + static void damage(const std::string& p) + { + migraphx::sqlite::write(p).execute("UPDATE cache_v1 SET entry = zeroblob(8);"); + } +}; + /// One of each backend over fresh storage in td: a directory at td/files and a database at /// db_path(td). Driven directly, these skip binary_cache and its version and device strings. static std::vector both_backends(const migraphx::tmp_dir& td) @@ -181,7 +191,7 @@ static std::vector both_backends(const migr auto db = migraphx::gpu::sqlite_binary_cache::open(db_path(td)); EXPECT(db.has_value()); if(db.has_value()) - result.push_back(*db); + result.emplace_back(std::move(*db)); return result; } @@ -211,16 +221,17 @@ TEST_CASE(memory_lookup_records_reuse) EXPECT(cache.get_stats().misses == 0); } -// The cases below are written once against a path and registered for both backends. The -// directory registrations use real temporary paths on purpose: entries sit under -// version_id()/device_dir(), and on Windows that depth once pushed the entry write past -// MAX_PATH, so running them there is what keeps it from coming back. +// The cases below are written once against a Backend and registered for each. The directory +// registrations use real temporary paths so that on Windows they exercise the full depth of an +// entry path against MAX_PATH. // A second cache shares nothing in memory, so anything it finds came out of storage. -static void disk_lookup_body(const std::string& path) +template +static void disk_lookup_records_a_hit() { + migraphx::tmp_dir td{"binary-cache"}; migraphx::gpu::context ctx; - migraphx::gpu::binary_cache_settings settings{path, false}; + migraphx::gpu::binary_cache_settings settings{Backend::path(td), false}; migraphx::gpu::binary_cache writer{settings}; writer.insert(ctx, make_entry("shared-key")); @@ -233,57 +244,30 @@ static void disk_lookup_body(const std::string& path) EXPECT(reader.get_stats().misses == 0); EXPECT(*found->fragment.get_main_module() == *make_code().fragment.get_main_module()); } +TEST_CASE_REGISTER(disk_lookup_records_a_hit); +TEST_CASE_REGISTER(disk_lookup_records_a_hit); -TEST_CASE(disk_lookup_records_a_hit) +// A damaged entry must cost a recompile and nothing more. +template +static void corrupt_entry_is_ignored() { migraphx::tmp_dir td{"binary-cache"}; - disk_lookup_body(dir_path(td)); -} - -TEST_CASE(sqlite_lookup_records_a_hit) -{ - migraphx::tmp_dir td{"binary-cache"}; - disk_lookup_body(db_path(td)); -} - -// A damaged entry must cost a recompile and nothing more. How an entry gets damaged is the only -// part of this that depends on the backend, so it comes in as a step. -static void corrupt_entry_body(const std::string& path, - const std::function& damage) -{ migraphx::gpu::context ctx; + auto path = Backend::path(td); migraphx::gpu::binary_cache_settings settings{path, false}; migraphx::gpu::binary_cache writer{settings}; writer.insert(ctx, make_entry("damaged")); - - damage(path); + EXPECT(Backend::stored(path) == 1); + Backend::damage(path); migraphx::gpu::binary_cache reader{settings}; EXPECT(not reader.get(ctx, "damaged").has_value()); EXPECT(reader.get_stats().misses == 1); } - -TEST_CASE(corrupt_entry_is_ignored) -{ - migraphx::tmp_dir td{"binary-cache"}; - corrupt_entry_body(dir_path(td), [](const std::string& dir) { - auto files = entry_files(dir); - EXPECT(files.size() == 1); - for(const auto& file : files) - migraphx::write_buffer(file, std::vector(8, 0)); - }); -} - -TEST_CASE(sqlite_corrupt_entry_is_ignored) -{ - migraphx::tmp_dir td{"binary-cache"}; - corrupt_entry_body(db_path(td), [](const std::string& db) { - EXPECT(row_count(db, "cache_v1") == 1); - migraphx::sqlite::write(db).execute("UPDATE cache_v1 SET entry = zeroblob(8);"); - }); -} +TEST_CASE_REGISTER(corrupt_entry_is_ignored); +TEST_CASE_REGISTER(corrupt_entry_is_ignored); // Without a directory nothing reaches disk, though results are still shared in memory. TEST_CASE(no_directory_writes_nothing) @@ -342,8 +326,11 @@ TEST_CASE(duplicate_kernels_compile_once_without_a_directory) // Compiling twice against the same cache has to leave entries behind and keep producing the // same numbers as the reference, whichever half of the run they came from. -static void compiling_twice_body(const std::string& path) +template +static void compiling_twice_populates_the_cache_and_matches_reference() { + migraphx::tmp_dir td{"binary-cache"}; + auto path = Backend::path(td); auto options = cache_options(path); auto p_ref = pointwise_program(); @@ -358,7 +345,7 @@ static void compiling_twice_body(const std::string& path) auto warmup = pointwise_program(); warmup.compile(migraphx::make_target("gpu"), options); - EXPECT(stored_entry_count(path) > 0); + EXPECT(Backend::stored(path) > 0); auto t = migraphx::make_target("gpu"); auto p = pointwise_program(); @@ -380,23 +367,16 @@ static void compiling_twice_body(const std::string& path) gpu_result.to_vector())); } -TEST_CASE(compiling_twice_populates_the_cache_and_matches_reference) -{ - migraphx::tmp_dir td{"binary-cache"}; - compiling_twice_body(dir_path(td)); -} - -TEST_CASE(sqlite_compiling_twice_populates_the_cache_and_matches_reference) -{ - migraphx::tmp_dir td{"binary-cache"}; - compiling_twice_body(db_path(td)); -} +TEST_CASE_REGISTER(compiling_twice_populates_the_cache_and_matches_reference); +TEST_CASE_REGISTER(compiling_twice_populates_the_cache_and_matches_reference); // With verification on, every reused result is compiled again and compared, so a run that does // not throw is one where the keys really do capture what the compilers depend on. -static void verified_reuse_body(const std::string& path) +template +static void verified_reuse_matches_fresh_compiles() { - auto options = cache_options(path, /* verify */ true); + migraphx::tmp_dir td{"binary-cache"}; + auto options = cache_options(Backend::path(td), /* verify */ true); auto warmup = pointwise_program(); warmup.compile(migraphx::make_target("gpu"), options); @@ -405,17 +385,8 @@ static void verified_reuse_body(const std::string& path) p.compile(migraphx::make_target("gpu"), options); } -TEST_CASE(verified_reuse_matches_fresh_compiles) -{ - migraphx::tmp_dir td{"binary-cache"}; - verified_reuse_body(dir_path(td)); -} - -TEST_CASE(sqlite_verified_reuse_matches_fresh_compiles) -{ - migraphx::tmp_dir td{"binary-cache"}; - verified_reuse_body(db_path(td)); -} +TEST_CASE_REGISTER(verified_reuse_matches_fresh_compiles); +TEST_CASE_REGISTER(verified_reuse_matches_fresh_compiles); // The extension of the path picks the backend and nothing else does, so the only way to see the // choice from outside is the artifact it leaves: a database file, or a directory tree. @@ -525,13 +496,9 @@ TEST_CASE(backends_store_identical_bytes) migraphx::gpu::binary_cache db_cache{migraphx::gpu::binary_cache_settings{path, false}}; db_cache.insert(ctx, e); - auto select = migraphx::sqlite::read(path).prepare("SELECT entry FROM cache_v1;"); - auto rows = select(); - auto it = rows.begin(); - EXPECT(it != rows.end()); - auto row = *it; - const auto& from_db = row.at("entry").get_binary(); - EXPECT((std::vector(from_db.begin(), from_db.end()) == from_file)); + auto from_db = db_entries(path); + EXPECT(from_db.size() == 1); + EXPECT((from_db.begin()->second == from_file)); } // A whole compile against each backend has to leave the same kernels behind: the same key @@ -599,7 +566,8 @@ TEST_CASE(entries_move_between_backends) auto select = migraphx::sqlite::read(db_path(db_td)) .prepare("SELECT version, device, key_hash, entry FROM cache_v1;"); - auto rows = collect_rows(select()); + auto result = select(); + auto rows = std::vector(result.begin(), result.end()); EXPECT(rows.size() == 1); const auto& row = rows.front(); EXPECT(row.at("version").get_string() == long_version); @@ -621,8 +589,7 @@ TEST_CASE(entries_move_between_backends) } // Two connections over one database, as two processes compiling against a shared cache would -// have. This cannot be two sqlite_binary_cache objects: only open() populates one, and it hands -// back the type-erased wrapper. +// have. TEST_CASE(two_connections_share_a_database) { migraphx::tmp_dir td{"binary-cache"}; @@ -775,20 +742,15 @@ TEST_CASE(file_store_leaves_only_entries_behind) migraphx::gpu::binary_cache second{settings}; second.insert(ctx, make_entry("one")); - std::vector files; - std::vector dirs; - for(const auto& item : migraphx::fs::recursive_directory_iterator(td.path)) - { - if(item.is_directory()) - dirs.push_back(item.path()); - else - files.push_back(item.path()); - } - EXPECT(files.size() == 2); - EXPECT(std::all_of( - files.begin(), files.end(), [](const auto& f) { return f.extension() == ".mxr"; })); - // Just the version directory and the device directory inside it. - EXPECT(dirs.size() == 2); + std::vector items{ + migraphx::fs::recursive_directory_iterator{td.path}, + migraphx::fs::recursive_directory_iterator{}}; + auto dirs = std::count_if( + items.begin(), items.end(), [](const auto& item) { return item.is_directory(); }); + // Just the version directory and the device directory inside it, and the two entries. + EXPECT(dirs == 2); + EXPECT(entry_files(td.path).size() == 2); + EXPECT(items.size() == 4); } // Storage is opened by the first lookup or insert, not by constructing the cache, since every @@ -807,16 +769,19 @@ TEST_CASE(storage_is_opened_on_first_use) // Inserts made inside a batch are all there once it ends. The database commits them together; // the directory backend has no batching and stores each one as it comes. -static void batched_inserts_body(const std::string& path) +template +static void batched_inserts_are_all_stored() { + migraphx::tmp_dir td{"binary-cache"}; migraphx::gpu::context ctx; + auto path = Backend::path(td); migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; { migraphx::gpu::binary_cache::store_batch batch{cache}; cache.insert(ctx, make_entry("first")); cache.insert(ctx, make_entry("second")); } - EXPECT(stored_entry_count(path) == 2); + EXPECT(Backend::stored(path) == 2); migraphx::gpu::binary_cache reader{migraphx::gpu::binary_cache_settings{path, false}}; EXPECT(reader.get(ctx, "first").has_value()); @@ -824,17 +789,8 @@ static void batched_inserts_body(const std::string& path) EXPECT(reader.get_stats().hits == 2); } -TEST_CASE(batched_inserts_are_all_stored) -{ - migraphx::tmp_dir td{"binary-cache"}; - batched_inserts_body(dir_path(td)); -} - -TEST_CASE(sqlite_batched_inserts_are_all_committed) -{ - migraphx::tmp_dir td{"binary-cache"}; - batched_inserts_body(db_path(td)); -} +TEST_CASE_REGISTER(batched_inserts_are_all_stored); +TEST_CASE_REGISTER(batched_inserts_are_all_stored); // A batch holds the database's write lock, so another connection must be able to write again as // soon as it ends. @@ -856,8 +812,7 @@ TEST_CASE(sqlite_batch_releases_the_database) } // The backend layer moves opaque bytes and never decodes them, so a payload that is not even -// msgpack still round-trips. Both backends go through the same type-erased wrapper here, which -// is the runtime half of the static_assert in each backend's .cpp. +// msgpack still round-trips. Both backends go through the same type-erased wrapper here. TEST_CASE(backends_round_trip_through_the_wrapper) { migraphx::tmp_dir td{"binary-cache"}; diff --git a/tools/include/gpu/binary_cache_backend.hpp b/tools/include/gpu/binary_cache_backend.hpp index cb31fb5d5c4..fa54e6b2843 100644 --- a/tools/include/gpu/binary_cache_backend.hpp +++ b/tools/include/gpu/binary_cache_backend.hpp @@ -37,12 +37,11 @@ // Notes: // * binary_cache_entry is defined in ; // the include below pulls in its full definition. -// * Backends typically own non-trivial resources (a cache directory, a SQLite -// connection) and are not meaningfully copyable beyond shared ownership. -// * The members are non-const: binary_cache::get and insert are themselves -// non-const, so nothing forces a const qualifier here, and a backend that -// tracks an open batch needs the mutability. A backend may still declare -// them const. +// * Backends must be copyable: the wrapper shares T and clones it on a +// non-const call while the handle is shared. sqlite_binary_cache shares its +// connection across copies. +// * The members are non-const so a backend can track an open batch; a backend +// may still declare them const. // #ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP #define MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP @@ -70,13 +69,12 @@ namespace gpu { /// Type-erased interface for binary-cache storage backends. /// /// A backend persists serialized binary_cache_entry blobs to some medium (a -/// directory of files, a SQLite database, an in-memory map for tests). Entries -/// are addressed by three strings the caller has already computed: +/// directory of files or a SQLite database). Entries are addressed by three +/// strings the caller has already computed: /// /// * `version` -- binary_cache::version_id(), identifying the toolchain and /// the embedded kernel sources that produced the entry. Never empty; the -/// caller skips persistence entirely when it is. The short form is used -/// for directories and the full form for databases. +/// caller skips persistence entirely when it is. /// * `device` -- the GPU the entry was compiled for. /// * `key_hash` -- md5 of the compile key. A hash rather than the key itself /// because a file backend needs a short name; a collision is harmless, @@ -109,8 +107,9 @@ struct binary_cache_backend /// decided entirely by the key, so a writer that loses a race replaces the /// entry with equivalent bytes. /// - /// Must not throw. A failure to store costs a recompile next run, nothing - /// more, and the caller still keeps the result in memory. + /// May throw: the caller reports a failed store as a warning. It costs a + /// recompile next run, nothing more, and the caller still keeps the result + /// in memory. void store(const std::string& version, const std::string& device, const std::string& key_hash, From e1d5f80c951dee41c18d39f5079479529259c30b Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Fri, 25 Sep 2026 17:12:02 +0200 Subject: [PATCH 36/49] CI tidy fixes --- src/include/migraphx/sqlite.hpp | 28 +++++++++++-------- src/sqlite.cpp | 9 +++++- src/targets/gpu/compile_ops.cpp | 49 ++++++++++++++++++--------------- test/gpu/binary_cache.cpp | 7 +++-- 4 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/include/migraphx/sqlite.hpp b/src/include/migraphx/sqlite.hpp index 1a9dc6a5355..5efad771be0 100644 --- a/src/include/migraphx/sqlite.hpp +++ b/src/include/migraphx/sqlite.hpp @@ -60,19 +60,10 @@ struct MIGRAPHX_EXPORT sqlite_stmt sqlite_stmt() = default; - /// Run the statement with xs bound to its parameters in order, and return its rows. + /// Run the statement with xs bound to its parameters in order, and return its rows. Defined + /// after rows, which has to be complete for a function returning it to be defined. template - rows operator()(const Ts&... xs) const - { - if(not valid()) - MIGRAPHX_THROW("sqlite: calling a statement that was never prepared"); - assert(sizeof...(Ts) == parameter_count()); - // Anything left from the previous call, bindings or an unfinished result, goes first. - reset(); - int i = 0; - each_args([&](const auto& x) { bind(++i, x); }, xs...); - return rows{*this}; - } + rows operator()(const Ts&... xs) const; bool valid() const { return impl != nullptr; } @@ -162,6 +153,19 @@ struct sqlite_stmt::rows bool first = false; }; +template +sqlite_stmt::rows sqlite_stmt::operator()(const Ts&... xs) const +{ + if(not valid()) + MIGRAPHX_THROW("sqlite: calling a statement that was never prepared"); + assert(sizeof...(Ts) == parameter_count()); + // Anything left from the previous call, bindings or an unfinished result, goes first. + reset(); + int i = 0; + each_args([&](const auto& x) { bind(++i, x); }, xs...); + return rows{*this}; +} + struct MIGRAPHX_EXPORT sqlite { sqlite() = default; diff --git a/src/sqlite.cpp b/src/sqlite.cpp index fb15a844481..222462df3c0 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -217,7 +217,9 @@ void sqlite_stmt::reset() const noexcept (void)sqlite3_clear_bindings(impl->get()); } -/// Column i of the current row, keyed by its name. +/// Column i of the current row, keyed by its name. The values are built with parentheses +/// rather than the braces tidy suggests: value has an initializer_list constructor, which braces +/// would select, turning a keyed value into a two-element array. static value column_value(sqlite3_stmt* stmt, int i) { std::string name = sqlite3_column_name(stmt, i); @@ -225,6 +227,7 @@ static value column_value(sqlite3_stmt* stmt, int i) switch(type) { case SQLITE_INTEGER: return value(name, std::int64_t{sqlite3_column_int64(stmt, i)}); + // NOLINTNEXTLINE(modernize-return-braced-init-list) case SQLITE_FLOAT: return value(name, sqlite3_column_double(stmt, i)); case SQLITE_TEXT: case SQLITE_BLOB: { @@ -236,9 +239,12 @@ static value column_value(sqlite3_stmt* stmt, int i) assert(bytes >= 0); auto size = data == nullptr ? 0 : static_cast(bytes); if(type == SQLITE_TEXT) + // NOLINTNEXTLINE(modernize-return-braced-init-list) return value(name, size == 0 ? std::string{} : std::string(data, size)); + // NOLINTNEXTLINE(modernize-return-braced-init-list) return value(name, value::binary{data, size}); } + // NOLINTNEXTLINE(modernize-return-braced-init-list) default: return value(name, nullptr); } } @@ -253,6 +259,7 @@ value sqlite_stmt::to_value() const indices.end(), std::back_inserter(columns), [&](std::ptrdiff_t i) { return column_value(stmt, static_cast(i)); }); + // NOLINTNEXTLINE(modernize-return-braced-init-list) return value(columns, /* array_on_empty */ false); } diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index b2e20598c3e..d621b110391 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -733,6 +733,32 @@ struct compile_manager par_compile(cps.size(), [&](auto i) { cps[i].update_config(exhaustive); }); } + /// Store every compiled result in the binary cache. + static void + store_results(const std::vector>>& tasks) + { + if(tasks.empty()) + return; + // Every plan compiles with the same context, so the stores all go to one cache, and + // batching them lets its storage commit them together rather than one at a time. + auto* ctx = tasks.front().first->ctx; + assert(std::all_of( + tasks.begin(), tasks.end(), [&](const auto& task) { return task.first->ctx == ctx; })); + binary_cache::store_batch batch{ctx->get_binary_cache()}; + for(const auto& [cp, cell] : tasks) + { + if(not cell->result.has_value()) + continue; + // When verifying, reused results are stored again, rewriting the same bytes + // harmlessly. + cp->store(cell->solution, cell->key, cell->result->code); + assert(not cell->result->code.empty()); + // Only the serializable code is used from here on; dropping the replace function + // releases what its closure holds and keeps it off other instructions. + cell->result->replace_fn = nullptr; + } + } + void compile(module& m, bool is_root) { for(auto& cp : cps) @@ -804,28 +830,7 @@ struct compile_manager cell->result = cp->run_compile(cell->solution); }); - if(not tasks.empty()) - { - // Every plan compiles with the same context, so the stores all go to one cache, and - // batching them lets its storage commit them together rather than one at a time. - auto* ctx = tasks.front().first->ctx; - assert(std::all_of(tasks.begin(), tasks.end(), [&](const auto& task) { - return task.first->ctx == ctx; - })); - binary_cache::store_batch batch{ctx->get_binary_cache()}; - for(const auto& [cp, cell] : tasks) - { - if(not cell->result.has_value()) - continue; - // When verifying, reused results are stored again, rewriting the same bytes - // harmlessly. - cp->store(cell->solution, cell->key, cell->result->code); - assert(not cell->result->code.empty()); - // Only the serializable code is used from here on; dropping the replace function - // releases what its closure holds and keeps it off other instructions. - cell->result->replace_fn = nullptr; - } - } + store_results(tasks); static const auto mxr_path = string_value_of(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR{}); bool dump_mxr = not mxr_path.empty(); diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index dcd083eefc8..bd9aa6f71c1 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -135,9 +135,10 @@ static stored_entries dir_entries(const migraphx::fs::path& dir) { stored_entries result; auto files = entry_files(dir); - std::transform(files.begin(), files.end(), std::inserter(result, result.end()), [](auto f) { - return std::make_pair(f.stem().string(), migraphx::read_buffer(f)); - }); + std::transform( + files.begin(), files.end(), std::inserter(result, result.end()), [](const auto& f) { + return std::make_pair(f.stem().string(), migraphx::read_buffer(f)); + }); return result; } From 389de61eeab6f7d67aaf7a99bccff33f93ec0774 Mon Sep 17 00:00:00 2001 From: Paul Date: Sat, 26 Sep 2026 18:34:43 -0500 Subject: [PATCH 37/49] Format --- src/targets/gpu/binary_cache.cpp | 4 ++-- src/targets/gpu/jit/reduce.cpp | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index e5dbabb43ca..44266be5a6e 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -84,8 +84,8 @@ static std::string make_version_id(bool use_short_digest) return {}; // The version numbers make the id readable; the hash of the full version string // separates builds that share them, since it also covers the source revision. - return std::string{binary_cache_format} + "-hip" + compiler.major + "." + compiler.minor + - "." + digest(compiler.version, use_short_digest) + "-kernels" + + return std::string{binary_cache_format} + "-hip" + compiler.major + "." + compiler.minor + "." + + digest(compiler.version, use_short_digest) + "-kernels" + digest(kernels_source(), use_short_digest) + "-rocmlir" + rocmlir_id; } diff --git a/src/targets/gpu/jit/reduce.cpp b/src/targets/gpu/jit/reduce.cpp index 72714fb86ba..b7c0aab6e32 100644 --- a/src/targets/gpu/jit/reduce.cpp +++ b/src/targets/gpu/jit/reduce.cpp @@ -469,14 +469,14 @@ struct simple_reduce_compiler : compiler } options.kernel_name = "reduce_kernel"; std::string identity = "[](auto x) { return x; }"; - auto src = interpolate_string(simple_reduce_kernel, - {{"reduction", v.at("reduction").to()}, - {"init", v.get("init", std::string{"0"})}, - {"read", v.get("read", identity)}, - {"write", v.get("write", identity)}, - {"algo", algo}, - {"transformers", make_transformer_args(vec)}, - {"preamble", v.get("preamble", std::string{})}}); + auto src = interpolate_string(simple_reduce_kernel, + {{"reduction", v.at("reduction").to()}, + {"init", v.get("init", std::string{"0"})}, + {"read", v.get("read", identity)}, + {"write", v.get("write", identity)}, + {"algo", algo}, + {"transformers", make_transformer_args(vec)}, + {"preamble", v.get("preamble", std::string{})}}); options.emplace_param("-Wno-float-equal"); return {src, options}; } From c48eb6e7f63b0a80e8ca4198698113e68ffe06b6 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 28 Sep 2026 14:17:27 -0500 Subject: [PATCH 38/49] Fix tidy warning --- src/targets/gpu/compile_ops.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 10953d793c7..d70ad2cc604 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -414,7 +414,7 @@ static void replace_inserted_device_ops(context& ctx, module& m); /// written by exactly one worker during the parallel compile; sharers read it after the join. struct compile_cell { - explicit compile_cell(value s) : solution(std::move(s)) {} + explicit compile_cell(const value& s) : solution(s) {} value solution = {}; /// Identifies the code the compile would produce, or an invented private key when the @@ -783,11 +783,10 @@ struct compile_manager par_compile(cps.size(), [&](auto i) { cps[i].update_config(exhaustive); }); } - void compile(module& m, bool is_root) + /// Fill every cell's result, from the cache or by compiling, sharing one compile among + /// the cells with the same key. + void compile_cells() { - for(auto& cp : cps) - cp.add_cells(skip_benchmark); - { // Every slot is collected so the keys can be computed in parallel. std::vector> slots; @@ -866,6 +865,13 @@ struct compile_manager // releases what its closure holds and keeps it off other instructions. cell->result->replace_fn = nullptr; } + } + + void compile(module& m, bool is_root) + { + for(auto& cp : cps) + cp.add_cells(skip_benchmark); + compile_cells(); static const auto mxr_path = string_value_of(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR{}); bool dump_mxr = not mxr_path.empty(); From 58f53b0e00bce0792a35de90d4fa098ca5d170f1 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Wed, 30 Sep 2026 16:29:32 +0200 Subject: [PATCH 39/49] Update tools/include/gpu/binary_cache_backend.hpp Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tools/include/gpu/binary_cache_backend.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/include/gpu/binary_cache_backend.hpp b/tools/include/gpu/binary_cache_backend.hpp index fa54e6b2843..3fb2eea0466 100644 --- a/tools/include/gpu/binary_cache_backend.hpp +++ b/tools/include/gpu/binary_cache_backend.hpp @@ -91,9 +91,8 @@ struct binary_cache_backend /// error, it is a cache miss, and the caller recompiles. /// /// Must not throw. - optional> load(const std::string& version, - const std::string& device, - const std::string& key_hash); + optional> + load(const std::string& version, const std::string& device, const std::string& key_hash); /// Persist `blob`, the msgpack encoding of `e`, under this key. /// From bd7f4a68d30c3dcdc6ab892151fe7148a2e6fa4a Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Wed, 30 Sep 2026 16:29:53 +0200 Subject: [PATCH 40/49] Update tools/include/gpu/binary_cache_backend.hpp Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tools/include/gpu/binary_cache_backend.hpp | 31 +++++++++++----------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/tools/include/gpu/binary_cache_backend.hpp b/tools/include/gpu/binary_cache_backend.hpp index 3fb2eea0466..5a2c7008372 100644 --- a/tools/include/gpu/binary_cache_backend.hpp +++ b/tools/include/gpu/binary_cache_backend.hpp @@ -134,22 +134,21 @@ struct binary_cache_backend #else <% - interface( - 'binary_cache_backend', - virtual('load', - returns = 'optional>', - version = 'const std::string&', - device = 'const std::string&', - key_hash = 'const std::string&'), - virtual('store', - returns = 'void', - version = 'const std::string&', - device = 'const std::string&', - key_hash = 'const std::string&', - e = 'const binary_cache_entry&', - blob = 'const std::vector&'), - virtual('begin_batch', returns = 'void', default = 'migraphx::nop'), - virtual('end_batch', returns = 'void', default = 'migraphx::nop')) + interface('binary_cache_backend', + virtual('load', + returns = 'optional>', + version = 'const std::string&', + device = 'const std::string&', + key_hash = 'const std::string&'), + virtual('store', + returns = 'void', + version = 'const std::string&', + device = 'const std::string&', + key_hash = 'const std::string&', + e = 'const binary_cache_entry&', + blob = 'const std::vector&'), + virtual('begin_batch', returns = 'void', default = 'migraphx::nop'), + virtual('end_batch', returns = 'void', default = 'migraphx::nop')) %> #endif From dd2404899318fec29b02e7beefaa90ea32749875 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Thu, 1 Oct 2026 15:30:28 +0200 Subject: [PATCH 41/49] - Backend load/store now take the key and return or accept --- src/include/migraphx/sqlite.hpp | 170 +++++++------ src/include/migraphx/value.hpp | 10 + src/sqlite.cpp | 35 +-- src/targets/gpu/binary_cache.cpp | 75 ++---- src/targets/gpu/compile_ops.cpp | 35 +-- src/targets/gpu/file_binary_cache.cpp | 51 ++-- .../gpu/include/migraphx/gpu/binary_cache.hpp | 26 +- .../migraphx/gpu/binary_cache_backend.hpp | 172 +++---------- .../migraphx/gpu/file_binary_cache.hpp | 14 +- .../migraphx/gpu/sqlite_binary_cache.hpp | 23 +- src/targets/gpu/sqlite_binary_cache.cpp | 114 ++++----- test/gpu/binary_cache.cpp | 229 ++++++++++-------- test/sqlite.cpp | 2 +- tools/include/gpu/binary_cache_backend.hpp | 86 +++---- 14 files changed, 449 insertions(+), 593 deletions(-) diff --git a/src/include/migraphx/sqlite.hpp b/src/include/migraphx/sqlite.hpp index 5efad771be0..8f6fc235436 100644 --- a/src/include/migraphx/sqlite.hpp +++ b/src/include/migraphx/sqlite.hpp @@ -56,14 +56,100 @@ struct sqlite_stmt_impl; /// Not thread safe: use a statement from one thread at a time. struct MIGRAPHX_EXPORT sqlite_stmt { - struct rows; + /// The rows produced by one call of a statement, as an input range of values. + /// + /// The first row is fetched when the call is made, so a statement that returns nothing, such + /// as an insert, has already run by the time the call returns, whether or not the range is + /// iterated. The statement is reset when the range is destroyed: an unfinished select holds + /// a read lock on the database until then, which would stall writers in other processes. + /// + /// Refers to the statement it came from, which must outlive it. + struct rows + { + // Only ever a prvalue returned from a call, so it never needs copying or moving, and a + // copy would reset the statement out from under the original. + rows(const rows&) = delete; + rows(rows&&) = delete; + rows& operator=(const rows&) = delete; + rows& operator=(rows&&) = delete; + ~rows() { stmt->reset(); } + + struct iterator : iterator_operators + { + using value_type = value; + using reference = value_type; + using difference_type = std::ptrdiff_t; + using iterator_category = std::input_iterator_tag; + using pointer = value*; + + iterator() = default; + + iterator(const rows* pparent, bool pavailable) : parent(pparent), available(pavailable) + { + } + + reference operator*() const + { + assert(parent != nullptr and available); + return parent->stmt->to_value(); + } + + static void increment(iterator& x) + { + assert(x.parent != nullptr and x.available); + x.available = x.parent->stmt->step(); + } + + static bool equal(const iterator& x, const iterator& y) + { + return x.parent == y.parent and x.available == y.available; + } + + private: + const rows* parent = nullptr; + bool available = false; + }; + + iterator begin() const { return {this, first}; } + iterator end() const { return {this, false}; } + + private: + friend struct sqlite_stmt; + + explicit rows(const sqlite_stmt& s) : stmt(&s) + { + // The destructor does not run when the constructor throws, so a failed first step + // resets the statement here instead. + try + { + first = stmt->step(); + } + catch(...) + { + stmt->reset(); + throw; + } + } + + const sqlite_stmt* stmt = nullptr; + bool first = false; + }; sqlite_stmt() = default; - /// Run the statement with xs bound to its parameters in order, and return its rows. Defined - /// after rows, which has to be complete for a function returning it to be defined. + /// Run the statement with xs bound to its parameters in order, and return its rows. template - rows operator()(const Ts&... xs) const; + rows operator()(const Ts&... xs) const + { + if(not valid()) + MIGRAPHX_THROW("sqlite: calling a statement that was never prepared"); + assert(sizeof...(Ts) == parameter_count()); + // Anything left from the previous call, bindings or an unfinished result, goes first. + reset(); + int i = 0; + each_args([&](const auto& x) { bind(++i, x); }, xs...); + return rows{*this}; + } bool valid() const { return impl != nullptr; } @@ -90,82 +176,6 @@ struct MIGRAPHX_EXPORT sqlite_stmt std::shared_ptr impl; }; -/// The rows produced by one call of a statement, as an input range of values. -/// -/// The first row is fetched when the call is made, so a statement that returns nothing, such as -/// an insert, has already run by the time the call returns, whether or not the range is -/// iterated. The statement is reset when the range is destroyed: an unfinished select holds a -/// read lock on the database until then, which would stall writers in other processes. -struct sqlite_stmt::rows -{ - // Only ever a prvalue returned from a call, so it never needs copying or moving, and a copy - // would reset the statement out from under the original. - rows(const rows&) = delete; - rows(rows&&) = delete; - rows& operator=(const rows&) = delete; - rows& operator=(rows&&) = delete; - ~rows() { stmt.reset(); } - - struct iterator : iterator_operators - { - using value_type = value; - using reference = value_type; - using difference_type = std::ptrdiff_t; - using iterator_category = std::input_iterator_tag; - using pointer = value*; - - iterator() = default; - - iterator(const rows* pparent, bool pavailable) : parent(pparent), available(pavailable) {} - - reference operator*() const - { - assert(parent != nullptr and available); - return parent->stmt.to_value(); - } - - template - static void increment(U& x) - { - assert(x.parent != nullptr and x.available); - x.available = x.parent->stmt.step(); - } - - template - static auto equal(const U& x, const V& y) - { - return x.parent == y.parent and x.available == y.available; - } - - private: - const rows* parent = nullptr; - bool available = false; - }; - - iterator begin() const { return {this, first}; } - iterator end() const { return {this, false}; } - - private: - friend struct sqlite_stmt; - explicit rows(sqlite_stmt s) : stmt(std::move(s)), first(stmt.step()) {} - - sqlite_stmt stmt; - bool first = false; -}; - -template -sqlite_stmt::rows sqlite_stmt::operator()(const Ts&... xs) const -{ - if(not valid()) - MIGRAPHX_THROW("sqlite: calling a statement that was never prepared"); - assert(sizeof...(Ts) == parameter_count()); - // Anything left from the previous call, bindings or an unfinished result, goes first. - reset(); - int i = 0; - each_args([&](const auto& x) { bind(++i, x); }, xs...); - return rows{*this}; -} - struct MIGRAPHX_EXPORT sqlite { sqlite() = default; diff --git a/src/include/migraphx/value.hpp b/src/include/migraphx/value.hpp index 70ce5613fde..f81e694654d 100644 --- a/src/include/migraphx/value.hpp +++ b/src/include/migraphx/value.hpp @@ -297,6 +297,16 @@ struct MIGRAPHX_EXPORT value value(const std::pair& p) : value(p.first, p.second) { } + + /// A keyed value. Braces would select the initializer_list constructor and make a two-element + /// array instead, so returning this spares callers from suppressing the tidy check that + /// suggests them. + template + static value pair(const std::string& pkey, Ts&&... xs) + { + // NOLINTNEXTLINE(modernize-return-braced-init-list) + return value(pkey, static_cast(xs)...); + } template {})> value& operator=(T rhs) { diff --git a/src/sqlite.cpp b/src/sqlite.cpp index 222462df3c0..105050b0d38 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -217,18 +217,15 @@ void sqlite_stmt::reset() const noexcept (void)sqlite3_clear_bindings(impl->get()); } -/// Column i of the current row, keyed by its name. The values are built with parentheses -/// rather than the braces tidy suggests: value has an initializer_list constructor, which braces -/// would select, turning a keyed value into a two-element array. +/// Column i of the current row, keyed by its name. static value column_value(sqlite3_stmt* stmt, int i) { std::string name = sqlite3_column_name(stmt, i); auto type = sqlite3_column_type(stmt, i); switch(type) { - case SQLITE_INTEGER: return value(name, std::int64_t{sqlite3_column_int64(stmt, i)}); - // NOLINTNEXTLINE(modernize-return-braced-init-list) - case SQLITE_FLOAT: return value(name, sqlite3_column_double(stmt, i)); + case SQLITE_INTEGER: return value::pair(name, std::int64_t{sqlite3_column_int64(stmt, i)}); + case SQLITE_FLOAT: return value::pair(name, sqlite3_column_double(stmt, i)); case SQLITE_TEXT: case SQLITE_BLOB: { // The data must be fetched before sqlite3_column_bytes: the other order can force a @@ -237,30 +234,22 @@ static value column_value(sqlite3_stmt* stmt, int i) const auto* data = static_cast(sqlite3_column_blob(stmt, i)); auto bytes = sqlite3_column_bytes(stmt, i); assert(bytes >= 0); - auto size = data == nullptr ? 0 : static_cast(bytes); + std::size_t size = data == nullptr ? 0 : bytes; if(type == SQLITE_TEXT) - // NOLINTNEXTLINE(modernize-return-braced-init-list) - return value(name, size == 0 ? std::string{} : std::string(data, size)); - // NOLINTNEXTLINE(modernize-return-braced-init-list) - return value(name, value::binary{data, size}); + return value::pair(name, size == 0 ? std::string{} : std::string(data, size)); + return value::pair(name, value::binary{data, size}); } - // NOLINTNEXTLINE(modernize-return-braced-init-list) - default: return value(name, nullptr); + default: return value::pair(name, nullptr); } } value sqlite_stmt::to_value() const { - auto* stmt = impl->get(); - // Built as keyed values rather than a map, so a blob is moved into place instead of copied. - std::vector columns; - auto indices = range(sqlite3_column_count(stmt)); - std::transform(indices.begin(), - indices.end(), - std::back_inserter(columns), - [&](std::ptrdiff_t i) { return column_value(stmt, static_cast(i)); }); - // NOLINTNEXTLINE(modernize-return-braced-init-list) - return value(columns, /* array_on_empty */ false); + auto* stmt = impl->get(); + value columns = value::object{}; + for(auto i : range(sqlite3_column_count(stmt))) + columns.insert(column_value(stmt, static_cast(i))); + return columns; } } // namespace MIGRAPHX_INLINE_NS diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index 8913a9f5992..35a660a2be6 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -28,10 +28,9 @@ #include #include #include -#include -#include #include #include +#include #include namespace migraphx { @@ -105,31 +104,6 @@ static std::string device_dir(const context& ctx) "_wf" + std::to_string(device.get_wavefront_size()); } -/// Turn a stored blob back into an entry. Any failure is a miss, so a damaged entry costs a -/// recompile. -static optional -decode_entry(const std::vector& blob, const std::string& key, const std::string& key_hash) -{ - binary_cache::entry e; - try - { - migraphx::from_value(from_msgpack(blob), e); - } - catch(const std::exception& ex) - { - log::warn() << "Ignoring unreadable binary cache entry " << key_hash << ": " << ex.what(); - return nullopt; - } - // Entries are addressed by a hash of the key, so the full key is checked here to make a - // collision a miss rather than a wrong kernel. - if(e.key != key) - { - log::warn() << "Ignoring binary cache entry with mismatched key: " << key_hash; - return nullopt; - } - return e; -} - binary_cache::binary_cache(binary_cache_settings s) : settings(std::move(s)) {} // The storage backend is selected by file type, the same rule make_problem_cache_backend applies @@ -157,18 +131,6 @@ binary_cache_backend* binary_cache::get_backend() return backend.has_value() ? &*backend : nullptr; } -binary_cache::store_batch::store_batch(binary_cache& c) : backend(c.get_backend()) -{ - if(backend != nullptr) - backend->begin_batch(); -} - -binary_cache::store_batch::~store_batch() -{ - if(backend != nullptr) - backend->end_batch(); -} - optional binary_cache::get(const context& ctx, const std::string& key) { if(key.empty()) @@ -181,44 +143,39 @@ optional binary_cache::get(const context& ctx, const std::string& } if(auto* b = get_backend()) { - // The key is the whole compile source, so it is hashed once for the lookup and any - // diagnostics. - auto key_hash = md5(key); - auto blob = b->load(version, device_dir(ctx), key_hash); - if(blob.has_value()) + auto e = b->load(version, device_dir(ctx), key); + if(e.has_value()) { - auto e = decode_entry(*blob, key, key_hash); - if(e.has_value()) - { - counters.hits++; - return memo.emplace(key, std::move(e->code)).first->second; - } + counters.hits++; + return memo.emplace(key, std::move(e->code)).first->second; } } counters.misses++; return nullopt; } -void binary_cache::insert(const context& ctx, entry e) +void binary_cache::insert(const context& ctx, std::vector es) { - if(e.key.empty()) + es.erase(std::remove_if(es.begin(), es.end(), [](const entry& e) { return e.key.empty(); }), + es.end()); + if(es.empty()) return; - counters.compiled++; + counters.compiled += es.size(); if(auto* b = get_backend()) { - auto key_hash = md5(e.key); try { - // A failure to serialize or store is a warning, not a failed compile. - auto blob = to_msgpack(migraphx::to_value(e)); - b->store(version, device_dir(ctx), key_hash, e, blob); + // A failure to store is a warning, not a failed compile. + b->store(version, device_dir(ctx), es); } catch(const std::exception& ex) { - log::warn() << "Failed to store binary cache entry " << key_hash << ": " << ex.what(); + log::warn() << "Failed to store " << es.size() + << " binary cache entries: " << ex.what(); } } - memo[std::move(e.key)] = std::move(e.code); + for(auto& e : es) + memo[std::move(e.key)] = std::move(e.code); } } // namespace gpu diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index f8977d16b0c..46d1f512bc6 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -217,23 +217,22 @@ static optional cache_lookup(context& ctx, const std::string& return cr; } -/// Record a freshly compiled result, under the same restriction as cache_lookup. -static void cache_store(context& ctx, - const operation& preop, - const value& solution, - const std::string& key, - const value& problem, - const compiled_code& code) +/// What to record for a freshly compiled result, or nullopt when its key must not be cached. +static optional make_cache_entry(const operation& preop, + const value& solution, + const std::string& key, + const value& problem, + const compiled_code& code) { if(is_private_key(key)) - return; + return nullopt; binary_cache::entry e; e.key = key; e.op_name = preop.name(); e.problem = problem; e.solution = solution; e.code = code; - ctx.get_binary_cache().insert(ctx, std::move(e)); + return e; } /// Reuse an earlier result for this key, or compile and record one. For callers with a single @@ -252,7 +251,8 @@ static compiler_replace compile_cached(context& ctx, return *cached; } auto cr = compile_fragment(ctx, ins, preop, solution); - cache_store(ctx, preop, solution, key, problem, cr.code); + if(auto e = make_cache_entry(preop, solution, key, problem, cr.code)) + ctx.get_binary_cache().insert(ctx, {std::move(*e)}); return cr; } @@ -448,9 +448,10 @@ struct compile_plan return cache_lookup(*ctx, key); } - void store(const value& solution, const std::string& key, const compiled_code& code) const + optional + cache_entry(const value& solution, const std::string& key, const compiled_code& code) const { - cache_store(*ctx, preop, solution, key, config ? config->problem : value{}, code); + return make_cache_entry(preop, solution, key, config ? config->problem : value{}, code); } /// True when the cache was configured to check reused results against a fresh compile. @@ -789,24 +790,26 @@ struct compile_manager { if(tasks.empty()) return; - // Every plan compiles with the same context, so the stores all go to one cache, and - // batching them lets its storage commit them together rather than one at a time. + // Every plan compiles with the same context, so the entries all go to one cache in a + // single insert, letting its storage commit them together rather than one at a time. auto* ctx = tasks.front().first->ctx; assert(std::all_of( tasks.begin(), tasks.end(), [&](const auto& task) { return task.first->ctx == ctx; })); - binary_cache::store_batch batch{ctx->get_binary_cache()}; + std::vector entries; for(const auto& [cp, cell] : tasks) { if(not cell->result.has_value()) continue; // When verifying, reused results are stored again, rewriting the same bytes // harmlessly. - cp->store(cell->solution, cell->key, cell->result->code); + if(auto e = cp->cache_entry(cell->solution, cell->key, cell->result->code)) + entries.push_back(std::move(*e)); assert(not cell->result->code.empty()); // Only the serializable code is used from here on; dropping the replace function // releases what its closure holds and keeps it off other instructions. cell->result->replace_fn = nullptr; } + ctx->get_binary_cache().insert(*ctx, std::move(entries)); } /// Fill every cell's result, from the cache or by compiling, sharing one compile among diff --git a/src/targets/gpu/file_binary_cache.cpp b/src/targets/gpu/file_binary_cache.cpp index 2ccd6f0d32b..585f6a107df 100644 --- a/src/targets/gpu/file_binary_cache.cpp +++ b/src/targets/gpu/file_binary_cache.cpp @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include #include #include @@ -38,14 +40,15 @@ namespace gpu { static_assert(std::is_constructible{}, "file_binary_cache must satisfy the binary_cache_backend concept"); -/// Where an entry lives. The caller guarantees a non-empty version, so entries compiled by -/// different toolchains can never land on the same path. +/// Where an entry lives. The key is the whole compile source, so it is hashed to keep the name +/// short. The caller guarantees a non-empty version, so entries compiled by different toolchains +/// can never land on the same path. static fs::path entry_path(const fs::path& root, const std::string& version, const std::string& device, - const std::string& key_hash) + const std::string& key) { - return root / version / device / (key_hash + ".mxr"); + return root / version / device / (md5(key) + ".mxr"); } /// Publish by rename so a reader never sees a half-written file. The temporary stays beside @@ -69,36 +72,46 @@ static void write_atomically(const fs::path& dest, const std::vector& cont } } -optional> file_binary_cache::load(const std::string& version, - const std::string& device, - const std::string& key_hash) const +optional file_binary_cache::load(const std::string& version, + const std::string& device, + const std::string& key) const { - auto path = entry_path(root, version, device, key_hash); + auto path = entry_path(root, version, device, key); + binary_cache_entry e; try { if(not fs::exists(path)) return nullopt; - return read_buffer(path); + migraphx::from_value(from_msgpack(read_buffer(path)), e); } catch(const std::exception& ex) { - // An unreadable entry is a miss, which costs a recompile and nothing else. - log::warn() << "Failed to read binary cache entry " << path << ": " << ex.what(); + // An unreadable or damaged entry is a miss, which costs a recompile and nothing else. + log::warn() << "Ignoring unreadable binary cache entry " << path << ": " << ex.what(); return nullopt; } + // Files are named by a hash of the key, so the full key is checked here to make a collision + // a miss rather than a wrong kernel. + if(e.key != key) + { + log::warn() << "Ignoring binary cache entry with mismatched key: " << path; + return nullopt; + } + return e; } void file_binary_cache::store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry&, - const std::vector& blob) const + const std::vector& entries) const { - auto path = entry_path(root, version, device, key_hash); - // The content is decided entirely by the key, so a writer that loses the publish race - // replaces the file with the same bytes and no locking is needed. - fs::create_directories(path.parent_path()); - write_atomically(path, blob); + for(const auto& e : entries) + { + auto path = entry_path(root, version, device, e.key); + // The content is decided entirely by the key, so a writer that loses the publish race + // replaces the file with the same bytes and no locking is needed. + fs::create_directories(path.parent_path()); + write_atomically(path, to_msgpack(migraphx::to_value(e))); + } } } // namespace gpu diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index f136b1b5dc3..9eac0dd1d01 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -33,6 +33,7 @@ #include #include #include +#include namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { @@ -82,31 +83,16 @@ struct MIGRAPHX_GPU_EXPORT binary_cache std::size_t compiled = 0; }; - /// Groups the inserts made while it lives, so the storage backend can commit them together - /// rather than one at a time. Scope it tightly around a run of inserts: a database holds a - /// write lock against other processes until it ends. - struct MIGRAPHX_GPU_EXPORT store_batch - { - explicit store_batch(binary_cache& c); - store_batch(const store_batch&) = delete; - store_batch(store_batch&&) = delete; - store_batch& operator=(const store_batch&) = delete; - store_batch& operator=(store_batch&&) = delete; - ~store_batch(); - - private: - binary_cache_backend* backend; - }; - - /// Nothing is opened here; storage is set up by the first lookup, insert or store_batch, so - /// a context that never compiles never touches the disk or probes the compiler. + /// Nothing is opened here; storage is set up by the first lookup or insert, so a context that + /// never compiles never touches the disk or probes the compiler. explicit binary_cache(binary_cache_settings s = {}); /// Look up a key, consulting memory first and then the storage backend. optional get(const context& ctx, const std::string& key); - /// Record a compiled result under its key. - void insert(const context& ctx, entry e); + /// Record compiled results under their keys. They are handed to the storage backend + /// together, so it can commit them at once rather than one at a time. + void insert(const context& ctx, std::vector es); /// True when reused results should be checked against a fresh compile. bool verify() const { return settings.verify; } diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp index 2c9d166bde4..0b91a7c14cd 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp @@ -30,8 +30,7 @@ // into the gpu target tree). Do not edit the generated header by hand. // // Any type T satisfies the binary_cache_backend concept if it provides the -// member functions listed below; begin_batch and end_batch are optional and -// default to doing nothing. The wrapper holds T by shared_ptr and forwards +// member functions listed below. The wrapper holds T by shared_ptr and forwards // each call through a virtual dispatch, matching problem_cache_backend. // // Notes: @@ -40,8 +39,6 @@ // * Backends must be copyable: the wrapper shares T and clones it on a // non-const call while the handle is shared. sqlite_binary_cache shares its // connection across copies. -// * The members are non-const so a backend can track an open batch; a backend -// may still declare them const. // #ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP #define MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP @@ -68,67 +65,46 @@ namespace gpu { /// Type-erased interface for binary-cache storage backends. /// -/// A backend persists serialized binary_cache_entry blobs to some medium (a -/// directory of files or a SQLite database). Entries are addressed by three -/// strings the caller has already computed: +/// A backend persists binary_cache_entry values to some medium (a directory of +/// files or a SQLite database), and decides for itself how to serialize them. +/// Entries are addressed by their key, scoped by two strings the caller has +/// already computed: /// /// * `version` -- binary_cache::version_id(), identifying the toolchain and /// the embedded kernel sources that produced the entry. Never empty; the /// caller skips persistence entirely when it is. /// * `device` -- the GPU the entry was compiled for. -/// * `key_hash` -- md5 of the compile key. A hash rather than the key itself -/// because a file backend needs a short name; a collision is harmless, -/// since the caller re-checks the full key against the decoded entry. /// -/// Together these three form the identity of an entry. A backend must keep -/// entries with different scopes distinct rather than overwriting across them. +/// A backend must keep entries with different scopes distinct rather than +/// overwriting across them. It may address entries by a hash of the key, for +/// instance to keep file names short, but must then check the full key when +/// loading so that a collision is a miss rather than a wrong kernel. struct binary_cache_backend { - /// Return the serialized entry for this key, or nullopt for a miss. + /// Return the entry stored for this key, or nullopt for a miss. /// /// nullopt also covers every failure: a missing file, an unreadable - /// database, a permissions problem. A cache that cannot be read is not an - /// error, it is a cache miss, and the caller recompiles. + /// database, a damaged entry, a permissions problem. A cache that cannot be + /// read is not an error, it is a cache miss, and the caller recompiles. /// /// Must not throw. - optional> - load(const std::string& version, const std::string& device, const std::string& key_hash); + optional + load(const std::string& version, const std::string& device, const std::string& key); - /// Persist `blob`, the msgpack encoding of `e`, under this key. - /// - /// `e` is passed alongside `blob` so a backend may denormalize op_name, - /// problem and solution into queryable columns. Those fields are also - /// inside `blob`, which stays the authoritative record -- a backend that - /// stores them separately must still be able to answer a load() with the - /// blob alone. + /// Persist every entry in `entries` under its key. The entries arrive + /// together so a backend can commit them at once, such as in one database + /// transaction, rather than one at a time. /// /// Overwriting an existing entry is expected and safe: the content is /// decided entirely by the key, so a writer that loses a race replaces the - /// entry with equivalent bytes. + /// entry with an equivalent one. /// /// May throw: the caller reports a failed store as a warning. It costs a - /// recompile next run, nothing more, and the caller still keeps the result - /// in memory. + /// recompile next run, nothing more, and the caller still keeps the results + /// in memory. A backend that throws must not leave anything locked. void store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry& e, - const std::vector& blob); - - /// Mark the start of a run of stores that may be committed together, such as - /// a database transaction, rather than one at a time. Every begin_batch is - /// followed by an end_batch, and the two are never nested. Optional: a - /// backend without them stores each entry as it comes. - /// - /// Must not throw. A backend that cannot start a batch stores entries one - /// at a time instead. - void begin_batch(); - - /// Commit the stores made since begin_batch. - /// - /// Must not throw. A failed commit costs those entries a recompile next - /// run, nothing more, and must not leave anything locked. - void end_batch(); + const std::vector& entries); }; #else @@ -139,18 +115,12 @@ struct binary_cache_backend struct MIGRAPHX_EXPORT binary_cache_backend { // - optional> - load(const std::string& version, const std::string& device, const std::string& key_hash); + optional + load(const std::string& version, const std::string& device, const std::string& key); // void store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry& e, - const std::vector& blob); - // (optional) - void begin_batch(); - // (optional) - void end_batch(); + const std::vector& entries); }; #else @@ -158,32 +128,6 @@ struct MIGRAPHX_EXPORT binary_cache_backend struct binary_cache_backend { private: - template - static auto private_detail_te_default_begin_batch(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.begin_batch()) - { - private_detail_te_self.begin_batch(); - } - - template - static void private_detail_te_default_begin_batch(float, T&& private_detail_te_self) - { - migraphx::nop(private_detail_te_self); - } - - template - static auto private_detail_te_default_end_batch(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.end_batch()) - { - private_detail_te_self.end_batch(); - } - - template - static void private_detail_te_default_end_batch(float, T&& private_detail_te_self) - { - migraphx::nop(private_detail_te_self); - } - template struct private_te_unwrap_reference { @@ -206,13 +150,7 @@ struct binary_cache_backend std::declval().store( std::declval(), std::declval(), - std::declval(), - std::declval(), - std::declval&>()), - private_detail_te_default_begin_batch(char(0), - std::declval()), - private_detail_te_default_end_batch(char(0), - std::declval()), + std::declval&>()), void()); template @@ -289,33 +227,19 @@ struct binary_cache_backend return private_detail_te_get_handle().type(); } - optional> - load(const std::string& version, const std::string& device, const std::string& key_hash) + optional + load(const std::string& version, const std::string& device, const std::string& key) { assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().load(version, device, key_hash); + return (*this).private_detail_te_get_handle().load(version, device, key); } void store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry& e, - const std::vector& blob) + const std::vector& entries) { assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().store(version, device, key_hash, e, blob); - } - - void begin_batch() - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().begin_batch(); - } - - void end_batch() - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().end_batch(); + (*this).private_detail_te_get_handle().store(version, device, entries); } friend bool is_shared(const binary_cache_backend& private_detail_x, @@ -332,16 +256,11 @@ struct binary_cache_backend virtual std::shared_ptr clone() const = 0; virtual const std::type_info& type() const = 0; - virtual optional> load(const std::string& version, - const std::string& device, - const std::string& key_hash) = 0; + virtual optional + load(const std::string& version, const std::string& device, const std::string& key) = 0; virtual void store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry& e, - const std::vector& blob) = 0; - virtual void begin_batch() = 0; - virtual void end_batch() = 0; + const std::vector& entries) = 0; }; template @@ -371,34 +290,19 @@ struct binary_cache_backend const std::type_info& type() const override { return typeid(private_detail_te_value); } - optional> load(const std::string& version, - const std::string& device, - const std::string& key_hash) override + optional + load(const std::string& version, const std::string& device, const std::string& key) override { - return private_detail_te_value.load(version, device, key_hash); + return private_detail_te_value.load(version, device, key); } void store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry& e, - const std::vector& blob) override - { - - private_detail_te_value.store(version, device, key_hash, e, blob); - } - - void begin_batch() override - { - - private_detail_te_default_begin_batch(char(0), private_detail_te_value); - } - - void end_batch() override + const std::vector& entries) override { - private_detail_te_default_end_batch(char(0), private_detail_te_value); + private_detail_te_value.store(version, device, entries); } PrivateDetailTypeErasedT private_detail_te_value; diff --git a/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp index ffc37708600..0e39e334fc7 100644 --- a/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/file_binary_cache.hpp @@ -36,18 +36,16 @@ namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { namespace gpu { -// A binary_cache_backend that keeps entries as files under a root directory, laid out -// ///.mxr. The version directory is named after the build that -// wrote it, so the tree is self-describing. +// A binary_cache_backend that keeps each entry as a msgpack file under a root directory, laid +// out ///.mxr. The version directory is named after the build +// that wrote it, so the tree is self-describing. struct MIGRAPHX_GPU_EXPORT file_binary_cache { - optional> - load(const std::string& version, const std::string& device, const std::string& key_hash) const; + optional + load(const std::string& version, const std::string& device, const std::string& key) const; void store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry& e, - const std::vector& blob) const; + const std::vector& entries) const; fs::path root = {}; }; diff --git a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp index 118e15c1e89..7fdec07df2a 100644 --- a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp @@ -37,9 +37,8 @@ inline namespace MIGRAPHX_INLINE_NS { namespace gpu { // A binary_cache_backend that keeps entries as rows in a SQLite database, one row per -// (version, device, key_hash). The stored blob is byte-identical to what the file backend -// writes into a .mxr file, so the two are interchangeable payloads; op_name, problem and -// solution are additionally denormalized into columns so a cache can be inspected with SQL. +// (version, device, md5 of key), with each field of the entry in its own column so a cache can +// be inspected with SQL. The compiled code, a program fragment, is stored as a msgpack blob. // // Holding sqlite and sqlite_stmt by value does not leak the SQLite dependency into this // target: migraphx/sqlite.hpp forward-declares both impl types and never includes sqlite3.h. @@ -53,25 +52,19 @@ struct MIGRAPHX_GPU_EXPORT sqlite_binary_cache /// memory-only rather than raising an error. static optional open(const std::string& path); - optional> - load(const std::string& version, const std::string& device, const std::string& key_hash) const; + optional + load(const std::string& version, const std::string& device, const std::string& key) const; + + /// Store the entries in one transaction, so they cost one commit rather than one each. A + /// failure rolls the whole transaction back and rethrows. void store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry& e, - const std::vector& blob) const; - - /// Open a transaction, so the stores that follow cost one commit rather than one each. - void begin_batch(); - /// Commit the transaction begin_batch opened, or roll it back if the commit fails. - void end_batch(); + const std::vector& entries); private: sqlite db = {}; sqlite_stmt get_stmt = {}; sqlite_stmt store_stmt = {}; - /// Whether begin_batch opened a transaction that end_batch still has to close. - bool in_batch = false; }; } // namespace gpu diff --git a/src/targets/gpu/sqlite_binary_cache.cpp b/src/targets/gpu/sqlite_binary_cache.cpp index 2140d550013..914b9a6bf09 100644 --- a/src/targets/gpu/sqlite_binary_cache.cpp +++ b/src/targets/gpu/sqlite_binary_cache.cpp @@ -27,7 +27,9 @@ #include #include #include -#include +#include +#include +#include #include #include @@ -52,6 +54,10 @@ constexpr int busy_timeout_ms = 5000; // stores the payload inside the index B-tree, which suits short JSON but not a whole serialized // program fragment, which would spill into overflow chains hanging off the index. // +// Rows are addressed by a hash of the key rather than the key itself, which is the whole compile +// source and would otherwise be stored a second time in the index. The full key is kept in its +// own column and checked on load, so a collision is a miss rather than a wrong kernel. +// // The primary key leads with version so that dropping everything belonging to a superseded // toolchain is a range scan rather than a full table scan. Point lookups bind all three and do // not care about the order. @@ -60,27 +66,28 @@ CREATE TABLE IF NOT EXISTS cache_v1 ( version TEXT NOT NULL, device TEXT NOT NULL, key_hash TEXT NOT NULL, + key TEXT NOT NULL, op_name TEXT NOT NULL, problem TEXT NOT NULL, solution TEXT NOT NULL, - entry BLOB NOT NULL, + code BLOB NOT NULL, timestamp INTEGER NOT NULL, PRIMARY KEY (version, device, key_hash) ); )__migraphx__"; -constexpr const char* get_sql = - "SELECT entry FROM cache_v1 WHERE version = ?1 AND device = ?2 AND key_hash = ?3;"; +constexpr const char* get_sql = "SELECT key, op_name, problem, solution, code FROM cache_v1" + " WHERE version = ?1 AND device = ?2 AND key_hash = ?3;"; // INSERT OR REPLACE is the analogue of the file backend's publish-by-rename: the content is // decided entirely by the key, so two processes compiling the same kernel is benign and the -// last writer wins with equivalent bytes. The timestamp is computed by the database rather +// last writer wins with an equivalent row. The timestamp is computed by the database rather // than the process so that rows written by different machines stay comparable. MIGraphX never // reads it; it lets a cache be pruned by age. constexpr const char* store_sql = "INSERT OR REPLACE INTO cache_v1" - " (version, device, key_hash, op_name, problem, solution, entry, timestamp)" - " VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, CAST(STRFTIME('%s','now') AS INTEGER));"; + " (version, device, key_hash, key, op_name, problem, solution, code, timestamp)" + " VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, CAST(STRFTIME('%s','now') AS INTEGER));"; // Stores come in a burst after each round of compiles, and outside a transaction every one of // them is its own commit, each waiting for the disk. IMMEDIATE takes the write lock up front, so @@ -140,86 +147,79 @@ optional sqlite_binary_cache::open(const std::string& path) return r; } -optional> sqlite_binary_cache::load(const std::string& version, - const std::string& device, - const std::string& key_hash) const +optional sqlite_binary_cache::load(const std::string& version, + const std::string& device, + const std::string& key) const { try { // The primary key makes this at most one row. - auto rows = get_stmt(version, device, key_hash); + auto rows = get_stmt(version, device, md5(key)); auto it = rows.begin(); if(it == rows.end()) return nullopt; - auto row = *it; - const auto& entry = row.at("entry").get_binary(); - return std::vector(entry.begin(), entry.end()); + auto row = *it; + binary_cache_entry e; + e.key = row.at("key").get_string(); + // Rows are addressed by a hash of the key, so the full key is checked here to make a + // collision a miss rather than a wrong kernel. + if(e.key != key) + { + log::warn() << "Ignoring binary cache entry with mismatched key: " << md5(key); + return nullopt; + } + e.op_name = row.at("op_name").get_string(); + e.problem = from_json_string(row.at("problem").get_string()); + e.solution = from_json_string(row.at("solution").get_string()); + const auto& code = row.at("code").get_binary(); + migraphx::from_value(from_msgpack(std::vector(code.begin(), code.end())), e.code); + return e; } catch(const std::exception& ex) { - // A cache that cannot be read is a miss, which costs a recompile and nothing else. - log::warn() << "Failed to read binary cache entry " << key_hash << ": " << ex.what(); + // A cache that cannot be read, or a damaged row, is a miss, which costs a recompile and + // nothing else. + log::warn() << "Ignoring unreadable binary cache entry " << md5(key) << ": " << ex.what(); return nullopt; } } void sqlite_binary_cache::store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry& e, - const std::vector& blob) const + const std::vector& entries) { // Not prepared for a read-only database, whose stores are skipped. - if(not store_stmt.valid()) - return; - store_stmt(version, - device, - key_hash, - e.op_name, - to_json_string(e.problem), - to_json_string(e.solution), - blob); -} - -void sqlite_binary_cache::begin_batch() -{ - assert(not in_batch); - if(not store_stmt.valid()) + if(not store_stmt.valid() or entries.empty()) return; + db.execute(begin_sql); try { - db.execute(begin_sql); - in_batch = true; - } - catch(const std::exception& ex) - { - // Without a transaction each store commits on its own, which is slower but still works. - log::warn() << "Binary cache stores will be committed one at a time: " << ex.what(); - } -} - -void sqlite_binary_cache::end_batch() -{ - if(not in_batch) - return; - in_batch = false; - try - { + for(const auto& e : entries) + { + store_stmt(version, + device, + md5(e.key), + e.key, + e.op_name, + to_json_string(e.problem), + to_json_string(e.solution), + to_msgpack(migraphx::to_value(e.code))); + } db.execute(commit_sql); } - catch(const std::exception& ex) + catch(...) { - // A commit that fails leaves the transaction open, holding the write lock against every - // other process, so it is rolled back and the batch's entries are lost instead. - log::warn() << "Failed to commit binary cache entries: " << ex.what(); + // A transaction left open would hold the write lock against every other process, so it + // is rolled back and the entries are lost instead. The caller reports the original error. try { db.execute(rollback_sql); } - catch(const std::exception& rex) + catch(const std::exception& ex) { - log::warn() << "Failed to roll back binary cache entries: " << rex.what(); + log::warn() << "Failed to roll back binary cache entries: " << ex.what(); } + throw; } } diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index bd9aa6f71c1..0aad959d9e7 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -128,34 +128,44 @@ static std::size_t row_count(const std::string& path, const std::string& table) return std::stoul(rows.front().at("n")); } -using stored_entries = std::map>; +/// The full key of every stored entry, keyed by the key hash that addresses it. +using stored_entries = std::map; -/// Every entry a cache directory holds, keyed by key hash, which names each file. +/// Every entry a cache directory holds. Each file is named by its key hash and holds the msgpack +/// of the whole entry. static stored_entries dir_entries(const migraphx::fs::path& dir) { stored_entries result; auto files = entry_files(dir); std::transform( files.begin(), files.end(), std::inserter(result, result.end()), [](const auto& f) { - return std::make_pair(f.stem().string(), migraphx::read_buffer(f)); + auto v = migraphx::from_msgpack(migraphx::read_buffer(f)); + return std::make_pair(f.stem().string(), v.at("key").get_string()); }); return result; } -/// Every entry a cache database holds, keyed by key hash. +/// Every entry a cache database holds. static stored_entries db_entries(const std::string& path) { stored_entries result; - auto select = migraphx::sqlite::read(path).prepare("SELECT key_hash, entry FROM cache_v1;"); + auto select = migraphx::sqlite::read(path).prepare("SELECT key_hash, key FROM cache_v1;"); auto rows = select(); std::transform(rows.begin(), rows.end(), std::inserter(result, result.end()), [](auto row) { - const auto& blob = row.at("entry").get_binary(); - return std::make_pair(row.at("key_hash").get_string(), - std::vector(blob.begin(), blob.end())); + return std::make_pair(row.at("key_hash").get_string(), row.at("key").get_string()); }); return result; } +/// Whether two entries hold the same thing. +static bool same_entry(const migraphx::gpu::binary_cache::entry& x, + const migraphx::gpu::binary_cache::entry& y) +{ + return x.key == y.key and x.op_name == y.op_name and x.problem == y.problem and + x.solution == y.solution and x.code.fill_map == y.code.fill_map and + *x.code.fragment.get_main_module() == *y.code.fragment.get_main_module(); +} + // What a case that must hold for both backends needs to know about each, so the case can be // written once as a template and registered for both. struct directory_backend @@ -179,7 +189,8 @@ struct database_backend /// Overwrite every stored entry with bytes that do not decode. static void damage(const std::string& p) { - migraphx::sqlite::write(p).execute("UPDATE cache_v1 SET entry = zeroblob(8);"); + // 0xc1 is never used in msgpack, so the code cannot be decoded. + migraphx::sqlite::write(p).execute("UPDATE cache_v1 SET code = X'c1c1c1c1';"); } }; @@ -213,7 +224,7 @@ TEST_CASE(memory_lookup_records_reuse) migraphx::gpu::context ctx; migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}}; - cache.insert(ctx, make_entry("a-key")); + cache.insert(ctx, {make_entry("a-key")}); EXPECT(cache.get_stats().compiled == 1); auto found = cache.get(ctx, "a-key"); @@ -235,7 +246,7 @@ static void disk_lookup_records_a_hit() migraphx::gpu::binary_cache_settings settings{Backend::path(td), false}; migraphx::gpu::binary_cache writer{settings}; - writer.insert(ctx, make_entry("shared-key")); + writer.insert(ctx, {make_entry("shared-key")}); migraphx::gpu::binary_cache reader{settings}; @@ -258,7 +269,7 @@ static void corrupt_entry_is_ignored() migraphx::gpu::binary_cache_settings settings{path, false}; migraphx::gpu::binary_cache writer{settings}; - writer.insert(ctx, make_entry("damaged")); + writer.insert(ctx, {make_entry("damaged")}); EXPECT(Backend::stored(path) == 1); Backend::damage(path); @@ -277,7 +288,7 @@ TEST_CASE(no_directory_writes_nothing) migraphx::gpu::context ctx; migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{.path = ""}}; - cache.insert(ctx, make_entry("in-memory-only")); + cache.insert(ctx, {make_entry("in-memory-only")}); EXPECT(cache.get(ctx, "in-memory-only").has_value()); EXPECT(cache.get_stats().reused == 1); EXPECT(migraphx::fs::is_empty(td.path)); @@ -399,7 +410,7 @@ TEST_CASE(extension_selects_the_backend) migraphx::tmp_dir dir_td{"binary-cache"}; migraphx::gpu::binary_cache dir_cache{ migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; - dir_cache.insert(ctx, make_entry("in-a-directory")); + dir_cache.insert(ctx, {make_entry("in-a-directory")}); auto files = entry_files(dir_td.path); EXPECT(files.size() == 1); EXPECT(migraphx::fs::is_directory(dir_td.path / version_dir)); @@ -412,7 +423,7 @@ TEST_CASE(extension_selects_the_backend) migraphx::tmp_dir db_td{"binary-cache"}; auto path = (db_td.path / name).string(); migraphx::gpu::binary_cache db_cache{migraphx::gpu::binary_cache_settings{path, false}}; - db_cache.insert(ctx, make_entry("in-a-database")); + db_cache.insert(ctx, {make_entry("in-a-database")}); EXPECT(migraphx::fs::is_regular_file(path)); EXPECT(row_count(path, "cache_v1") == 1); @@ -433,7 +444,7 @@ TEST_CASE(unusable_database_degrades_to_memory) migraphx::gpu::binary_cache_settings settings{(blocker / "cache.db").string(), false}; migraphx::gpu::binary_cache cache{settings}; - cache.insert(ctx, make_entry("nowhere")); + cache.insert(ctx, {make_entry("nowhere")}); EXPECT(cache.get(ctx, "nowhere").has_value()); EXPECT(cache.get_stats().reused == 1); @@ -454,7 +465,7 @@ TEST_CASE(not_a_database_degrades_to_memory) migraphx::write_buffer(path, garbage); migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; - cache.insert(ctx, make_entry("in-memory")); + cache.insert(ctx, {make_entry("in-memory")}); EXPECT(cache.get(ctx, "in-memory").has_value()); EXPECT(cache.get_stats().reused == 1); EXPECT((migraphx::read_buffer(path) == garbage)); @@ -472,13 +483,13 @@ TEST_CASE(incompatible_schema_degrades_to_memory) EXPECT(not migraphx::gpu::sqlite_binary_cache::open(path).has_value()); migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; - cache.insert(ctx, make_entry("in-memory")); + cache.insert(ctx, {make_entry("in-memory")}); EXPECT(cache.get(ctx, "in-memory").has_value()); EXPECT(row_count(path, "cache_v1") == 0); } -// Both backends store the same serialized entry, so a cache can move between them. -TEST_CASE(backends_store_identical_bytes) +// Both backends address an entry by the same key hash, and give back what they were given. +TEST_CASE(backends_store_the_same_entry) { migraphx::gpu::context ctx; auto e = make_entry("interchange"); @@ -486,24 +497,30 @@ TEST_CASE(backends_store_identical_bytes) migraphx::tmp_dir dir_td{"binary-cache"}; migraphx::gpu::binary_cache dir_cache{ migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; - dir_cache.insert(ctx, e); - auto files = entry_files(dir_td.path); - EXPECT(files.size() == 1); - auto from_file = migraphx::read_buffer(files.front()); - EXPECT(not from_file.empty()); + dir_cache.insert(ctx, {e}); + auto from_dir = dir_entries(dir_td.path); + EXPECT(from_dir.size() == 1); + EXPECT(from_dir.begin()->first == migraphx::md5(e.key)); + EXPECT(from_dir.begin()->second == e.key); migraphx::tmp_dir db_td{"binary-cache"}; auto path = db_path(db_td); migraphx::gpu::binary_cache db_cache{migraphx::gpu::binary_cache_settings{path, false}}; - db_cache.insert(ctx, e); + db_cache.insert(ctx, {e}); + EXPECT((db_entries(path) == from_dir)); - auto from_db = db_entries(path); - EXPECT(from_db.size() == 1); - EXPECT((from_db.begin()->second == from_file)); + migraphx::tmp_dir td{"binary-cache"}; + for(auto& backend : both_backends(td)) + { + backend.store("v", "dev", {e}); + auto got = backend.load("v", "dev", e.key); + EXPECT(got.has_value()); + EXPECT(same_entry(*got, e)); + } } -// A whole compile against each backend has to leave the same kernels behind: the same key -// hashes, each with the same bytes. That makes the choice of backend purely a storage decision. +// A whole compile against each backend has to leave the same kernels behind, under the same key +// hashes. That makes the choice of backend purely a storage decision. TEST_CASE(backends_hold_the_same_entries_after_a_compile) { migraphx::tmp_dir dir_td{"binary-cache"}; @@ -522,9 +539,9 @@ TEST_CASE(backends_hold_the_same_entries_after_a_compile) EXPECT((from_dir == from_db)); } -// An entry copied from one backend into the other is a hit there and decodes to the same code, -// so an existing cache can be converted rather than rebuilt. The only translation is the -// version, which a directory names with the short id and a database records in full. +// An entry loaded from one backend and stored into the other is a hit there and decodes to the +// same code, so an existing cache can be converted rather than rebuilt. The only translation is +// the version, which a directory names with the short id and a database records in full. TEST_CASE(entries_move_between_backends) { migraphx::gpu::context ctx; @@ -539,15 +556,17 @@ TEST_CASE(entries_move_between_backends) migraphx::tmp_dir db_td{"binary-cache"}; migraphx::gpu::binary_cache writer{ migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; - writer.insert(ctx, e); + writer.insert(ctx, {e}); auto files = entry_files(dir_td.path); EXPECT(files.size() == 1); - const auto& file = files.front(); - auto device = file.parent_path().filename().string(); - auto db = migraphx::gpu::sqlite_binary_cache::open(db_path(db_td)); + auto device = files.front().parent_path().filename().string(); + migraphx::gpu::file_binary_cache dir{dir_td.path}; + auto loaded = dir.load(short_version, device, e.key); + EXPECT(loaded.has_value()); + auto db = migraphx::gpu::sqlite_binary_cache::open(db_path(db_td)); EXPECT(db.has_value()); - db->store(long_version, device, file.stem().string(), e, migraphx::read_buffer(file)); + db->store(long_version, device, {*loaded}); migraphx::gpu::binary_cache reader{ migraphx::gpu::binary_cache_settings{db_path(db_td), false}}; @@ -563,22 +582,19 @@ TEST_CASE(entries_move_between_backends) migraphx::tmp_dir dir_td{"binary-cache"}; migraphx::gpu::binary_cache writer{ migraphx::gpu::binary_cache_settings{db_path(db_td), false}}; - writer.insert(ctx, e); + writer.insert(ctx, {e}); - auto select = migraphx::sqlite::read(db_path(db_td)) - .prepare("SELECT version, device, key_hash, entry FROM cache_v1;"); - auto result = select(); - auto rows = std::vector(result.begin(), result.end()); + auto rows = + migraphx::sqlite::read(db_path(db_td)).execute("SELECT version, device FROM cache_v1;"); EXPECT(rows.size() == 1); - const auto& row = rows.front(); - EXPECT(row.at("version").get_string() == long_version); - const auto& blob = row.at("entry").get_binary(); - migraphx::gpu::file_binary_cache files{dir_td.path}; - files.store(short_version, - row.at("device").get_string(), - row.at("key_hash").get_string(), - e, - std::vector(blob.begin(), blob.end())); + EXPECT(rows.front().at("version") == long_version); + const auto& device = rows.front().at("device"); + auto db = migraphx::gpu::sqlite_binary_cache::open(db_path(db_td)); + EXPECT(db.has_value()); + auto loaded = db->load(long_version, device, e.key); + EXPECT(loaded.has_value()); + migraphx::gpu::file_binary_cache dir{dir_td.path}; + dir.store(short_version, device, {*loaded}); migraphx::gpu::binary_cache reader{ migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; @@ -601,18 +617,15 @@ TEST_CASE(two_connections_share_a_database) EXPECT(a.has_value()); EXPECT(b.has_value()); - const std::vector first{'f', 'i', 'r', 's', 't'}; - const std::vector second{'s', 'e', 'c', 'o', 'n', 'd'}; - - a->store("v", "dev", "k1", make_entry("k1"), first); + a->store("v", "dev", {make_entry("k1")}); auto from_b = b->load("v", "dev", "k1"); EXPECT(from_b.has_value()); - EXPECT((*from_b == first)); + EXPECT(from_b->key == "k1"); - b->store("v", "dev", "k2", make_entry("k2"), second); + b->store("v", "dev", {make_entry("k2")}); auto from_a = a->load("v", "dev", "k2"); EXPECT(from_a.has_value()); - EXPECT((*from_a == second)); + EXPECT(from_a->key == "k2"); EXPECT(not a->load("v", "dev", "absent").has_value()); } @@ -627,8 +640,8 @@ TEST_CASE(sqlite_records_the_full_version_id) auto path = db_path(td); migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; - cache.insert(ctx, make_entry("one")); - cache.insert(ctx, make_entry("two")); + cache.insert(ctx, {make_entry("one")}); + cache.insert(ctx, {make_entry("two")}); EXPECT(row_count(path, "cache_v1") == 2); auto rows = migraphx::sqlite::read(path).execute("SELECT DISTINCT version FROM cache_v1;"); @@ -636,8 +649,7 @@ TEST_CASE(sqlite_records_the_full_version_id) EXPECT(rows.front().at("version") == migraphx::gpu::binary_cache::version_id(false)); } -// The op name, problem and solution are copied into columns only so a cache can be inspected -// with SQL; loads never read them, so nothing else would notice them being wrong. +// Each field of an entry has its own column, so a cache can be inspected with SQL. TEST_CASE(sqlite_records_what_each_entry_was_compiled_for) { migraphx::tmp_dir td{"binary-cache"}; @@ -645,13 +657,14 @@ TEST_CASE(sqlite_records_what_each_entry_was_compiled_for) auto path = db_path(td); auto e = make_entry("described"); migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; - cache.insert(ctx, e); + cache.insert(ctx, {e}); auto rows = migraphx::sqlite::read(path).execute( - "SELECT key_hash, op_name, problem, solution FROM cache_v1;"); + "SELECT key_hash, key, op_name, problem, solution FROM cache_v1;"); EXPECT(rows.size() == 1); const auto& row = rows.front(); EXPECT(row.at("key_hash") == migraphx::md5(e.key)); + EXPECT(row.at("key") == e.key); EXPECT(row.at("op_name") == e.op_name); EXPECT(migraphx::from_json_string(row.at("problem")) == e.problem); EXPECT(migraphx::from_json_string(row.at("solution")) == e.solution); @@ -667,7 +680,7 @@ TEST_CASE(sqlite_read_only_database_still_serves_hits) migraphx::gpu::binary_cache_settings settings{path, false}; { migraphx::gpu::binary_cache writer{settings}; - writer.insert(ctx, make_entry("existing")); + writer.insert(ctx, {make_entry("existing")}); } const auto writable = migraphx::fs::perms::owner_write | migraphx::fs::perms::group_write | @@ -681,7 +694,7 @@ TEST_CASE(sqlite_read_only_database_still_serves_hits) EXPECT(reader.get(ctx, "existing").has_value()); EXPECT(reader.get_stats().hits == 1); - reader.insert(ctx, make_entry("new")); + reader.insert(ctx, {make_entry("new")}); EXPECT(reader.get(ctx, "new").has_value()); if(protected_file) { @@ -698,10 +711,9 @@ TEST_CASE(sqlite_read_only_database_still_serves_hits) TEST_CASE(backends_scope_entries_by_version_and_device) { migraphx::tmp_dir td{"binary-cache"}; - const std::vector blob{'p', 'a', 'y'}; for(auto& backend : both_backends(td)) { - backend.store("v1", "dev1", "k", make_entry("k"), blob); + backend.store("v1", "dev1", {make_entry("k")}); EXPECT(backend.load("v1", "dev1", "k").has_value()); EXPECT(not backend.load("v2", "dev1", "k").has_value()); @@ -714,15 +726,17 @@ TEST_CASE(backends_scope_entries_by_version_and_device) TEST_CASE(backends_store_overwrites_in_place) { migraphx::tmp_dir td{"binary-cache"}; - const std::vector replacement{'n', 'e', 'w'}; + auto original = make_entry("k"); + auto replacement = make_entry("k"); + replacement.solution = migraphx::value{{"algo", "replaced"}}; for(auto& backend : both_backends(td)) { - backend.store("v", "dev", "k", make_entry("k"), {'o', 'l', 'd'}); - backend.store("v", "dev", "k", make_entry("k"), replacement); + backend.store("v", "dev", {original}); + backend.store("v", "dev", {replacement}); auto got = backend.load("v", "dev", "k"); EXPECT(got.has_value()); - EXPECT((*got == replacement)); + EXPECT(got->solution == replacement.solution); } EXPECT(row_count(db_path(td), "cache_v1") == 1); EXPECT(entry_files(td.path / "files").size() == 1); @@ -737,11 +751,11 @@ TEST_CASE(file_store_leaves_only_entries_behind) migraphx::gpu::binary_cache_settings settings{dir_path(td), false}; migraphx::gpu::binary_cache first{settings}; - first.insert(ctx, make_entry("one")); - first.insert(ctx, make_entry("two")); + first.insert(ctx, {make_entry("one")}); + first.insert(ctx, {make_entry("two")}); // A second cache stores the same key again, over the file the first one published. migraphx::gpu::binary_cache second{settings}; - second.insert(ctx, make_entry("one")); + second.insert(ctx, {make_entry("one")}); std::vector items{ migraphx::fs::recursive_directory_iterator{td.path}, @@ -768,20 +782,16 @@ TEST_CASE(storage_is_opened_on_first_use) EXPECT(migraphx::fs::exists(path)); } -// Inserts made inside a batch are all there once it ends. The database commits them together; -// the directory backend has no batching and stores each one as it comes. +// Entries inserted together are all there afterwards. The database commits them in one +// transaction; the directory backend writes each file as it comes. template -static void batched_inserts_are_all_stored() +static void inserts_of_many_entries_are_all_stored() { migraphx::tmp_dir td{"binary-cache"}; migraphx::gpu::context ctx; auto path = Backend::path(td); migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; - { - migraphx::gpu::binary_cache::store_batch batch{cache}; - cache.insert(ctx, make_entry("first")); - cache.insert(ctx, make_entry("second")); - } + cache.insert(ctx, {make_entry("first"), make_entry("second")}); EXPECT(Backend::stored(path) == 2); migraphx::gpu::binary_cache reader{migraphx::gpu::binary_cache_settings{path, false}}; @@ -790,43 +800,54 @@ static void batched_inserts_are_all_stored() EXPECT(reader.get_stats().hits == 2); } -TEST_CASE_REGISTER(batched_inserts_are_all_stored); -TEST_CASE_REGISTER(batched_inserts_are_all_stored); +TEST_CASE_REGISTER(inserts_of_many_entries_are_all_stored); +TEST_CASE_REGISTER(inserts_of_many_entries_are_all_stored); -// A batch holds the database's write lock, so another connection must be able to write again as -// soon as it ends. -TEST_CASE(sqlite_batch_releases_the_database) +// A store holds the database's write lock for its transaction, so another connection must be +// able to write again as soon as it returns. +TEST_CASE(sqlite_store_releases_the_database) { migraphx::tmp_dir td{"binary-cache"}; migraphx::gpu::context ctx; auto path = db_path(td); migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; - { - migraphx::gpu::binary_cache::store_batch batch{cache}; - cache.insert(ctx, make_entry("batched")); - } + cache.insert(ctx, {make_entry("first"), make_entry("second")}); auto other = migraphx::gpu::sqlite_binary_cache::open(path); EXPECT(other.has_value()); - other->store("v", "dev", "k", make_entry("k"), {'x'}); - EXPECT(row_count(path, "cache_v1") == 2); + other->store("v", "dev", {make_entry("k")}); + EXPECT(row_count(path, "cache_v1") == 3); } -// The backend layer moves opaque bytes and never decodes them, so a payload that is not even -// msgpack still round-trips. Both backends go through the same type-erased wrapper here. +// Rows are addressed by a hash of the key, so a row whose stored key differs from the one asked +// for, as a hash collision would leave, is a miss rather than a wrong kernel. +TEST_CASE(sqlite_mismatched_key_is_a_miss) +{ + migraphx::tmp_dir td{"binary-cache"}; + auto path = db_path(td); + auto db = migraphx::gpu::sqlite_binary_cache::open(path); + EXPECT(db.has_value()); + db->store("v", "dev", {make_entry("k")}); + EXPECT(db->load("v", "dev", "k").has_value()); + + migraphx::sqlite::write(path).execute("UPDATE cache_v1 SET key = 'another';"); + EXPECT(not db->load("v", "dev", "k").has_value()); +} + +// Both backends go through the same type-erased wrapper and give back the entry they were +// given, field for field. TEST_CASE(backends_round_trip_through_the_wrapper) { migraphx::tmp_dir td{"binary-cache"}; - const std::vector blob{'\0', 'n', 'o', 't', '\0', 'm', 's', 'g', '\xff'}; - auto e = make_entry("opaque"); + auto e = make_entry("round-trip"); for(auto& backend : both_backends(td)) { - EXPECT(not backend.load("v", "dev", "k").has_value()); - backend.store("v", "dev", "k", e, blob); - auto got = backend.load("v", "dev", "k"); + EXPECT(not backend.load("v", "dev", e.key).has_value()); + backend.store("v", "dev", {e}); + auto got = backend.load("v", "dev", e.key); EXPECT(got.has_value()); - EXPECT((*got == blob)); + EXPECT(same_entry(*got, e)); } } diff --git a/test/sqlite.cpp b/test/sqlite.cpp index 8f84e111c6c..02910f109af 100644 --- a/test/sqlite.cpp +++ b/test/sqlite.cpp @@ -33,7 +33,7 @@ /// Every row a call produced, so a test can count and inspect them. static std::vector collect(const migraphx::sqlite_stmt::rows& r) { - return std::vector(r.begin(), r.end()); + return {r.begin(), r.end()}; } TEST_CASE(read_write) diff --git a/tools/include/gpu/binary_cache_backend.hpp b/tools/include/gpu/binary_cache_backend.hpp index 5a2c7008372..c2d3d5fb21b 100644 --- a/tools/include/gpu/binary_cache_backend.hpp +++ b/tools/include/gpu/binary_cache_backend.hpp @@ -30,8 +30,7 @@ // into the gpu target tree). Do not edit the generated header by hand. // // Any type T satisfies the binary_cache_backend concept if it provides the -// member functions listed below; begin_batch and end_batch are optional and -// default to doing nothing. The wrapper holds T by shared_ptr and forwards +// member functions listed below. The wrapper holds T by shared_ptr and forwards // each call through a virtual dispatch, matching problem_cache_backend. // // Notes: @@ -40,8 +39,6 @@ // * Backends must be copyable: the wrapper shares T and clones it on a // non-const call while the handle is shared. sqlite_binary_cache shares its // connection across copies. -// * The members are non-const so a backend can track an open batch; a backend -// may still declare them const. // #ifndef MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP #define MIGRAPHX_GUARD_GPU_BINARY_CACHE_BACKEND_HPP @@ -68,67 +65,46 @@ namespace gpu { /// Type-erased interface for binary-cache storage backends. /// -/// A backend persists serialized binary_cache_entry blobs to some medium (a -/// directory of files or a SQLite database). Entries are addressed by three -/// strings the caller has already computed: +/// A backend persists binary_cache_entry values to some medium (a directory of +/// files or a SQLite database), and decides for itself how to serialize them. +/// Entries are addressed by their key, scoped by two strings the caller has +/// already computed: /// /// * `version` -- binary_cache::version_id(), identifying the toolchain and /// the embedded kernel sources that produced the entry. Never empty; the /// caller skips persistence entirely when it is. /// * `device` -- the GPU the entry was compiled for. -/// * `key_hash` -- md5 of the compile key. A hash rather than the key itself -/// because a file backend needs a short name; a collision is harmless, -/// since the caller re-checks the full key against the decoded entry. /// -/// Together these three form the identity of an entry. A backend must keep -/// entries with different scopes distinct rather than overwriting across them. +/// A backend must keep entries with different scopes distinct rather than +/// overwriting across them. It may address entries by a hash of the key, for +/// instance to keep file names short, but must then check the full key when +/// loading so that a collision is a miss rather than a wrong kernel. struct binary_cache_backend { - /// Return the serialized entry for this key, or nullopt for a miss. + /// Return the entry stored for this key, or nullopt for a miss. /// /// nullopt also covers every failure: a missing file, an unreadable - /// database, a permissions problem. A cache that cannot be read is not an - /// error, it is a cache miss, and the caller recompiles. + /// database, a damaged entry, a permissions problem. A cache that cannot be + /// read is not an error, it is a cache miss, and the caller recompiles. /// /// Must not throw. - optional> - load(const std::string& version, const std::string& device, const std::string& key_hash); + optional + load(const std::string& version, const std::string& device, const std::string& key); - /// Persist `blob`, the msgpack encoding of `e`, under this key. - /// - /// `e` is passed alongside `blob` so a backend may denormalize op_name, - /// problem and solution into queryable columns. Those fields are also - /// inside `blob`, which stays the authoritative record -- a backend that - /// stores them separately must still be able to answer a load() with the - /// blob alone. + /// Persist every entry in `entries` under its key. The entries arrive + /// together so a backend can commit them at once, such as in one database + /// transaction, rather than one at a time. /// /// Overwriting an existing entry is expected and safe: the content is /// decided entirely by the key, so a writer that loses a race replaces the - /// entry with equivalent bytes. + /// entry with an equivalent one. /// /// May throw: the caller reports a failed store as a warning. It costs a - /// recompile next run, nothing more, and the caller still keeps the result - /// in memory. + /// recompile next run, nothing more, and the caller still keeps the results + /// in memory. A backend that throws must not leave anything locked. void store(const std::string& version, const std::string& device, - const std::string& key_hash, - const binary_cache_entry& e, - const std::vector& blob); - - /// Mark the start of a run of stores that may be committed together, such as - /// a database transaction, rather than one at a time. Every begin_batch is - /// followed by an end_batch, and the two are never nested. Optional: a - /// backend without them stores each entry as it comes. - /// - /// Must not throw. A backend that cannot start a batch stores entries one - /// at a time instead. - void begin_batch(); - - /// Commit the stores made since begin_batch. - /// - /// Must not throw. A failed commit costs those entries a recompile next - /// run, nothing more, and must not leave anything locked. - void end_batch(); + const std::vector& entries); }; #else @@ -136,19 +112,15 @@ struct binary_cache_backend <% interface('binary_cache_backend', virtual('load', - returns = 'optional>', - version = 'const std::string&', - device = 'const std::string&', - key_hash = 'const std::string&'), + returns = 'optional', + version = 'const std::string&', + device = 'const std::string&', + key = 'const std::string&'), virtual('store', - returns = 'void', - version = 'const std::string&', - device = 'const std::string&', - key_hash = 'const std::string&', - e = 'const binary_cache_entry&', - blob = 'const std::vector&'), - virtual('begin_batch', returns = 'void', default = 'migraphx::nop'), - virtual('end_batch', returns = 'void', default = 'migraphx::nop')) + returns = 'void', + version = 'const std::string&', + device = 'const std::string&', + entries = 'const std::vector&')) %> #endif From 217796699f1642ccad74858b9b64767192c4e3fe Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Thu, 1 Oct 2026 15:43:20 +0200 Subject: [PATCH 42/49] sqlite: reject empty statements at runtime; a failed binary cache store leaves the database read-only instead of retrying. --- src/sqlite.cpp | 6 ++++-- .../gpu/include/migraphx/gpu/sqlite_binary_cache.hpp | 2 +- src/targets/gpu/sqlite_binary_cache.cpp | 7 ++++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/sqlite.cpp b/src/sqlite.cpp index 105050b0d38..e00d230b8ac 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -152,8 +152,10 @@ sqlite_stmt sqlite::prepare(const std::string& sql) result.impl->ptr = sqlite3_stmt_ptr{stmt_tmp}; if(rc != SQLITE_OK) MIGRAPHX_THROW("error preparing '" + sql + "': " + impl->error_message()); - // sqlite succeeds without a statement for text that holds none, such as only a comment. - assert(stmt_tmp != nullptr); + // sqlite succeeds without a statement for text that holds none, such as only a comment, and + // a statement with no handle would pass null into sqlite on its first call. + if(stmt_tmp == nullptr) + MIGRAPHX_THROW("error preparing '" + sql + "': no statement in the text"); return result; } diff --git a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp index 7fdec07df2a..921c9e87343 100644 --- a/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/sqlite_binary_cache.hpp @@ -56,7 +56,7 @@ struct MIGRAPHX_GPU_EXPORT sqlite_binary_cache load(const std::string& version, const std::string& device, const std::string& key) const; /// Store the entries in one transaction, so they cost one commit rather than one each. A - /// failure rolls the whole transaction back and rethrows. + /// failure rolls the whole transaction back, leaves the cache read-only and rethrows. void store(const std::string& version, const std::string& device, const std::vector& entries); diff --git a/src/targets/gpu/sqlite_binary_cache.cpp b/src/targets/gpu/sqlite_binary_cache.cpp index 914b9a6bf09..c6b1f32ff04 100644 --- a/src/targets/gpu/sqlite_binary_cache.cpp +++ b/src/targets/gpu/sqlite_binary_cache.cpp @@ -191,9 +191,9 @@ void sqlite_binary_cache::store(const std::string& version, // Not prepared for a read-only database, whose stores are skipped. if(not store_stmt.valid() or entries.empty()) return; - db.execute(begin_sql); try { + db.execute(begin_sql); for(const auto& e : entries) { store_stmt(version, @@ -219,6 +219,11 @@ void sqlite_binary_cache::store(const std::string& version, { log::warn() << "Failed to roll back binary cache entries: " << ex.what(); } + // A database that refused one write, often after waiting out the busy timeout, would + // most likely refuse the next one as well, so later stores are skipped and the cache + // carries on read-only. Lookups keep working. + store_stmt = {}; + log::warn() << "Binary cache is read-only from now on"; throw; } } From 342d806a9d85f528debc7eac6fb42d91f1bf2982 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Thu, 1 Oct 2026 16:00:15 +0200 Subject: [PATCH 43/49] Simplify tool --- src/include/migraphx/sqlite.hpp | 4 +-- src/sqlite.cpp | 25 +++++++++---- src/targets/gpu/compile_ops.cpp | 12 +++++-- src/targets/gpu/file_binary_cache.cpp | 35 ++++++++++--------- .../migraphx/gpu/binary_cache_backend.hpp | 5 +-- src/targets/gpu/sqlite_binary_cache.cpp | 23 ++++++------ test/gpu/binary_cache.cpp | 31 +++++----------- tools/include/gpu/binary_cache_backend.hpp | 5 +-- 8 files changed, 76 insertions(+), 64 deletions(-) diff --git a/src/include/migraphx/sqlite.hpp b/src/include/migraphx/sqlite.hpp index 8f6fc235436..b5792a9afef 100644 --- a/src/include/migraphx/sqlite.hpp +++ b/src/include/migraphx/sqlite.hpp @@ -46,8 +46,8 @@ inline namespace MIGRAPHX_INLINE_NS { struct sqlite_impl; struct sqlite_stmt_impl; -/// A prepared statement, holding a reference to the connection it was prepared on so it can -/// never outlive it. Copies share the same statement. +/// A prepared statement. It shares ownership of the connection it was prepared on, which stays +/// open for as long as the statement exists. Copies share the same statement. /// /// Calling it with arguments runs it: the arguments are bound to the parameters in order and the /// result comes back as a range of rows. Since copies share one statement, only the rows of one diff --git a/src/sqlite.cpp b/src/sqlite.cpp index e00d230b8ac..f5cf58219ca 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -144,6 +143,7 @@ std::vector> sqlite::execute(const sqlite_stmt sqlite::prepare(const std::string& sql) { + assert(impl != nullptr); sqlite3_stmt* stmt_tmp = nullptr; int rc = sqlite3_prepare_v2(impl->get(), sql.c_str(), -1, &stmt_tmp, nullptr); sqlite_stmt result; @@ -159,9 +159,17 @@ sqlite_stmt sqlite::prepare(const std::string& sql) return result; } -void sqlite::set_busy_timeout(int ms) { sqlite3_busy_timeout(impl->get(), ms); } +void sqlite::set_busy_timeout(int ms) +{ + assert(impl != nullptr); + sqlite3_busy_timeout(impl->get(), ms); +} -bool sqlite::read_only() const { return sqlite3_db_readonly(impl->get(), "main") == 1; } +bool sqlite::read_only() const +{ + assert(impl != nullptr); + return sqlite3_db_readonly(impl->get(), "main") == 1; +} void sqlite_stmt::bind(int i, std::string_view s) const { @@ -222,8 +230,10 @@ void sqlite_stmt::reset() const noexcept /// Column i of the current row, keyed by its name. static value column_value(sqlite3_stmt* stmt, int i) { - std::string name = sqlite3_column_name(stmt, i); - auto type = sqlite3_column_type(stmt, i); + // Null only when sqlite runs out of memory, which would make the string below undefined. + const char* name = sqlite3_column_name(stmt, i); + assert(name != nullptr); + auto type = sqlite3_column_type(stmt, i); switch(type) { case SQLITE_INTEGER: return value::pair(name, std::int64_t{sqlite3_column_int64(stmt, i)}); @@ -249,8 +259,9 @@ value sqlite_stmt::to_value() const { auto* stmt = impl->get(); value columns = value::object{}; - for(auto i : range(sqlite3_column_count(stmt))) - columns.insert(column_value(stmt, static_cast(i))); + const int n = sqlite3_column_count(stmt); + for(int i = 0; i < n; ++i) + columns.insert(column_value(stmt, i)); return columns; } diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 46d1f512bc6..7a2cb2acfac 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -252,7 +252,12 @@ static compiler_replace compile_cached(context& ctx, } auto cr = compile_fragment(ctx, ins, preop, solution); if(auto e = make_cache_entry(preop, solution, key, problem, cr.code)) - ctx.get_binary_cache().insert(ctx, {std::move(*e)}); + { + // Built by push_back, since an initializer list would copy the entry rather than move it. + std::vector es; + es.push_back(std::move(*e)); + ctx.get_binary_cache().insert(ctx, std::move(es)); + } return cr; } @@ -796,12 +801,13 @@ struct compile_manager assert(std::all_of( tasks.begin(), tasks.end(), [&](const auto& task) { return task.first->ctx == ctx; })); std::vector entries; + entries.reserve(tasks.size()); for(const auto& [cp, cell] : tasks) { if(not cell->result.has_value()) continue; - // When verifying, reused results are stored again, rewriting the same bytes - // harmlessly. + // When verifying, reused results are stored again, harmlessly replacing each entry + // with an equivalent one. if(auto e = cp->cache_entry(cell->solution, cell->key, cell->result->code)) entries.push_back(std::move(*e)); assert(not cell->result->code.empty()); diff --git a/src/targets/gpu/file_binary_cache.cpp b/src/targets/gpu/file_binary_cache.cpp index 585f6a107df..63731546b5b 100644 --- a/src/targets/gpu/file_binary_cache.cpp +++ b/src/targets/gpu/file_binary_cache.cpp @@ -40,15 +40,19 @@ namespace gpu { static_assert(std::is_constructible{}, "file_binary_cache must satisfy the binary_cache_backend concept"); -/// Where an entry lives. The key is the whole compile source, so it is hashed to keep the name -/// short. The caller guarantees a non-empty version, so entries compiled by different toolchains -/// can never land on the same path. -static fs::path entry_path(const fs::path& root, - const std::string& version, - const std::string& device, - const std::string& key) +/// The directory holding one device's entries for one toolchain. The caller guarantees a +/// non-empty version, so entries compiled by different toolchains can never land in the same one. +static fs::path +entry_dir(const fs::path& root, const std::string& version, const std::string& device) { - return root / version / device / (md5(key) + ".mxr"); + return root / version / device; +} + +/// Where an entry lives in its directory. The key is the whole compile source, so it is hashed to +/// keep the name short. +static fs::path entry_path(const fs::path& dir, const std::string& key) +{ + return dir / (md5(key) + ".mxr"); } /// Publish by rename so a reader never sees a half-written file. The temporary stays beside @@ -76,7 +80,7 @@ optional file_binary_cache::load(const std::string& version, const std::string& device, const std::string& key) const { - auto path = entry_path(root, version, device, key); + auto path = entry_path(entry_dir(root, version, device), key); binary_cache_entry e; try { @@ -104,14 +108,13 @@ void file_binary_cache::store(const std::string& version, const std::string& device, const std::vector& entries) const { + // Every entry shares one directory, so it is created once for the whole batch. + auto dir = entry_dir(root, version, device); + fs::create_directories(dir); + // The content is decided entirely by the key, so a writer that loses the publish race + // replaces the file with the same bytes and no locking is needed. for(const auto& e : entries) - { - auto path = entry_path(root, version, device, e.key); - // The content is decided entirely by the key, so a writer that loses the publish race - // replaces the file with the same bytes and no locking is needed. - fs::create_directories(path.parent_path()); - write_atomically(path, to_msgpack(migraphx::to_value(e))); - } + write_atomically(entry_path(dir, e.key), to_msgpack(migraphx::to_value(e))); } } // namespace gpu diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp index 0b91a7c14cd..0ded21c4c61 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache_backend.hpp @@ -70,8 +70,9 @@ namespace gpu { /// Entries are addressed by their key, scoped by two strings the caller has /// already computed: /// -/// * `version` -- binary_cache::version_id(), identifying the toolchain and -/// the embedded kernel sources that produced the entry. Never empty; the +/// * `version` -- binary_cache::version_id(), short for the directory +/// backend and full for the database, identifying the toolchain and the +/// embedded kernel sources that produced the entry. Never empty; the /// caller skips persistence entirely when it is. /// * `device` -- the GPU the entry was compiled for. /// diff --git a/src/targets/gpu/sqlite_binary_cache.cpp b/src/targets/gpu/sqlite_binary_cache.cpp index c6b1f32ff04..56d8614fa1a 100644 --- a/src/targets/gpu/sqlite_binary_cache.cpp +++ b/src/targets/gpu/sqlite_binary_cache.cpp @@ -48,7 +48,7 @@ constexpr int busy_timeout_ms = 5000; // The table name carries the schema version, so an incompatible change is a new table that old // binaries ignore rather than a migration. This is orthogonal to binary_cache_format, which -// versions the entry payload and reaches the row through the version column. +// versions how entries are serialized and reaches each row through the version column. // // Deliberately not WITHOUT ROWID, unlike the sibling table in sqlite_problem_cache: that clause // stores the payload inside the index B-tree, which suits short JSON but not a whole serialized @@ -151,35 +151,37 @@ optional sqlite_binary_cache::load(const std::string& versio const std::string& device, const std::string& key) const { + auto key_hash = md5(key); try { // The primary key makes this at most one row. - auto rows = get_stmt(version, device, md5(key)); + auto rows = get_stmt(version, device, key_hash); auto it = rows.begin(); if(it == rows.end()) return nullopt; auto row = *it; - binary_cache_entry e; - e.key = row.at("key").get_string(); // Rows are addressed by a hash of the key, so the full key is checked here to make a // collision a miss rather than a wrong kernel. - if(e.key != key) + if(row.at("key").get_string() != key) { - log::warn() << "Ignoring binary cache entry with mismatched key: " << md5(key); + log::warn() << "Ignoring binary cache entry with mismatched key: " << key_hash; return nullopt; } + binary_cache_entry e; + e.key = key; e.op_name = row.at("op_name").get_string(); e.problem = from_json_string(row.at("problem").get_string()); e.solution = from_json_string(row.at("solution").get_string()); const auto& code = row.at("code").get_binary(); - migraphx::from_value(from_msgpack(std::vector(code.begin(), code.end())), e.code); + migraphx::from_value(from_msgpack(reinterpret_cast(code.data()), code.size()), + e.code); return e; } catch(const std::exception& ex) { // A cache that cannot be read, or a damaged row, is a miss, which costs a recompile and // nothing else. - log::warn() << "Ignoring unreadable binary cache entry " << md5(key) << ": " << ex.what(); + log::warn() << "Ignoring unreadable binary cache entry " << key_hash << ": " << ex.what(); return nullopt; } } @@ -188,8 +190,9 @@ void sqlite_binary_cache::store(const std::string& version, const std::string& device, const std::vector& entries) { - // Not prepared for a read-only database, whose stores are skipped. - if(not store_stmt.valid() or entries.empty()) + // Never prepared for a read-only database, and cleared after a failed store; either way + // stores are skipped. + if(not store_stmt.valid()) return; try { diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 0aad959d9e7..a380bf26647 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -186,7 +186,7 @@ struct database_backend { static std::string path(const migraphx::tmp_dir& td) { return db_path(td); } static std::size_t stored(const std::string& p) { return row_count(p, "cache_v1"); } - /// Overwrite every stored entry with bytes that do not decode. + /// Overwrite every stored entry's code with bytes that do not decode. static void damage(const std::string& p) { // 0xc1 is never used in msgpack, so the code cannot be decoded. @@ -195,7 +195,8 @@ struct database_backend }; /// One of each backend over fresh storage in td: a directory at td/files and a database at -/// db_path(td). Driven directly, these skip binary_cache and its version and device strings. +/// db_path(td). Driven directly, bypassing binary_cache, so callers pass their own version and +/// device strings. static std::vector both_backends(const migraphx::tmp_dir& td) { std::vector result; @@ -488,7 +489,7 @@ TEST_CASE(incompatible_schema_degrades_to_memory) EXPECT(row_count(path, "cache_v1") == 0); } -// Both backends address an entry by the same key hash, and give back what they were given. +// Both backends address an entry by the same key hash. TEST_CASE(backends_store_the_same_entry) { migraphx::gpu::context ctx; @@ -508,18 +509,9 @@ TEST_CASE(backends_store_the_same_entry) migraphx::gpu::binary_cache db_cache{migraphx::gpu::binary_cache_settings{path, false}}; db_cache.insert(ctx, {e}); EXPECT((db_entries(path) == from_dir)); - - migraphx::tmp_dir td{"binary-cache"}; - for(auto& backend : both_backends(td)) - { - backend.store("v", "dev", {e}); - auto got = backend.load("v", "dev", e.key); - EXPECT(got.has_value()); - EXPECT(same_entry(*got, e)); - } } -// A whole compile against each backend has to leave the same kernels behind, under the same key +// A whole compile against each backend has to leave the same keys behind, under the same key // hashes. That makes the choice of backend purely a storage decision. TEST_CASE(backends_hold_the_same_entries_after_a_compile) { @@ -535,7 +527,6 @@ TEST_CASE(backends_hold_the_same_entries_after_a_compile) auto from_dir = dir_entries(dir_td.path); auto from_db = db_entries(path); EXPECT(not from_dir.empty()); - EXPECT(from_dir.size() == from_db.size()); EXPECT((from_dir == from_db)); } @@ -630,9 +621,9 @@ TEST_CASE(two_connections_share_a_database) EXPECT(not a->load("v", "dev", "absent").has_value()); } -// A table of hashes says nothing about which build wrote it, so each row records the full -// version id. A database has no path length to protect, unlike the directory backend, which -// names its directories with the short one. +// A database has no directory to name the build that wrote a row, so each row records the +// version id, and in full, since unlike the directory backend's directory names it has no path +// length to protect. TEST_CASE(sqlite_records_the_full_version_id) { migraphx::tmp_dir td{"binary-cache"}; @@ -859,11 +850,7 @@ TEST_CASE(entry_round_trip) migraphx::gpu::binary_cache::entry loaded; migraphx::from_value(migraphx::from_msgpack(buffer), loaded); - EXPECT(loaded.key == e.key); - EXPECT(loaded.op_name == e.op_name); - EXPECT(loaded.solution == e.solution); - EXPECT(loaded.code.fill_map == e.code.fill_map); - EXPECT(*loaded.code.fragment.get_main_module() == *e.code.fragment.get_main_module()); + EXPECT(same_entry(loaded, e)); } // The key has to cover everything handed to the compiler, not just the source text. Two diff --git a/tools/include/gpu/binary_cache_backend.hpp b/tools/include/gpu/binary_cache_backend.hpp index c2d3d5fb21b..c5d0f93bf20 100644 --- a/tools/include/gpu/binary_cache_backend.hpp +++ b/tools/include/gpu/binary_cache_backend.hpp @@ -70,8 +70,9 @@ namespace gpu { /// Entries are addressed by their key, scoped by two strings the caller has /// already computed: /// -/// * `version` -- binary_cache::version_id(), identifying the toolchain and -/// the embedded kernel sources that produced the entry. Never empty; the +/// * `version` -- binary_cache::version_id(), short for the directory +/// backend and full for the database, identifying the toolchain and the +/// embedded kernel sources that produced the entry. Never empty; the /// caller skips persistence entirely when it is. /// * `device` -- the GPU the entry was compiled for. /// From a641728bc8b279a4adddf4509445e20e36c8a140 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Thu, 1 Oct 2026 16:04:05 +0200 Subject: [PATCH 44/49] Misspell resolve --- src/targets/gpu/sqlite_binary_cache.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/targets/gpu/sqlite_binary_cache.cpp b/src/targets/gpu/sqlite_binary_cache.cpp index 56d8614fa1a..7bf302713d1 100644 --- a/src/targets/gpu/sqlite_binary_cache.cpp +++ b/src/targets/gpu/sqlite_binary_cache.cpp @@ -79,7 +79,7 @@ CREATE TABLE IF NOT EXISTS cache_v1 ( constexpr const char* get_sql = "SELECT key, op_name, problem, solution, code FROM cache_v1" " WHERE version = ?1 AND device = ?2 AND key_hash = ?3;"; -// INSERT OR REPLACE is the analogue of the file backend's publish-by-rename: the content is +// INSERT OR REPLACE is the analog of the file backend's publish-by-rename: the content is // decided entirely by the key, so two processes compiling the same kernel is benign and the // last writer wins with an equivalent row. The timestamp is computed by the database rather // than the process so that rows written by different machines stay comparable. MIGraphX never From cead10ec567f3ea0d8da4064c1488792bcd238d9 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Wed, 7 Oct 2026 16:44:13 +0200 Subject: [PATCH 45/49] Fix test_gpu_binary_cache build with std::experimental::filesystem (SLES) --- test/gpu/binary_cache.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index a380bf26647..994b860d8fa 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -676,7 +676,7 @@ TEST_CASE(sqlite_read_only_database_still_serves_hits) const auto writable = migraphx::fs::perms::owner_write | migraphx::fs::perms::group_write | migraphx::fs::perms::others_write; - migraphx::fs::permissions(path, writable, migraphx::fs::perm_options::remove); + migraphx::fs::permissions(path, migraphx::fs::status(path).permissions() & ~writable); // Permissions do not stop root, so what can be checked about stores depends on whether the // write protection actually took. const bool protected_file = migraphx::sqlite::write(path).read_only(); @@ -694,7 +694,7 @@ TEST_CASE(sqlite_read_only_database_still_serves_hits) // Restored so the temporary directory can be removed, which Windows refuses otherwise. migraphx::fs::permissions( - path, migraphx::fs::perms::owner_write, migraphx::fs::perm_options::add); + path, migraphx::fs::status(path).permissions() | migraphx::fs::perms::owner_write); } // version and device separate entries this build may use from entries it may not, so an entry @@ -751,8 +751,9 @@ TEST_CASE(file_store_leaves_only_entries_behind) std::vector items{ migraphx::fs::recursive_directory_iterator{td.path}, migraphx::fs::recursive_directory_iterator{}}; - auto dirs = std::count_if( - items.begin(), items.end(), [](const auto& item) { return item.is_directory(); }); + auto dirs = std::count_if(items.begin(), items.end(), [](const auto& item) { + return migraphx::fs::is_directory(item.status()); + }); // Just the version directory and the device directory inside it, and the two entries. EXPECT(dirs == 2); EXPECT(entry_files(td.path).size() == 2); From ae721a9b270139fb639845190c9b75129d348880 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Wed, 7 Oct 2026 17:17:49 +0200 Subject: [PATCH 46/49] Simplify --- src/include/migraphx/sqlite.hpp | 9 +++++++- src/include/migraphx/value.hpp | 6 ++--- src/sqlite.cpp | 8 ++++++- src/targets/gpu/binary_cache.cpp | 15 ++++++------- src/targets/gpu/compile_ops.cpp | 4 ++-- src/targets/gpu/file_binary_cache.cpp | 1 - .../gpu/include/migraphx/gpu/binary_cache.hpp | 2 +- src/targets/gpu/problem_cache.cpp | 3 ++- test/gpu/binary_cache.cpp | 22 +++++++++---------- 9 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/include/migraphx/sqlite.hpp b/src/include/migraphx/sqlite.hpp index b5792a9afef..47e1be5563f 100644 --- a/src/include/migraphx/sqlite.hpp +++ b/src/include/migraphx/sqlite.hpp @@ -139,7 +139,7 @@ struct MIGRAPHX_EXPORT sqlite_stmt /// Run the statement with xs bound to its parameters in order, and return its rows. template - rows operator()(const Ts&... xs) const + rows operator()(const Ts&... xs) const& { if(not valid()) MIGRAPHX_THROW("sqlite: calling a statement that was never prepared"); @@ -151,6 +151,10 @@ struct MIGRAPHX_EXPORT sqlite_stmt return rows{*this}; } + /// The rows refer back to the statement, so a temporary one would leave them dangling. + template + rows operator()(const Ts&...) const&& = delete; + bool valid() const { return impl != nullptr; } private: @@ -186,6 +190,9 @@ struct MIGRAPHX_EXPORT sqlite /// that treat an unusable database as "no cache" rather than as an error. static optional try_write(const fs::path& p); + /// True when a cache path names a SQLite database, by its ".db" or ".sqlite" extension. + static bool is_database_path(const std::string& path); + std::vector> execute(const std::string& s); sqlite_stmt prepare(const std::string& sql); diff --git a/src/include/migraphx/value.hpp b/src/include/migraphx/value.hpp index f81e694654d..572f4742931 100644 --- a/src/include/migraphx/value.hpp +++ b/src/include/migraphx/value.hpp @@ -298,9 +298,9 @@ struct MIGRAPHX_EXPORT value { } - /// A keyed value. Braces would select the initializer_list constructor and make a two-element - /// array instead, so returning this spares callers from suppressing the tidy check that - /// suggests them. + /// A keyed value. Tidy suggests returning {key, x} instead of value(key, x), but braces pick + /// the initializer_list constructor and build a two-element array; this keeps that + /// suppression in one place. template static value pair(const std::string& pkey, Ts&&... xs) { diff --git a/src/sqlite.cpp b/src/sqlite.cpp index f5cf58219ca..600001248fa 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -124,6 +125,11 @@ optional sqlite::try_write(const fs::path& p) return r; } +bool sqlite::is_database_path(const std::string& path) +{ + return ends_with(path, ".db") or ends_with(path, ".sqlite"); +} + std::vector> sqlite::execute(const std::string& s) { std::vector> result; @@ -221,7 +227,7 @@ bool sqlite_stmt::step() const void sqlite_stmt::reset() const noexcept { assert(impl != nullptr); - // The return of sqlite3_reset is the error from the preceding step(), which the caller + // sqlite3_reset only repeats the error, if any, from the preceding step(), which the caller // has already seen as a throw. There is nothing new to report, and this must not throw. (void)sqlite3_reset(impl->get()); (void)sqlite3_clear_bindings(impl->get()); diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index 35a660a2be6..b485d92a514 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -104,13 +104,12 @@ static std::string device_dir(const context& ctx) "_wf" + std::to_string(device.get_wavefront_size()); } -binary_cache::binary_cache(binary_cache_settings s) : settings(std::move(s)) {} - -// The storage backend is selected by file type, the same rule make_problem_cache_backend applies -// in problem_cache.cpp: a ".db"/".sqlite" path is a SQLite database, anything else is a -// directory of entries. A directory is named with the short version id to keep paths short; a -// database records the full id, which is self-describing. Nothing is persisted when the compiler -// cannot be identified, since entries from different toolchains would be indistinguishable. +// The storage backend is selected by the path's extension, the same rule +// make_problem_cache_backend applies: a SQLite database for sqlite::is_database_path, anything +// else is a directory of entries. A directory is named with the short version id to keep paths +// short; a database records the full id, which is self-describing. Nothing is persisted when the +// compiler cannot be identified, since entries from different toolchains would be +// indistinguishable. binary_cache_backend* binary_cache::get_backend() { if(backend_opened) @@ -120,7 +119,7 @@ binary_cache_backend* binary_cache::get_backend() // Checked first so that a memory-only cache never compiles the version probe. if(path.empty()) return nullptr; - const bool database = ends_with(path, ".db") or ends_with(path, ".sqlite"); + const bool database = sqlite::is_database_path(path); version = version_id(not database); if(version.empty()) return nullptr; diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 7a2cb2acfac..23a87eb9c2e 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -798,8 +799,7 @@ struct compile_manager // Every plan compiles with the same context, so the entries all go to one cache in a // single insert, letting its storage commit them together rather than one at a time. auto* ctx = tasks.front().first->ctx; - assert(std::all_of( - tasks.begin(), tasks.end(), [&](const auto& task) { return task.first->ctx == ctx; })); + assert(all_of(tasks, [&](const auto& task) { return task.first->ctx == ctx; })); std::vector entries; entries.reserve(tasks.size()); for(const auto& [cp, cell] : tasks) diff --git a/src/targets/gpu/file_binary_cache.cpp b/src/targets/gpu/file_binary_cache.cpp index 63731546b5b..4ea8bdba6f7 100644 --- a/src/targets/gpu/file_binary_cache.cpp +++ b/src/targets/gpu/file_binary_cache.cpp @@ -108,7 +108,6 @@ void file_binary_cache::store(const std::string& version, const std::string& device, const std::vector& entries) const { - // Every entry shares one directory, so it is created once for the whole batch. auto dir = entry_dir(root, version, device); fs::create_directories(dir); // The content is decided entirely by the key, so a writer that loses the publish race diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index 9eac0dd1d01..f101351711b 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -85,7 +85,7 @@ struct MIGRAPHX_GPU_EXPORT binary_cache /// Nothing is opened here; storage is set up by the first lookup or insert, so a context that /// never compiles never touches the disk or probes the compiler. - explicit binary_cache(binary_cache_settings s = {}); + explicit binary_cache(binary_cache_settings s = {}) : settings(std::move(s)) {} /// Look up a key, consulting memory first and then the storage backend. optional get(const context& ctx, const std::string& key); diff --git a/src/targets/gpu/problem_cache.cpp b/src/targets/gpu/problem_cache.cpp index c9ab6e930ee..95fc81d57ac 100644 --- a/src/targets/gpu/problem_cache.cpp +++ b/src/targets/gpu/problem_cache.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -44,7 +45,7 @@ problem_cache::problem_cache() { writable_backends.emplace_back(json_problem_cac // priority list mix cache formats. static problem_cache_backend make_problem_cache_backend(const std::string& path) { - if(ends_with(path, ".db") or ends_with(path, ".sqlite")) + if(sqlite::is_database_path(path)) return problem_cache_backend{sqlite_problem_cache{}}; return problem_cache_backend{json_problem_cache{}}; } diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index 994b860d8fa..e24689a134c 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -118,13 +118,11 @@ static std::vector entry_files(const migraphx::fs::path& dir return result; } -/// Rows in one table of a cache database. The count is aliased because sqlite::execute keys its +/// Rows in the cache table of a database. The count is aliased because sqlite::execute keys its /// rows by column name, and an unaliased count(*) would be keyed by the text of the expression. -static std::size_t row_count(const std::string& path, const std::string& table) +static std::size_t row_count(const std::string& path) { - auto rows = migraphx::sqlite::read(path).execute("SELECT count(*) AS n FROM " + table + ";"); - if(rows.empty()) - return 0; + auto rows = migraphx::sqlite::read(path).execute("SELECT count(*) AS n FROM cache_v1;"); return std::stoul(rows.front().at("n")); } @@ -185,7 +183,7 @@ struct directory_backend struct database_backend { static std::string path(const migraphx::tmp_dir& td) { return db_path(td); } - static std::size_t stored(const std::string& p) { return row_count(p, "cache_v1"); } + static std::size_t stored(const std::string& p) { return row_count(p); } /// Overwrite every stored entry's code with bytes that do not decode. static void damage(const std::string& p) { @@ -427,7 +425,7 @@ TEST_CASE(extension_selects_the_backend) db_cache.insert(ctx, {make_entry("in-a-database")}); EXPECT(migraphx::fs::is_regular_file(path)); - EXPECT(row_count(path, "cache_v1") == 1); + EXPECT(row_count(path) == 1); EXPECT(entry_files(db_td.path).empty()); EXPECT(not migraphx::fs::exists(db_td.path / version_dir)); } @@ -486,7 +484,7 @@ TEST_CASE(incompatible_schema_degrades_to_memory) migraphx::gpu::binary_cache cache{migraphx::gpu::binary_cache_settings{path, false}}; cache.insert(ctx, {make_entry("in-memory")}); EXPECT(cache.get(ctx, "in-memory").has_value()); - EXPECT(row_count(path, "cache_v1") == 0); + EXPECT(row_count(path) == 0); } // Both backends address an entry by the same key hash. @@ -633,7 +631,7 @@ TEST_CASE(sqlite_records_the_full_version_id) cache.insert(ctx, {make_entry("one")}); cache.insert(ctx, {make_entry("two")}); - EXPECT(row_count(path, "cache_v1") == 2); + EXPECT(row_count(path) == 2); auto rows = migraphx::sqlite::read(path).execute("SELECT DISTINCT version FROM cache_v1;"); EXPECT(rows.size() == 1); @@ -689,7 +687,7 @@ TEST_CASE(sqlite_read_only_database_still_serves_hits) EXPECT(reader.get(ctx, "new").has_value()); if(protected_file) { - EXPECT(row_count(path, "cache_v1") == 1); + EXPECT(row_count(path) == 1); } // Restored so the temporary directory can be removed, which Windows refuses otherwise. @@ -729,7 +727,7 @@ TEST_CASE(backends_store_overwrites_in_place) EXPECT(got.has_value()); EXPECT(got->solution == replacement.solution); } - EXPECT(row_count(db_path(td), "cache_v1") == 1); + EXPECT(row_count(db_path(td)) == 1); EXPECT(entry_files(td.path / "files").size() == 1); } @@ -808,7 +806,7 @@ TEST_CASE(sqlite_store_releases_the_database) auto other = migraphx::gpu::sqlite_binary_cache::open(path); EXPECT(other.has_value()); other->store("v", "dev", {make_entry("k")}); - EXPECT(row_count(path, "cache_v1") == 3); + EXPECT(row_count(path) == 3); } // Rows are addressed by a hash of the key, so a row whose stored key differs from the one asked From 2bd99b54c3f4b43bef93ad9782e8b33f23aef80d Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Thu, 8 Oct 2026 11:12:12 +0200 Subject: [PATCH 47/49] Add sqlite tests --- test/sqlite.cpp | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/sqlite.cpp b/test/sqlite.cpp index 02910f109af..6d74d5a46aa 100644 --- a/test/sqlite.cpp +++ b/test/sqlite.cpp @@ -208,4 +208,61 @@ TEST_CASE(try_write_unusable_path) EXPECT(migraphx::sqlite::try_write(td.path / "ok.db").has_value()); } +TEST_CASE(is_database_path_matches_extensions) +{ + EXPECT(migraphx::sqlite::is_database_path("cache.db")); + EXPECT(migraphx::sqlite::is_database_path("/tmp/kernels/cache.sqlite")); + EXPECT(not migraphx::sqlite::is_database_path("")); + EXPECT(not migraphx::sqlite::is_database_path("/tmp/kernels")); + EXPECT(not migraphx::sqlite::is_database_path("/tmp/kernels/")); + EXPECT(not migraphx::sqlite::is_database_path("cache.json")); + EXPECT(not migraphx::sqlite::is_database_path("cache.db.bak")); +} + +// Text that is not SQL, or that holds no statement at all, is refused when prepared rather than +// handing back a statement that fails on its first call. +TEST_CASE(prepare_rejects_invalid_sql) +{ + migraphx::tmp_dir td{}; + auto db = migraphx::sqlite::write(td.path / "invalid.db"); + EXPECT(test::throws([&] { db.prepare("NOT SQL;"); })); + EXPECT(test::throws([&] { db.prepare("-- only a comment"); })); +} + +// A call whose statement fails throws, and leaves the statement ready to be called again. +TEST_CASE(failed_call_resets_statement) +{ + migraphx::tmp_dir td{}; + auto db = migraphx::sqlite::write(td.path / "fail.db"); + db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY);"); + auto insert = db.prepare("INSERT INTO t (id) VALUES (?);"); + + insert(std::int64_t{1}); + // The primary key is already taken, so the first step fails. + EXPECT(test::throws([&] { insert(std::int64_t{1}); })); + insert(std::int64_t{2}); + EXPECT(db.execute("SELECT id FROM t;").size() == 2); +} + +// A writer that cannot get the lock waits out its busy timeout and then fails, rather than +// blocking; once the lock is released, the same statement succeeds. +TEST_CASE(busy_timeout_gives_up_on_locked_database) +{ + migraphx::tmp_dir td{}; + auto path = td.path / "busy.db"; + auto holder = migraphx::sqlite::write(path); + holder.execute("CREATE TABLE t (id INTEGER PRIMARY KEY);"); + + auto writer = migraphx::sqlite::write(path); + writer.set_busy_timeout(1); + auto insert = writer.prepare("INSERT INTO t (id) VALUES (?);"); + + holder.execute("BEGIN IMMEDIATE;"); + EXPECT(test::throws([&] { insert(std::int64_t{1}); })); + holder.execute("COMMIT;"); + + insert(std::int64_t{1}); + EXPECT(holder.execute("SELECT id FROM t;").size() == 1); +} + int main(int argc, const char* argv[]) { test::run(argc, argv); } From 2016267c3ac0b9dbdd6b2dca84aff896da461a87 Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Fri, 9 Oct 2026 16:46:15 +0200 Subject: [PATCH 48/49] Fix mlir keys --- src/targets/gpu/mlir.cpp | 19 +++++- test/gpu/mlir_compile_key.cpp | 112 ++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 test/gpu/mlir_compile_key.cpp diff --git a/src/targets/gpu/mlir.cpp b/src/targets/gpu/mlir.cpp index a335b8fc264..987e6c9e08a 100644 --- a/src/targets/gpu/mlir.cpp +++ b/src/targets/gpu/mlir.cpp @@ -1431,7 +1431,24 @@ std::string mlir_compile_key(const context& migraphx_ctx, // also become fields of the code object that the caller inserts. for(const auto& s : in_shapes) ss << "input=" << s << "\n"; - ss << mlir_print(&mlirOperationPrint, mlirModuleGetOperation(mp.mmodule.get())); + // The printer's defaults come from LLVM's global command-line options, which rocMLIR sets + // when it is registered (local scope) and which the linker resets during the first compile. + // Pinning the flags keeps the key the same however many kernels the process has compiled. + // Local scope is the form rocMLIR selects, so keys computed before any compile, which is + // most of those already stored, still match. + mlir_op_printing_flags flags{mlirOpPrintingFlagsCreate()}; + mlirOpPrintingFlagsUseLocalScope(flags.get()); + mlirOpPrintingFlagsEnableDebugInfo(flags.get(), /*enable=*/false, /*prettyForm=*/false); + // Never elide constants: kernels that differ only in a large literal must not share a key. + mlirOpPrintingFlagsElideLargeElementsAttrs(flags.get(), INTPTR_MAX); + mlirOpPrintingFlagsElideLargeResourceString(flags.get(), INTPTR_MAX); + // A module that fails to verify would otherwise be printed in the generic form. + mlirOpPrintingFlagsAssumeVerified(flags.get()); + ss << mlir_print( + [&](MlirOperation op, MlirStringCallback callback, void* data) { + mlirOperationPrintWithFlags(op, flags.get(), callback, data); + }, + mlirModuleGetOperation(mp.mmodule.get())); return ss.str(); } diff --git a/test/gpu/mlir_compile_key.cpp b/test/gpu/mlir_compile_key.cpp new file mode 100644 index 00000000000..715015af6a7 --- /dev/null +++ b/test/gpu/mlir_compile_key.cpp @@ -0,0 +1,112 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include + +// The binary cache addresses MLIR kernels by mlir_compile_key, so the key for a module must not +// depend on what the process has compiled before it. rocMLIR sets global printer options when it +// is registered and its linker resets them during the first compile, so this test only means +// something while no MLIR kernel has been compiled yet. It must stay the first test in this +// file, and this file must stay its own executable. +TEST_CASE(mlir_compile_key_is_stable_across_compiles) +{ + migraphx::module m; + auto arg0 = m.add_parameter("arg0", {migraphx::shape::half_type, {1, 64, 32}}); + auto arg1 = m.add_parameter("arg1", {migraphx::shape::half_type, {1, 32, 48}}); + auto arg2 = m.add_parameter("arg2", {migraphx::shape::half_type, {1, 64, 48}}); + auto dot = m.add_instruction(migraphx::make_op("dot"), arg0, arg1); + auto add = m.add_instruction(migraphx::make_op("add"), dot, arg2); + m.add_return({add}); + std::vector shapes = {{migraphx::shape::half_type, {1, 64, 32}}, + {migraphx::shape::half_type, {1, 32, 48}}, + {migraphx::shape::half_type, {1, 64, 48}}, + {migraphx::shape::half_type, {1, 64, 48}}}; + + migraphx::gpu::context ctx; + auto tc = migraphx::gpu::get_tuning_config_mlir(ctx, m, shapes, false); + // Skip when MLIR is not enabled + if(tc.solutions.empty()) + return; + const auto& solution = tc.solutions.front(); + + auto before = migraphx::gpu::mlir_compile_key(ctx, m, shapes, solution); + // Skip when MLIR is not enabled + if(before.empty()) + return; + auto between = migraphx::gpu::mlir_compile_key(ctx, m, shapes, solution); + migraphx::gpu::compile_mlir(ctx, m, shapes, solution); + auto after = migraphx::gpu::mlir_compile_key(ctx, m, shapes, solution); + + EXPECT(before == between); + EXPECT(before == after); + // The key is printed in local scope, the form rocMLIR selects before any compile, so keys + // stored by earlier builds still match. A top-level print would end with a newline. + EXPECT(not migraphx::ends_with(after, "\n")); + // Locations never reach the key + EXPECT(after.find("loc(") == std::string::npos); +} + +// Kernels that differ only in the value of a literal must not share a key, however large the +// literal is, since the printer can elide large constants. +TEST_CASE(mlir_compile_key_distinguishes_literals) +{ + migraphx::shape ws{migraphx::shape::float_type, {1, 64, 32}}; + std::vector weights(ws.elements()); + std::iota(weights.begin(), weights.end(), 0.0f); + + migraphx::module m1; + { + auto arg0 = m1.add_parameter("arg0", {migraphx::shape::float_type, {1, 16, 64}}); + auto lit = m1.add_literal(migraphx::literal{ws, weights}); + auto dot = m1.add_instruction(migraphx::make_op("dot"), arg0, lit); + m1.add_return({dot}); + } + weights.back() += 1.0f; + migraphx::module m2; + { + auto arg0 = m2.add_parameter("arg0", {migraphx::shape::float_type, {1, 16, 64}}); + auto lit = m2.add_literal(migraphx::literal{ws, weights}); + auto dot = m2.add_instruction(migraphx::make_op("dot"), arg0, lit); + m2.add_return({dot}); + } + std::vector shapes = {{migraphx::shape::float_type, {1, 16, 64}}, + {migraphx::shape::float_type, {1, 16, 32}}}; + + migraphx::gpu::context ctx; + auto key1 = migraphx::gpu::mlir_compile_key(ctx, m1, shapes, migraphx::value{}); + // Skip when MLIR is not enabled + if(key1.empty()) + return; + auto key2 = migraphx::gpu::mlir_compile_key(ctx, m2, shapes, migraphx::value{}); + EXPECT(key1 != key2); +} + +int main(int argc, const char* argv[]) { test::run(argc, argv); } From 33c641ce5844fc44f05dbc618ae2d57a95d85cfa Mon Sep 17 00:00:00 2001 From: pnikolic-amd Date: Fri, 9 Oct 2026 16:55:19 +0200 Subject: [PATCH 49/49] Simplify --- src/include/migraphx/value.hpp | 10 ------ src/sqlite.cpp | 23 +++++++------ src/targets/gpu/binary_cache.cpp | 15 ++++----- src/targets/gpu/compile_ops.cpp | 18 +++++------ .../gpu/include/migraphx/gpu/binary_cache.hpp | 5 +-- src/targets/gpu/mlir.cpp | 7 ++-- test/gpu/binary_cache.cpp | 32 ++++++++----------- test/gpu/mlir_compile_key.cpp | 5 +-- 8 files changed, 49 insertions(+), 66 deletions(-) diff --git a/src/include/migraphx/value.hpp b/src/include/migraphx/value.hpp index 572f4742931..70ce5613fde 100644 --- a/src/include/migraphx/value.hpp +++ b/src/include/migraphx/value.hpp @@ -297,16 +297,6 @@ struct MIGRAPHX_EXPORT value value(const std::pair& p) : value(p.first, p.second) { } - - /// A keyed value. Tidy suggests returning {key, x} instead of value(key, x), but braces pick - /// the initializer_list constructor and build a two-element array; this keeps that - /// suppression in one place. - template - static value pair(const std::string& pkey, Ts&&... xs) - { - // NOLINTNEXTLINE(modernize-return-braced-init-list) - return value(pkey, static_cast(xs)...); - } template {})> value& operator=(T rhs) { diff --git a/src/sqlite.cpp b/src/sqlite.cpp index 600001248fa..48951f2ace3 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -233,17 +233,14 @@ void sqlite_stmt::reset() const noexcept (void)sqlite3_clear_bindings(impl->get()); } -/// Column i of the current row, keyed by its name. +/// Column i of the current row. static value column_value(sqlite3_stmt* stmt, int i) { - // Null only when sqlite runs out of memory, which would make the string below undefined. - const char* name = sqlite3_column_name(stmt, i); - assert(name != nullptr); auto type = sqlite3_column_type(stmt, i); switch(type) { - case SQLITE_INTEGER: return value::pair(name, std::int64_t{sqlite3_column_int64(stmt, i)}); - case SQLITE_FLOAT: return value::pair(name, sqlite3_column_double(stmt, i)); + case SQLITE_INTEGER: return std::int64_t{sqlite3_column_int64(stmt, i)}; + case SQLITE_FLOAT: return sqlite3_column_double(stmt, i); case SQLITE_TEXT: case SQLITE_BLOB: { // The data must be fetched before sqlite3_column_bytes: the other order can force a @@ -254,10 +251,10 @@ static value column_value(sqlite3_stmt* stmt, int i) assert(bytes >= 0); std::size_t size = data == nullptr ? 0 : bytes; if(type == SQLITE_TEXT) - return value::pair(name, size == 0 ? std::string{} : std::string(data, size)); - return value::pair(name, value::binary{data, size}); + return size == 0 ? std::string{} : std::string(data, size); + return value::binary{data, size}; } - default: return value::pair(name, nullptr); + default: return nullptr; } } @@ -267,7 +264,13 @@ value sqlite_stmt::to_value() const value columns = value::object{}; const int n = sqlite3_column_count(stmt); for(int i = 0; i < n; ++i) - columns.insert(column_value(stmt, i)); + { + // Null only when sqlite runs out of memory; a key cannot be built from a null name. + const char* name = sqlite3_column_name(stmt, i); + assert(name != nullptr); + // Assigned rather than inserted, which would copy the column, blob and all. + columns[name] = column_value(stmt, i); + } return columns; } diff --git a/src/targets/gpu/binary_cache.cpp b/src/targets/gpu/binary_cache.cpp index b485d92a514..7b2d40915f4 100644 --- a/src/targets/gpu/binary_cache.cpp +++ b/src/targets/gpu/binary_cache.cpp @@ -104,12 +104,10 @@ static std::string device_dir(const context& ctx) "_wf" + std::to_string(device.get_wavefront_size()); } -// The storage backend is selected by the path's extension, the same rule -// make_problem_cache_backend applies: a SQLite database for sqlite::is_database_path, anything -// else is a directory of entries. A directory is named with the short version id to keep paths -// short; a database records the full id, which is self-describing. Nothing is persisted when the -// compiler cannot be identified, since entries from different toolchains would be -// indistinguishable. +// A path sqlite::is_database_path accepts is a SQLite database; anything else is a directory of +// entries. A directory is named with the short version id to keep paths short; a database records +// the full id. Nothing is persisted when the compiler cannot be identified, since entries from +// different toolchains would be indistinguishable. binary_cache_backend* binary_cache::get_backend() { if(backend_opened) @@ -153,9 +151,10 @@ optional binary_cache::get(const context& ctx, const std::string& return nullopt; } -void binary_cache::insert(const context& ctx, std::vector es) +void binary_cache::insert(const context& ctx, std::vector es) { - es.erase(std::remove_if(es.begin(), es.end(), [](const entry& e) { return e.key.empty(); }), + es.erase(std::remove_if( + es.begin(), es.end(), [](const binary_cache_entry& e) { return e.key.empty(); }), es.end()); if(es.empty()) return; diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 23a87eb9c2e..ef10de689d7 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -219,15 +219,15 @@ static optional cache_lookup(context& ctx, const std::string& } /// What to record for a freshly compiled result, or nullopt when its key must not be cached. -static optional make_cache_entry(const operation& preop, - const value& solution, - const std::string& key, - const value& problem, - const compiled_code& code) +static optional make_cache_entry(const operation& preop, + const value& solution, + const std::string& key, + const value& problem, + const compiled_code& code) { if(is_private_key(key)) return nullopt; - binary_cache::entry e; + binary_cache_entry e; e.key = key; e.op_name = preop.name(); e.problem = problem; @@ -255,7 +255,7 @@ static compiler_replace compile_cached(context& ctx, if(auto e = make_cache_entry(preop, solution, key, problem, cr.code)) { // Built by push_back, since an initializer list would copy the entry rather than move it. - std::vector es; + std::vector es; es.push_back(std::move(*e)); ctx.get_binary_cache().insert(ctx, std::move(es)); } @@ -454,7 +454,7 @@ struct compile_plan return cache_lookup(*ctx, key); } - optional + optional cache_entry(const value& solution, const std::string& key, const compiled_code& code) const { return make_cache_entry(preop, solution, key, config ? config->problem : value{}, code); @@ -800,7 +800,7 @@ struct compile_manager // single insert, letting its storage commit them together rather than one at a time. auto* ctx = tasks.front().first->ctx; assert(all_of(tasks, [&](const auto& task) { return task.first->ctx == ctx; })); - std::vector entries; + std::vector entries; entries.reserve(tasks.size()); for(const auto& [cp, cell] : tasks) { diff --git a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp index f101351711b..0663293f888 100644 --- a/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp +++ b/src/targets/gpu/include/migraphx/gpu/binary_cache.hpp @@ -67,9 +67,6 @@ struct binary_cache_settings */ struct MIGRAPHX_GPU_EXPORT binary_cache { - /// What gets stored for one compiled kernel; see binary_cache_entry.hpp. - using entry = binary_cache_entry; - /// Counts of what the cache did. struct stats { @@ -92,7 +89,7 @@ struct MIGRAPHX_GPU_EXPORT binary_cache /// Record compiled results under their keys. They are handed to the storage backend /// together, so it can commit them at once rather than one at a time. - void insert(const context& ctx, std::vector es); + void insert(const context& ctx, std::vector es); /// True when reused results should be checked against a fresh compile. bool verify() const { return settings.verify; } diff --git a/src/targets/gpu/mlir.cpp b/src/targets/gpu/mlir.cpp index 987e6c9e08a..45a62fd58e0 100644 --- a/src/targets/gpu/mlir.cpp +++ b/src/targets/gpu/mlir.cpp @@ -1431,9 +1431,10 @@ std::string mlir_compile_key(const context& migraphx_ctx, // also become fields of the code object that the caller inserts. for(const auto& s : in_shapes) ss << "input=" << s << "\n"; - // The printer's defaults come from LLVM's global command-line options, which rocMLIR sets - // when it is registered (local scope) and which the linker resets during the first compile. - // Pinning the flags keeps the key the same however many kernels the process has compiled. + // The printer's defaults come from LLVM's global command-line options: rocMLIR sets + // --mlir-print-local-scope when it is registered, and the in-process lld linker resets every + // option during the first compile. Pinning the flags keeps the key the same however many + // kernels the process has compiled. // Local scope is the form rocMLIR selects, so keys computed before any compile, which is // most of those already stored, still match. mlir_op_printing_flags flags{mlirOpPrintingFlagsCreate()}; diff --git a/test/gpu/binary_cache.cpp b/test/gpu/binary_cache.cpp index e24689a134c..d077108dcde 100644 --- a/test/gpu/binary_cache.cpp +++ b/test/gpu/binary_cache.cpp @@ -90,9 +90,9 @@ static migraphx::gpu::compiled_code make_code() return code; } -static migraphx::gpu::binary_cache::entry make_entry(const std::string& key) +static migraphx::gpu::binary_cache_entry make_entry(const std::string& key) { - migraphx::gpu::binary_cache::entry e; + migraphx::gpu::binary_cache_entry e; e.key = key; e.op_name = "pointwise"; e.problem = migraphx::value{{"shape", "float_type{4, 8}"}}; @@ -102,7 +102,6 @@ static migraphx::gpu::binary_cache::entry make_entry(const std::string& key) } // The storage backend is chosen by the extension of the cache path. -static std::string dir_path(const migraphx::tmp_dir& td) { return td.path.string(); } static std::string db_path(const migraphx::tmp_dir& td) { return (td.path / "cache.db").string(); } /// The entry files a directory-backed cache has written. @@ -156,8 +155,8 @@ static stored_entries db_entries(const std::string& path) } /// Whether two entries hold the same thing. -static bool same_entry(const migraphx::gpu::binary_cache::entry& x, - const migraphx::gpu::binary_cache::entry& y) +static bool same_entry(const migraphx::gpu::binary_cache_entry& x, + const migraphx::gpu::binary_cache_entry& y) { return x.key == y.key and x.op_name == y.op_name and x.problem == y.problem and x.solution == y.solution and x.code.fill_map == y.code.fill_map and @@ -168,7 +167,7 @@ static bool same_entry(const migraphx::gpu::binary_cache::entry& x, // written once as a template and registered for both. struct directory_backend { - static std::string path(const migraphx::tmp_dir& td) { return dir_path(td); } + static std::string path(const migraphx::tmp_dir& td) { return td.path.string(); } static std::size_t stored(const std::string& p) { return entry_files(p).size(); } /// Overwrite every stored entry with bytes that do not decode. static void damage(const std::string& p) @@ -232,8 +231,7 @@ TEST_CASE(memory_lookup_records_reuse) EXPECT(cache.get_stats().misses == 0); } -// The cases below are written once against a Backend and registered for each. The directory -// registrations use real temporary paths so that on Windows they exercise the full depth of an +// Directory cases use real temporary paths so that on Windows they exercise the full depth of an // entry path against MAX_PATH. // A second cache shares nothing in memory, so anything it finds came out of storage. @@ -408,14 +406,12 @@ TEST_CASE(extension_selects_the_backend) migraphx::tmp_dir dir_td{"binary-cache"}; migraphx::gpu::binary_cache dir_cache{ - migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; + migraphx::gpu::binary_cache_settings{dir_td.path.string(), false}}; dir_cache.insert(ctx, {make_entry("in-a-directory")}); auto files = entry_files(dir_td.path); EXPECT(files.size() == 1); EXPECT(migraphx::fs::is_directory(dir_td.path / version_dir)); - EXPECT(std::all_of(files.begin(), files.end(), [&](const auto& f) { - return f.parent_path().parent_path() == dir_td.path / version_dir; - })); + EXPECT(files.front().parent_path().parent_path() == dir_td.path / version_dir); for(const char* name : {"cache.db", "cache.sqlite"}) { @@ -495,7 +491,7 @@ TEST_CASE(backends_store_the_same_entry) migraphx::tmp_dir dir_td{"binary-cache"}; migraphx::gpu::binary_cache dir_cache{ - migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; + migraphx::gpu::binary_cache_settings{dir_td.path.string(), false}}; dir_cache.insert(ctx, {e}); auto from_dir = dir_entries(dir_td.path); EXPECT(from_dir.size() == 1); @@ -518,7 +514,7 @@ TEST_CASE(backends_hold_the_same_entries_after_a_compile) auto path = db_path(db_td); auto p_dir = pointwise_program(); - p_dir.compile(migraphx::make_target("gpu"), cache_options(dir_path(dir_td))); + p_dir.compile(migraphx::make_target("gpu"), cache_options(dir_td.path.string())); auto p_db = pointwise_program(); p_db.compile(migraphx::make_target("gpu"), cache_options(path)); @@ -544,7 +540,7 @@ TEST_CASE(entries_move_between_backends) migraphx::tmp_dir dir_td{"binary-cache"}; migraphx::tmp_dir db_td{"binary-cache"}; migraphx::gpu::binary_cache writer{ - migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; + migraphx::gpu::binary_cache_settings{dir_td.path.string(), false}}; writer.insert(ctx, {e}); auto files = entry_files(dir_td.path); EXPECT(files.size() == 1); @@ -586,7 +582,7 @@ TEST_CASE(entries_move_between_backends) dir.store(short_version, device, {*loaded}); migraphx::gpu::binary_cache reader{ - migraphx::gpu::binary_cache_settings{dir_path(dir_td), false}}; + migraphx::gpu::binary_cache_settings{dir_td.path.string(), false}}; auto found = reader.get(ctx, e.key); EXPECT(found.has_value()); EXPECT(reader.get_stats().hits == 1); @@ -737,7 +733,7 @@ TEST_CASE(file_store_leaves_only_entries_behind) { migraphx::tmp_dir td{"binary-cache"}; migraphx::gpu::context ctx; - migraphx::gpu::binary_cache_settings settings{dir_path(td), false}; + migraphx::gpu::binary_cache_settings settings{td.path.string(), false}; migraphx::gpu::binary_cache first{settings}; first.insert(ctx, {make_entry("one")}); @@ -846,7 +842,7 @@ TEST_CASE(entry_round_trip) auto e = make_entry("some-key"); auto buffer = migraphx::to_msgpack(migraphx::to_value(e)); - migraphx::gpu::binary_cache::entry loaded; + migraphx::gpu::binary_cache_entry loaded; migraphx::from_value(migraphx::from_msgpack(buffer), loaded); EXPECT(same_entry(loaded, e)); diff --git a/test/gpu/mlir_compile_key.cpp b/test/gpu/mlir_compile_key.cpp index 715015af6a7..b6f116656ff 100644 --- a/test/gpu/mlir_compile_key.cpp +++ b/test/gpu/mlir_compile_key.cpp @@ -57,10 +57,7 @@ TEST_CASE(mlir_compile_key_is_stable_across_compiles) return; const auto& solution = tc.solutions.front(); - auto before = migraphx::gpu::mlir_compile_key(ctx, m, shapes, solution); - // Skip when MLIR is not enabled - if(before.empty()) - return; + auto before = migraphx::gpu::mlir_compile_key(ctx, m, shapes, solution); auto between = migraphx::gpu::mlir_compile_key(ctx, m, shapes, solution); migraphx::gpu::compile_mlir(ctx, m, shapes, solution); auto after = migraphx::gpu::mlir_compile_key(ctx, m, shapes, solution);