|
| 1 | +//***************************************************************************** |
| 2 | +// Copyright (c) 2026, Intel Corporation |
| 3 | +// All rights reserved. |
| 4 | +// |
| 5 | +// Redistribution and use in source and binary forms, with or without |
| 6 | +// modification, are permitted provided that the following conditions are met: |
| 7 | +// - Redistributions of source code must retain the above copyright notice, |
| 8 | +// this list of conditions and the following disclaimer. |
| 9 | +// - Redistributions in binary form must reproduce the above copyright notice, |
| 10 | +// this list of conditions and the following disclaimer in the documentation |
| 11 | +// and/or other materials provided with the distribution. |
| 12 | +// - Neither the name of the copyright holder nor the names of its contributors |
| 13 | +// may be used to endorse or promote products derived from this software |
| 14 | +// without specific prior written permission. |
| 15 | +// |
| 16 | +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| 17 | +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 18 | +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 19 | +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
| 20 | +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 21 | +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 22 | +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 23 | +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 24 | +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 25 | +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF |
| 26 | +// THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | +//***************************************************************************** |
| 28 | + |
| 29 | +#include <algorithm> |
| 30 | +#include <cstdint> |
| 31 | +#include <sstream> |
| 32 | +#include <stdexcept> |
| 33 | +#include <string> |
| 34 | +#include <utility> |
| 35 | +#include <vector> |
| 36 | + |
| 37 | +#include <pybind11/pybind11.h> |
| 38 | + |
| 39 | +// utils extension header |
| 40 | +#include "ext/common.hpp" |
| 41 | + |
| 42 | +// dpnp tensor headers |
| 43 | +#include "utils/memory_overlap.hpp" |
| 44 | +#include "utils/output_validation.hpp" |
| 45 | +#include "utils/type_utils.hpp" |
| 46 | + |
| 47 | +#include "trsm.hpp" |
| 48 | +#include "types_matrix.hpp" |
| 49 | + |
| 50 | +namespace dpnp::extensions::blas |
| 51 | +{ |
| 52 | +namespace mkl_blas = oneapi::mkl::blas; |
| 53 | +namespace py = pybind11; |
| 54 | +namespace type_utils = dpnp::tensor::type_utils; |
| 55 | + |
| 56 | +using ext::common::init_dispatch_vector; |
| 57 | + |
| 58 | +typedef sycl::event (*trsm_batch_impl_fn_ptr_t)( |
| 59 | + sycl::queue &, |
| 60 | + const oneapi::mkl::uplo, |
| 61 | + const oneapi::mkl::transpose, |
| 62 | + const std::int64_t, // m |
| 63 | + const std::int64_t, // n |
| 64 | + const char *, // a |
| 65 | + const std::int64_t, // lda |
| 66 | + const std::int64_t, // stride_a |
| 67 | + char *, // b |
| 68 | + const std::int64_t, // ldb |
| 69 | + const std::int64_t, // stride_b |
| 70 | + const std::int64_t, // batch_size |
| 71 | + const std::vector<sycl::event> &); |
| 72 | + |
| 73 | +static trsm_batch_impl_fn_ptr_t |
| 74 | + trsm_batch_dispatch_vector[dpnp_td_ns::num_types]; |
| 75 | + |
| 76 | +template <typename T> |
| 77 | +static sycl::event trsm_batch_impl(sycl::queue &exec_q, |
| 78 | + const oneapi::mkl::uplo upper_lower, |
| 79 | + const oneapi::mkl::transpose trans, |
| 80 | + const std::int64_t m, |
| 81 | + const std::int64_t n, |
| 82 | + const char *matrixA, |
| 83 | + const std::int64_t lda, |
| 84 | + const std::int64_t stride_a, |
| 85 | + char *matrixB, |
| 86 | + const std::int64_t ldb, |
| 87 | + const std::int64_t stride_b, |
| 88 | + const std::int64_t batch_size, |
| 89 | + const std::vector<sycl::event> &depends) |
| 90 | +{ |
| 91 | + type_utils::validate_type_for_device<T>(exec_q); |
| 92 | + |
| 93 | + const T *a = reinterpret_cast<const T *>(matrixA); |
| 94 | + T *b = reinterpret_cast<T *>(matrixB); |
| 95 | + |
| 96 | + std::stringstream error_msg; |
| 97 | + bool is_exception_caught = false; |
| 98 | + |
| 99 | + sycl::event trsm_batch_event; |
| 100 | + try { |
| 101 | + // Release GIL to avoid serialization of host task submissions |
| 102 | + // to the same queue in OneMKL |
| 103 | + py::gil_scoped_release lock{}; |
| 104 | + |
| 105 | + trsm_batch_event = mkl_blas::column_major::trsm_batch( |
| 106 | + exec_q, |
| 107 | + oneapi::mkl::side::left, // op(A) is on the left: op(A) X = B |
| 108 | + upper_lower, // Whether A is upper or lower triangular. |
| 109 | + trans, // Defines op(A): 'N' for A, 'T' for A^T, or 'C' |
| 110 | + // for A^H. |
| 111 | + oneapi::mkl::diag::nonunit, // A is not unit triangular. |
| 112 | + m, // Number of rows of B, and the order of A. |
| 113 | + n, // Number of columns of B. |
| 114 | + T(1), // Scaling factor for B. |
| 115 | + a, // Pointer to the batch of triangular matrices A. |
| 116 | + lda, // Leading dimension of each matrix A. |
| 117 | + stride_a, // Stride between consecutive A matrices. |
| 118 | + b, // Pointer to the batch of matrices B, overwritten |
| 119 | + // by the solutions X. |
| 120 | + ldb, // Leading dimension of each matrix B. |
| 121 | + stride_b, // Stride between consecutive B matrices. |
| 122 | + batch_size, // Total number of matrices in the batch. |
| 123 | + depends); |
| 124 | + } catch (oneapi::mkl::exception const &e) { |
| 125 | + error_msg << "Unexpected MKL exception caught during trsm_batch() " |
| 126 | + "call:\nreason: " |
| 127 | + << e.what(); |
| 128 | + is_exception_caught = true; |
| 129 | + } catch (sycl::exception const &e) { |
| 130 | + error_msg |
| 131 | + << "Unexpected SYCL exception caught during trsm_batch() call:\n" |
| 132 | + << e.what(); |
| 133 | + is_exception_caught = true; |
| 134 | + } |
| 135 | + |
| 136 | + if (is_exception_caught) // an unexpected error occurs |
| 137 | + { |
| 138 | + throw std::runtime_error(error_msg.str()); |
| 139 | + } |
| 140 | + |
| 141 | + return trsm_batch_event; |
| 142 | +} |
| 143 | + |
| 144 | +std::pair<sycl::event, sycl::event> |
| 145 | + trsm_batch(sycl::queue &exec_q, |
| 146 | + const dpnp::tensor::usm_ndarray &matrixA, |
| 147 | + const dpnp::tensor::usm_ndarray &matrixB, |
| 148 | + const std::int8_t upper_lower, |
| 149 | + const int trans_op, |
| 150 | + const std::vector<sycl::event> &depends) |
| 151 | +{ |
| 152 | + const int a_nd = matrixA.get_ndim(); |
| 153 | + const int b_nd = matrixB.get_ndim(); |
| 154 | + |
| 155 | + // batch is the last dimension: (m, m, batch) and (m, n, batch), |
| 156 | + // a 2-D (m, m) matrix A is shared by all right-hand sides |
| 157 | + if ((a_nd != 2 && a_nd != 3) || b_nd != 3) { |
| 158 | + throw py::value_error( |
| 159 | + "The triangular and right-hand sides arrays must be " |
| 160 | + "2- or 3-dimensional and 3-dimensional, but got ndim=" + |
| 161 | + std::to_string(a_nd) + " and ndim=" + std::to_string(b_nd) + "."); |
| 162 | + } |
| 163 | + if (trans_op < 0 || trans_op > 2) { |
| 164 | + throw py::value_error( |
| 165 | + "trsm_batch: trans_op must be 0 (N), 1 (T), or 2 (C)."); |
| 166 | + } |
| 167 | + |
| 168 | + const py::ssize_t *a_shape = matrixA.get_shape_raw(); |
| 169 | + const py::ssize_t *b_shape = matrixB.get_shape_raw(); |
| 170 | + |
| 171 | + if (a_shape[0] != a_shape[1]) { |
| 172 | + throw py::value_error("Expected batch of square matrices, but got " |
| 173 | + "matrix shape (" + |
| 174 | + std::to_string(a_shape[0]) + ", " + |
| 175 | + std::to_string(a_shape[1]) + ") in batch"); |
| 176 | + } |
| 177 | + if (a_shape[0] != b_shape[0] || (a_nd == 3 && a_shape[2] != b_shape[2])) { |
| 178 | + throw py::value_error("The shapes of the triangular and right-hand " |
| 179 | + "sides arrays are incompatible."); |
| 180 | + } |
| 181 | + |
| 182 | + // check compatibility of execution queue and allocation queue |
| 183 | + if (!dpnp::utils::queues_are_compatible(exec_q, {matrixA, matrixB})) { |
| 184 | + throw py::value_error( |
| 185 | + "Execution queue is not compatible with allocation queues"); |
| 186 | + } |
| 187 | + |
| 188 | + auto const &overlap = dpnp::tensor::overlap::MemoryOverlap(); |
| 189 | + if (overlap(matrixA, matrixB)) { |
| 190 | + throw py::value_error("The triangular and right-hand sides arrays " |
| 191 | + "are overlapping segments of memory"); |
| 192 | + } |
| 193 | + |
| 194 | + dpnp::tensor::validation::CheckWritable::throw_if_not_writable(matrixB); |
| 195 | + |
| 196 | + if (!matrixA.is_f_contiguous() || !matrixB.is_f_contiguous()) { |
| 197 | + throw py::value_error("The triangular and right-hand sides arrays " |
| 198 | + "must be F-contiguous"); |
| 199 | + } |
| 200 | + |
| 201 | + auto array_types = dpnp_td_ns::usm_ndarray_types(); |
| 202 | + const int a_type_id = |
| 203 | + array_types.typenum_to_lookup_id(matrixA.get_typenum()); |
| 204 | + const int b_type_id = |
| 205 | + array_types.typenum_to_lookup_id(matrixB.get_typenum()); |
| 206 | + |
| 207 | + if (a_type_id != b_type_id) { |
| 208 | + throw py::value_error("The types of the triangular and right-hand " |
| 209 | + "sides arrays are mismatched"); |
| 210 | + } |
| 211 | + |
| 212 | + trsm_batch_impl_fn_ptr_t trsm_batch_fn = |
| 213 | + trsm_batch_dispatch_vector[a_type_id]; |
| 214 | + if (trsm_batch_fn == nullptr) { |
| 215 | + throw py::value_error( |
| 216 | + "No trsm_batch implementation defined for the provided type " |
| 217 | + "of the input matrix"); |
| 218 | + } |
| 219 | + |
| 220 | + const std::int64_t m = a_shape[0]; |
| 221 | + const std::int64_t n = b_shape[1]; |
| 222 | + const std::int64_t batch_size = b_shape[2]; |
| 223 | + |
| 224 | + // nothing to solve, oneMKL does not need to be called |
| 225 | + if (m == 0 || n == 0 || batch_size == 0) { |
| 226 | + return std::make_pair(sycl::event(), sycl::event()); |
| 227 | + } |
| 228 | + |
| 229 | + const std::int64_t lda = std::max<std::int64_t>(1, m); |
| 230 | + const std::int64_t ldb = std::max<std::int64_t>(1, m); |
| 231 | + // stride 0 reuses the shared matrix A for each batch element |
| 232 | + const std::int64_t stride_a = (a_nd == 2) ? 0 : m * m; |
| 233 | + const std::int64_t stride_b = m * n; |
| 234 | + |
| 235 | + const oneapi::mkl::uplo uplo_val = |
| 236 | + static_cast<oneapi::mkl::uplo>(upper_lower); |
| 237 | + const oneapi::mkl::transpose trans = |
| 238 | + (trans_op == 0) ? oneapi::mkl::transpose::N |
| 239 | + : (trans_op == 1) ? oneapi::mkl::transpose::T |
| 240 | + : oneapi::mkl::transpose::C; |
| 241 | + |
| 242 | + sycl::event trsm_batch_ev = trsm_batch_fn( |
| 243 | + exec_q, uplo_val, trans, m, n, matrixA.get_data(), lda, stride_a, |
| 244 | + matrixB.get_data(), ldb, stride_b, batch_size, depends); |
| 245 | + |
| 246 | + sycl::event args_ev = dpnp::utils::keep_args_alive( |
| 247 | + exec_q, {matrixA, matrixB}, {trsm_batch_ev}); |
| 248 | + |
| 249 | + return std::make_pair(args_ev, trsm_batch_ev); |
| 250 | +} |
| 251 | + |
| 252 | +template <typename fnT, typename T> |
| 253 | +struct TrsmBatchContigFactory |
| 254 | +{ |
| 255 | + fnT get() |
| 256 | + { |
| 257 | + if constexpr (types::TrsmBatchTypePairSupportFactory<T>::is_defined) { |
| 258 | + return trsm_batch_impl<T>; |
| 259 | + } |
| 260 | + else { |
| 261 | + return nullptr; |
| 262 | + } |
| 263 | + } |
| 264 | +}; |
| 265 | + |
| 266 | +void init_trsm_batch_dispatch_vector(void) |
| 267 | +{ |
| 268 | + init_dispatch_vector<trsm_batch_impl_fn_ptr_t, TrsmBatchContigFactory>( |
| 269 | + trsm_batch_dispatch_vector); |
| 270 | +} |
| 271 | +} // namespace dpnp::extensions::blas |
0 commit comments