diff --git a/LLama.Unittest/ModelsParamsTests.cs b/LLama.Unittest/ModelsParamsTests.cs index 59cf70bf5..ff6980370 100644 --- a/LLama.Unittest/ModelsParamsTests.cs +++ b/LLama.Unittest/ModelsParamsTests.cs @@ -1,6 +1,8 @@ using LLama.Common; using System.Text.Json; using LLama.Abstractions; +using LLama.Exceptions; +using LLama.Extensions; namespace LLama.Unittest { @@ -20,6 +22,7 @@ public void SerializeRoundTripSystemTextJson() ContextSize = 42, GpuLayerCount = 111, TensorSplits = { [0] = 3 }, + Devices = { "Vulkan1", "CPU" }, MetadataOverrides = { new MetadataOverride("hello", true), @@ -46,6 +49,11 @@ public void SerializeRoundTripSystemTextJson() actual.TensorBufferOverrides = null!; expected.TensorBufferOverrides = null!; + // Same deal + Assert.True(expected.Devices.SequenceEqual(actual.Devices)); + actual.Devices = null!; + expected.Devices = null!; + // Check encoding is the same var b1 = expected.Encoding.GetBytes("Hello"); var b2 = actual.Encoding.GetBytes("Hello"); @@ -53,5 +61,36 @@ public void SerializeRoundTripSystemTextJson() Assert.Equal(expected, actual); } + + [Fact] + public void UnknownDeviceThrows() + { + var @params = new ModelParams("abc/123") + { + Devices = { "NoSuchDevice" }, + }; + + var ex = Assert.Throws(() => @params.ToLlamaModelParams(out _)); + + Assert.Equal("NoSuchDevice", ex.RequestedDevice); + Assert.Contains("CPU", ex.AvailableDevices); + Assert.Contains("NoSuchDevice", ex.Message); + Assert.Contains("CPU", ex.Message); + } + + [Fact] + public unsafe void DeviceNameMatchIsCaseInsensitive() + { + var @params = new ModelParams("abc/123") + { + Devices = { "cpu" }, + }; + + using var disposer = @params.ToLlamaModelParams(out var result); + + Assert.True(result.devices != null); + Assert.NotEqual(IntPtr.Zero, result.devices[0]); + Assert.Equal(IntPtr.Zero, result.devices[1]); + } } } diff --git a/LLama.Web/Common/ModelOptions.cs b/LLama.Web/Common/ModelOptions.cs index 5f0bb64d2..593e27078 100644 --- a/LLama.Web/Common/ModelOptions.cs +++ b/LLama.Web/Common/ModelOptions.cs @@ -31,6 +31,9 @@ public class ModelOptions /// public List TensorBufferOverrides { get; set; } = new(); + /// + public List Devices { get; set; } = new(); + /// public int GpuLayerCount { get; set; } = 20; diff --git a/LLama/Abstractions/IModelParams.cs b/LLama/Abstractions/IModelParams.cs index 6cce4372c..cc2145cf4 100644 --- a/LLama/Abstractions/IModelParams.cs +++ b/LLama/Abstractions/IModelParams.cs @@ -44,6 +44,17 @@ public interface IModelParams /// List TensorBufferOverrides { get; } + /// + /// Names of the backend devices the model may use, in priority order, e.g. "Vulkan1" or "CUDA0" (see ). + /// Equivalent to --device on the llama.cpp command line or devices in llama_model_params. + /// + /// + /// When empty, llama.cpp picks the devices itself: all discrete GPUs, or the first integrated GPU when there is no discrete GPU. + /// Setting this list explicitly bypasses that selection, which is the only way to run on an integrated GPU in a machine that also has a discrete GPU. + /// is an index into this list when it is non-empty. Names are matched case insensitively; a name that does not match any available device throws when the model is loaded. + /// + List Devices { get; } + /// /// Number of layers to run in VRAM / GPU memory (n_gpu_layers) /// diff --git a/LLama/Common/ModelParams.cs b/LLama/Common/ModelParams.cs index 55e7c4300..0c03757de 100644 --- a/LLama/Common/ModelParams.cs +++ b/LLama/Common/ModelParams.cs @@ -28,6 +28,9 @@ public record ModelParams /// public List TensorBufferOverrides { get; set; } = new(); + /// + public List Devices { get; set; } = new(); + /// public int GpuLayerCount { get; set; } = 20; diff --git a/LLama/Exceptions/UnknownDeviceException.cs b/LLama/Exceptions/UnknownDeviceException.cs new file mode 100644 index 000000000..b8dd6784d --- /dev/null +++ b/LLama/Exceptions/UnknownDeviceException.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; + +namespace LLama.Exceptions; + +/// +/// Thrown when a device name in does not match any available ggml backend device +/// +public class UnknownDeviceException + : Exception +{ + /// + /// The device name which was requested but could not be found + /// + public string RequestedDevice { get; } + + /// + /// Names of all devices available on this machine, as returned by + /// + public IReadOnlyList AvailableDevices { get; } + + /// + /// Create a new UnknownDeviceException + /// + /// The device name which was requested but could not be found + /// Names of all devices available on this machine + public UnknownDeviceException(string requestedDevice, IReadOnlyList availableDevices) + : base($"Unknown device '{requestedDevice}'. Available devices: {string.Join(", ", availableDevices)}") + { + RequestedDevice = requestedDevice; + AvailableDevices = availableDevices; + } +} diff --git a/LLama/Extensions/IModelParamsExtensions.cs b/LLama/Extensions/IModelParamsExtensions.cs index 5ed152c7e..f5753b389 100644 --- a/LLama/Extensions/IModelParamsExtensions.cs +++ b/LLama/Extensions/IModelParamsExtensions.cs @@ -2,6 +2,7 @@ using System; using System.Text; using LLama.Abstractions; +using LLama.Exceptions; using LLama.Native; using System.Collections.Generic; @@ -20,6 +21,7 @@ public static class IModelParamsExtensions /// /// /// + /// Thrown if a name in does not match any available device public static IDisposable ToLlamaModelParams(this IModelParams @params, out LLamaModelParams result) { var supportsMmap = NativeApi.llama_supports_mmap(); @@ -56,6 +58,12 @@ public static IDisposable ToLlamaModelParams(this IModelParams @params, out LLam result.tensor_buft_overrides = ConvertOverrides(@params.TensorBufferOverrides, disposer); } + // Add device list + unsafe + { + result.devices = ConvertDevices(@params.Devices, disposer); + } + // Add metadata overrides if (@params.MetadataOverrides.Count == 0) { @@ -129,6 +137,62 @@ private static IReadOnlyDictionary GetAvailableBufferTypes() return result; } + private static unsafe IntPtr* ConvertDevices(List deviceNames, GroupDisposable disposer) + { + // Early out if no devices were requested (llama.cpp will choose its own) + if (deviceNames.Count == 0) + return null; + + // Map device name -> ggml_backend_dev_t, keeping the names in native order for error messages + var devicesByName = new Dictionary(); + var availableNames = new List(); + var deviceCount = NativeApi.ggml_backend_dev_count(); + for (nuint i = 0; i < deviceCount; i++) + { + var device = NativeApi.ggml_backend_dev_get(i); + if (device == IntPtr.Zero) + continue; + + var name = NativeApi.ggml_backend_dev_name(device).PtrToString(); + if (string.IsNullOrEmpty(name)) + continue; + + devicesByName[name!] = device; + availableNames.Add(name!); + } + + // Resolve the requested names in order. One extra slot for the null terminator. + var devicesArray = new IntPtr[deviceNames.Count + 1]; + for (var i = 0; i < deviceNames.Count; i++) + devicesArray[i] = ResolveDevice(deviceNames[i], devicesByName, availableNames); + + // Pin the array so it can be passed to native code + var pin = devicesArray.AsMemory().Pin(); + disposer.Add(pin); + return (IntPtr*)pin.Pointer; + } + + /// + /// Find the device with the given name. Tries an exact match first, then a case insensitive match. + /// + /// Thrown if no device matches the name + private static IntPtr ResolveDevice(string name, Dictionary devicesByName, List availableNames) + { + if (!string.IsNullOrEmpty(name)) + { + if (devicesByName.TryGetValue(name, out var device)) + return device; + + foreach (var pair in devicesByName) + { + if (string.Equals(pair.Key, name, StringComparison.OrdinalIgnoreCase)) + return pair.Value; + } + } + + throw new UnknownDeviceException(name ?? "", availableNames); + } + private static unsafe LLamaModelTensorBufferOverride* ConvertOverrides(List overrides, GroupDisposable disposer) { // Early out if there are no overrides diff --git a/LLama/Native/LLamaModelParams.cs b/LLama/Native/LLamaModelParams.cs index ce6ab617a..796ba60e1 100644 --- a/LLama/Native/LLamaModelParams.cs +++ b/LLama/Native/LLamaModelParams.cs @@ -9,10 +9,10 @@ namespace LLama.Native public unsafe struct LLamaModelParams { /// - /// NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) - /// todo: add support for llama_model_params.devices + /// NULL-terminated list of devices to use for offloading (if NULL, all available devices are used). + /// Each element is a ggml_backend_dev_t as returned by . /// - private IntPtr devices; + public IntPtr* devices; /// /// NULL-terminated list of buffer types to use for tensors that match a pattern diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs index 03d436447..b90ff93dc 100644 --- a/LLama/Native/Load/NativeLibraryUtils.cs +++ b/LLama/Native/Load/NativeLibraryUtils.cs @@ -8,6 +8,18 @@ namespace LLama.Native { internal static class NativeLibraryUtils { + /// + /// Handle of the ggml library loaded as a dependency by , or IntPtr.Zero if it has not been loaded. + /// Used by the DllImport resolver so that P/Invokes into ggml reuse this library instead of relying on the system search path. + /// + internal static IntPtr LoadedGgmlHandle; + + /// + /// Handle of the ggml-base library loaded as a dependency by , or IntPtr.Zero if it has not been loaded. + /// Used by the DllImport resolver so that P/Invokes into ggml-base reuse this library instead of relying on the system search path. + /// + internal static IntPtr LoadedGgmlBaseHandle; + /// /// Try to load libllama/mtmd, using CPU feature detection to try and load a more specialised DLL if possible /// @@ -64,7 +76,8 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLib var dependencyPaths = new List(); // We should always load ggml-base from the current runtime directory - dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-base{ext}")); + var ggmlBasePath = Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-base{ext}"); + dependencyPaths.Add(ggmlBasePath); // If the library has metadata, we can check if we need to load additional dependencies if (library.Metadata != null) @@ -114,13 +127,20 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLib } // And finally, we can add ggml - dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml{ext}")); + var ggmlPath = Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml{ext}"); + dependencyPaths.Add(ggmlPath); // Now, we will loop through our dependencyPaths and try to load them one by one foreach (var dependencyPath in dependencyPaths) { // Try to load the dependency var dependencyResult = TryLoad(dependencyPath, description.SearchDirectories, config.LogCallback); + + // Keep the ggml/ggml-base handles so the DllImport resolver can hand them out for P/Invokes into those libraries + if (dependencyPath == ggmlBasePath) + LoadedGgmlBaseHandle = dependencyResult; + else if (dependencyPath == ggmlPath) + LoadedGgmlHandle = dependencyResult; // If we successfully loaded the library, log it if (dependencyResult != IntPtr.Zero) diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 57bb2d146..2a0052b7d 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -72,6 +72,17 @@ private static void SetDllImportResolver() return _loadedLlamaHandle; } + if (name == ggmlLibraryName || name == ggmlBaseLibraryName) + { + // ggml and ggml-base are loaded by full path as dependencies of llama, so the default runtime + // resolution cannot be relied on to find them by name (it fails on macOS). Make sure llama (and + // therefore its dependencies) has been loaded, then return the dependency handle. + if (_loadedLlamaHandle == IntPtr.Zero) + _loadedLlamaHandle = NativeLibraryUtils.TryLoadLibrary(NativeLibraryConfig.LLama, out _loadedLLamaLibrary); + + return name == ggmlLibraryName ? NativeLibraryUtils.LoadedGgmlHandle : NativeLibraryUtils.LoadedGgmlBaseHandle; + } + if (name == "mtmd") { // If we've already loaded Mtmd return the handle that was loaded last time. diff --git a/LLama/Native/NativeApi.cs b/LLama/Native/NativeApi.cs index 2e06f83ee..26b3c9dcd 100644 --- a/LLama/Native/NativeApi.cs +++ b/LLama/Native/NativeApi.cs @@ -448,6 +448,14 @@ public static string llama_split_prefix(string splitPath, int splitNo, int split [DllImport(ggmlLibraryName, CallingConvention = CallingConvention.Cdecl)] public static extern IntPtr ggml_backend_dev_get(nuint i); + /// + /// Get the name of a backend device (e.g. "CPU", "Vulkan0", "CUDA1") + /// + /// Backend device pointer + /// Pointer to a null terminated UTF-8 string, owned by the device + [DllImport(ggmlBaseLibraryName, CallingConvention = CallingConvention.Cdecl)] + public static extern IntPtr ggml_backend_dev_name(IntPtr dev); + /// /// Get the buffer type for a backend device ///