1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
{
lib,
stdenv,
gcc13Stdenv,
buildPythonPackage,
fetchFromGitHub,
fetchpatch,
# nativeBuildInputs
cmake,
ninja,
autoAddDriverRunpath,
# build-system
pathspec,
pyproject-metadata,
scikit-build-core,
# dependencies
diskcache,
jinja2,
numpy,
typing-extensions,
# tests
scipy,
huggingface-hub,
# passthru
gitUpdater,
pytestCheckHook,
llama-cpp-python,
config,
cudaSupport ? config.cudaSupport,
cudaPackages ? { },
}:
let
stdenvTarget = if cudaSupport then gcc13Stdenv else stdenv;
in
buildPythonPackage.override { stdenv = stdenvTarget; } rec {
pname = "llama-cpp-python";
version = "0.3.16";
pyproject = true;
src = fetchFromGitHub {
owner = "abetlen";
repo = "llama-cpp-python";
tag = "v${version}";
hash = "sha256-EUDtCv86J4bznsTqNsdgj1IYkAu83cf+RydFTUb2NEE=";
fetchSubmodules = true;
};
# src = /home/gaetan/llama-cpp-python;
patches = [
# Fix test failure on a machine with no metal devices (e.g. nix-community darwin builder)
# https://github.com/ggml-org/llama.cpp/pull/15531
(fetchpatch {
url = "https://github.com/ggml-org/llama.cpp/pull/15531/commits/63a83ffefe4d478ebadff89300a0a3c5d660f56a.patch";
stripLen = 1;
extraPrefix = "vendor/llama.cpp/";
hash = "sha256-9LGnzviBgYYOOww8lhiLXf7xgd/EtxRXGQMredOO4qM=";
})
];
dontUseCmakeConfigure = true;
cmakeFlags = [
# Set GGML_NATIVE=off. Otherwise, cmake attempts to build with
# -march=native* which is either a no-op (if cc-wrapper is able to ignore
# it), or an attempt to build a non-reproducible binary.
#
# This issue was spotted when cmake rules appended feature modifiers to
# -mcpu, breaking linux build as follows:
#
# cc1: error: unknown value ‘native+nodotprod+noi8mm+nosve’ for ‘-mcpu’
(lib.cmakeBool "GGML_NATIVE" false)
(lib.cmakeFeature "GGML_BUILD_NUMBER" "1")
]
++ lib.optionals cudaSupport [
(lib.cmakeBool "GGML_CUDA" true)
(lib.cmakeFeature "CUDAToolkit_ROOT" "${lib.getDev cudaPackages.cuda_nvcc}")
(lib.cmakeFeature "CMAKE_CUDA_COMPILER" "${lib.getExe cudaPackages.cuda_nvcc}")
];
enableParallelBuilding = true;
nativeBuildInputs = [
cmake
ninja
]
++ lib.optionals cudaSupport [
autoAddDriverRunpath
];
build-system = [
pathspec
pyproject-metadata
scikit-build-core
];
buildInputs = lib.optionals cudaSupport (
with cudaPackages;
[
cuda_cudart # cuda_runtime.h
cuda_cccl # <thrust/*>
libcublas # cublas_v2.h
]
);
dependencies = [
diskcache
jinja2
numpy
typing-extensions
];
nativeCheckInputs = [
pytestCheckHook
scipy
huggingface-hub
];
disabledTests = [
# tries to download model from huggingface-hub
"test_real_model"
"test_real_llama"
];
pythonImportsCheck = lib.optionals (!cudaSupport) [
# `libllama.so` is loaded at import time, and failing when cudaSupport is enabled as the cuda
# driver is missing in the sandbox:
# RuntimeError: Failed to load shared library '/nix/store/...-python3.13-llama-cpp-python-0.3.16/lib/python3.13/site-packages/llama_cpp/lib/libllama.so':
# libcuda.so.1: cannot open shared object file: No such file or directory
"llama_cpp"
];
passthru = {
updateScript = gitUpdater {
rev-prefix = "v";
allowedVersions = "^[.0-9]+$";
};
tests = lib.optionalAttrs stdenvTarget.hostPlatform.isLinux {
withCuda = llama-cpp-python.override {
cudaSupport = true;
};
};
};
meta = {
description = "Python bindings for llama.cpp";
homepage = "https://github.com/abetlen/llama-cpp-python";
changelog = "https://github.com/abetlen/llama-cpp-python/blob/v${version}/CHANGELOG.md";
license = lib.licenses.mit;
maintainers = with lib.maintainers; [
booxter
kirillrdy
];
};
}
|
