|
3 | 3 | #include "ggml-impl.h" |
4 | 4 | #include <algorithm> |
5 | 5 | #include <cstring> |
| 6 | +#include <string> |
6 | 7 | #include <vector> |
7 | 8 |
|
8 | 9 | #ifdef _WIN32 |
|
11 | 12 | # define NOMINMAX |
12 | 13 | # endif |
13 | 14 | # include <windows.h> |
| 15 | +#elif defined(__APPLE__) |
| 16 | +# include <mach-o/dyld.h> |
| 17 | +# include <dlfcn.h> |
14 | 18 | #else |
15 | 19 | # include <dlfcn.h> |
| 20 | +# include <unistd.h> |
16 | 21 | #endif |
17 | 22 |
|
18 | | - |
19 | 23 | // Backend registry |
20 | 24 | #ifdef GGML_USE_CPU |
21 | 25 | #include "ggml-cpu.h" |
@@ -128,10 +132,59 @@ struct ggml_backend_registry { |
128 | 132 | devices.push_back(device); |
129 | 133 | } |
130 | 134 |
|
131 | | - void unload_backend(ggml_backend_reg_t reg, bool silent) { |
132 | | - if (!silent) { |
133 | | - GGML_LOG_INFO("%s: unloading %s backend\n", __func__, ggml_backend_reg_name(reg)); |
| 135 | + ggml_backend_reg_t load_backend(const char * path, bool silent) { |
| 136 | +#ifdef _WIN32 |
| 137 | + HMODULE handle = LoadLibraryA(path); |
| 138 | + if (!handle) { |
| 139 | + if (!silent) { |
| 140 | + GGML_LOG_ERROR("%s: failed to load %s: %lu\n", __func__, path, GetLastError()); |
| 141 | + } |
| 142 | + return nullptr; |
| 143 | + } |
| 144 | + ggml_backend_init_t backend_init = (ggml_backend_init_t) GetProcAddress(handle, "ggml_backend_init"); |
| 145 | + if (!backend_init) { |
| 146 | + if (!silent) { |
| 147 | + GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %lu\n", __func__, path, GetLastError()); |
| 148 | + } |
| 149 | + FreeLibrary(handle); |
| 150 | + return nullptr; |
| 151 | + } |
| 152 | +#else |
| 153 | + void * handle = dlopen(path, RTLD_NOW | RTLD_LOCAL); |
| 154 | + if (!handle) { |
| 155 | + if (!silent) { |
| 156 | + GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path, dlerror()); |
| 157 | + } |
| 158 | + return nullptr; |
| 159 | + } |
| 160 | + auto * backend_init = (ggml_backend_init_t) dlsym(handle, "ggml_backend_init"); |
| 161 | + if (!backend_init) { |
| 162 | + if (!silent) { |
| 163 | + GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %s\n", __func__, path, dlerror()); |
| 164 | + } |
| 165 | + dlclose(handle); |
| 166 | + return nullptr; |
| 167 | + } |
| 168 | +#endif |
| 169 | + ggml_backend_reg_t reg = backend_init(); |
| 170 | + if (!reg) { |
| 171 | + if (!silent) { |
| 172 | + GGML_LOG_ERROR("%s: failed to initialize backend from %s\n", __func__, path); |
| 173 | + } |
| 174 | + #ifdef _WIN32 |
| 175 | + FreeLibrary(handle); |
| 176 | + #else |
| 177 | + dlclose(handle); |
| 178 | + #endif |
| 179 | + return nullptr; |
134 | 180 | } |
| 181 | + |
| 182 | + GGML_LOG_INFO("%s: loaded %s backend from %s\n", __func__, ggml_backend_reg_name(reg), path); |
| 183 | + register_backend(reg, handle); |
| 184 | + return reg; |
| 185 | + } |
| 186 | + |
| 187 | + void unload_backend(ggml_backend_reg_t reg, bool silent) { |
135 | 188 | auto it = std::find_if(backends.begin(), backends.end(), |
136 | 189 | [reg](ggml_backend_reg_entry entry) { return entry.reg == reg; }); |
137 | 190 |
|
@@ -258,75 +311,92 @@ ggml_backend_t ggml_backend_init_best(void) { |
258 | 311 | return ggml_backend_dev_init(dev, nullptr); |
259 | 312 | } |
260 | 313 |
|
261 | | -typedef ggml_backend_reg_t (*ggml_backend_init_t)(void); |
262 | | - |
| 314 | +// Dynamic loading |
263 | 315 | ggml_backend_reg_t ggml_backend_load(const char * path) { |
264 | | -#ifdef _WIN32 |
265 | | - HMODULE handle = LoadLibraryA(path); |
266 | | - if (!handle) { |
267 | | - GGML_LOG_ERROR("%s: failed to load %s: %lu\n", __func__, path, GetLastError()); |
268 | | - return nullptr; |
269 | | - } |
270 | | - ggml_backend_init_t backend_init = (ggml_backend_init_t) GetProcAddress(handle, "ggml_backend_init"); |
271 | | - if (!backend_init) { |
272 | | - GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %lu\n", __func__, path, GetLastError()); |
273 | | - FreeLibrary(handle); |
274 | | - return nullptr; |
275 | | - } |
276 | | -#else |
277 | | - void * handle = dlopen(path, RTLD_NOW | RTLD_LOCAL); |
278 | | - if (!handle) { |
279 | | - GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path, dlerror()); |
280 | | - return nullptr; |
281 | | - } |
282 | | - auto * backend_init = (ggml_backend_init_t) dlsym(handle, "ggml_backend_init"); |
283 | | - if (!backend_init) { |
284 | | - GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %s\n", __func__, path, dlerror()); |
285 | | - dlclose(handle); |
286 | | - return nullptr; |
287 | | - } |
288 | | -#endif |
289 | | - ggml_backend_reg_t reg = backend_init(); |
290 | | - if (!reg) { |
291 | | - GGML_LOG_ERROR("%s: failed to initialize backend from %s\n", __func__, path); |
292 | | -#ifdef _WIN32 |
293 | | - FreeLibrary(handle); |
294 | | -#else |
295 | | - dlclose(handle); |
296 | | -#endif |
297 | | - return nullptr; |
298 | | - } |
299 | | - |
300 | | - GGML_LOG_DEBUG("%s: loaded %s backend from %s\n", __func__, ggml_backend_reg_name(reg), path); |
301 | | - get_reg().register_backend(reg, handle); |
302 | | - return reg; |
| 316 | + return get_reg().load_backend(path, false); |
303 | 317 | } |
304 | 318 |
|
305 | 319 | void ggml_backend_unload(ggml_backend_reg_t reg) { |
306 | 320 | get_reg().unload_backend(reg, true); |
307 | 321 | } |
308 | 322 |
|
309 | 323 | void ggml_backend_load_all() { |
| 324 | + std::vector<std::string> search_prefix; |
| 325 | + |
| 326 | + // add the executable directory to the search path |
| 327 | + // FIXME: this is convenient for development, but it should probably be disabled in production |
| 328 | + |
| 329 | +#if defined(__APPLE__) |
| 330 | + // get executable path |
| 331 | + std::vector<char> path; |
| 332 | + uint32_t size; |
| 333 | + while (true) { |
| 334 | + size = path.size(); |
| 335 | + if (_NSGetExecutablePath(path.data(), &size) == 0) { |
| 336 | + break; |
| 337 | + } |
| 338 | + path.resize(size); |
| 339 | + } |
| 340 | + std::string base_path(path.data(), size); |
| 341 | + // remove executable name |
| 342 | + auto last_slash = base_path.find_last_of('/'); |
| 343 | + if (last_slash != std::string::npos) { |
| 344 | + base_path = base_path.substr(0, last_slash); |
| 345 | + } |
| 346 | + search_prefix.push_back(base_path + "/"); |
| 347 | +#elif defined(__linux__) |
| 348 | + std::string base_path = "."; |
| 349 | + std::vector<char> path(1024); |
| 350 | + while (true) { |
| 351 | + // get executable path |
| 352 | + ssize_t len = readlink("/proc/self/exe", path.data(), path.size()); |
| 353 | + if (len == -1) { |
| 354 | + break; |
| 355 | + } |
| 356 | + if (len < (ssize_t) path.size()) { |
| 357 | + base_path = std::string(path.data(), len); |
| 358 | + // remove executable name |
| 359 | + auto last_slash = base_path.find_last_of('/'); |
| 360 | + if (last_slash != std::string::npos) { |
| 361 | + base_path = base_path.substr(0, last_slash); |
| 362 | + } |
| 363 | + break; |
| 364 | + } |
| 365 | + path.resize(path.size() * 2); |
| 366 | + } |
| 367 | + |
| 368 | + search_prefix.push_back(base_path + "/"); |
| 369 | +#endif |
| 370 | + |
| 371 | + auto & reg = get_reg(); |
| 372 | + |
| 373 | + auto try_load = [&](const std::string & name) { |
| 374 | + std::string os_name; |
310 | 375 | #ifdef _WIN32 |
311 | | - #define GGML_BACKEND_PATH(backend) "ggml-" backend ".dll" |
312 | | -#elif defined(__APPLE__) |
313 | | - // path is hardcoded to the cmake build directory for now |
314 | | - // FIXME: should also search default system paths |
315 | | - #define GGML_BACKEND_PATH(backend) "build/ggml/src/ggml-" backend "/libggml-" backend ".dylib" |
| 376 | + os_name = "ggml-" + name + ".dll"; |
316 | 377 | #else |
317 | | - #define GGML_BACKEND_PATH(backend) "build/ggml/src/ggml-" backend "/libggml-" backend ".so" |
| 378 | + os_name = "libggml-" + name + ".so"; |
318 | 379 | #endif |
319 | | - |
320 | | - ggml_backend_load(GGML_BACKEND_PATH("amx")); |
321 | | - ggml_backend_load(GGML_BACKEND_PATH("blas")); |
322 | | - ggml_backend_load(GGML_BACKEND_PATH("cann")); |
323 | | - ggml_backend_load(GGML_BACKEND_PATH("cuda")); |
324 | | - ggml_backend_load(GGML_BACKEND_PATH("hip")); |
325 | | - ggml_backend_load(GGML_BACKEND_PATH("kompute")); |
326 | | - ggml_backend_load(GGML_BACKEND_PATH("metal")); |
327 | | - ggml_backend_load(GGML_BACKEND_PATH("rpc")); |
328 | | - ggml_backend_load(GGML_BACKEND_PATH("sycl")); |
329 | | - ggml_backend_load(GGML_BACKEND_PATH("vulkan")); |
330 | | - ggml_backend_load(GGML_BACKEND_PATH("musa")); |
331 | | - ggml_backend_load(GGML_BACKEND_PATH("cpu")); |
| 380 | + if (reg.load_backend(os_name.c_str(), true)) { |
| 381 | + return; |
| 382 | + } |
| 383 | + for (const auto & prefix : search_prefix) { |
| 384 | + if (reg.load_backend((prefix + os_name).c_str(), true)) { |
| 385 | + return; |
| 386 | + } |
| 387 | + } |
| 388 | + }; |
| 389 | + |
| 390 | + try_load("amx"); |
| 391 | + try_load("blas"); |
| 392 | + try_load("cann"); |
| 393 | + try_load("cuda"); |
| 394 | + try_load("hip"); |
| 395 | + try_load("kompute"); |
| 396 | + try_load("metal"); |
| 397 | + try_load("rpc"); |
| 398 | + try_load("sycl"); |
| 399 | + try_load("vulkan"); |
| 400 | + try_load("musa"); |
| 401 | + try_load("cpu"); |
332 | 402 | } |
0 commit comments