dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-26 20:55:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 22

changed to initialization order of the avx512 version of llama.cpp, now tries before avx2

Browse source 
Signed-off-by: Bas Hulsken <bhulsken@hotmail.com>
This commit is contained in:
Bas Hulsken 2025-02-16 15:08:26 +01:00
parent 27617a1b06
commit bfc6a0d200
1 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

14
pkg/model/initializers.go
View file

@ -262,18 +262,18 @@ func selectGRPCProcessByHostCapabilities(backend, assetDir string, f16 bool) str
}
// IF we find any optimized binary, we use that
if xsysinfo.HasCPUCaps(cpuid.AVX2) {
p := backendPath(assetDir, LLamaCPPAVX2)
if _, err := os.Stat(p); err == nil {
log.Info().Msgf("[%s] attempting to load with AVX2 variant", backend)
selectedProcess = p
}
} else if xsysinfo.HasCPUCaps(cpuid.AVX512F) {
if xsysinfo.HasCPUCaps(cpuid.AVX512F) {
p := backendPath(assetDir, LLamaCPPAVX512)
if _, err := os.Stat(p); err == nil {
log.Info().Msgf("[%s] attempting to load with AVX512 variant", backend)
selectedProcess = p
}
} else if xsysinfo.HasCPUCaps(cpuid.AVX2) {
p := backendPath(assetDir, LLamaCPPAVX2)
if _, err := os.Stat(p); err == nil {
log.Info().Msgf("[%s] attempting to load with AVX2 variant", backend)
selectedProcess = p
}
} else if xsysinfo.HasCPUCaps(cpuid.AVX) {
p := backendPath(assetDir, LLamaCPPAVX)
if _, err := os.Stat(p); err == nil {