diff --git a/core/gallery/backends.go b/core/gallery/backends.go index b198a7f801c9..88d49dcbebfb 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -1,3 +1,5 @@ +// Package gallery provides installation and registration utilities for LocalAI backends, +// including meta-backend resolution based on system capabilities. package gallery import ( @@ -5,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/mudler/LocalAI/core/config" @@ -20,6 +23,12 @@ const ( runFile = "run.sh" ) +// backendCandidate represents an installed concrete backend option for a given alias +type backendCandidate struct { + name string + runFile string +} + // readBackendMetadata reads the metadata JSON file for a backend func readBackendMetadata(backendPath string) (*BackendMetadata, error) { metadataPath := filepath.Join(backendPath, metadataFile) @@ -58,7 +67,7 @@ func writeBackendMetadata(backendPath string, metadata *BackendMetadata) error { return nil } -// Installs a model from the gallery +// InstallBackendFromGallery installs a backend from the gallery. func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.SystemState, modelLoader *model.ModelLoader, name string, downloadStatus func(string, string, string, float64), force bool) error { if !force { // check if we already have the backend installed @@ -282,23 +291,18 @@ func (b SystemBackends) GetAll() []SystemBackend { } func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) { - potentialBackends, err := os.ReadDir(systemState.Backend.BackendsPath) - if err != nil { - return nil, err - } - + // Gather backends from system and user paths, then resolve alias conflicts by capability. backends := make(SystemBackends) - systemBackends, err := os.ReadDir(systemState.Backend.BackendsSystemPath) - if err == nil { - // system backends are special, they are provided by the system and not managed by LocalAI + // System-provided backends + if systemBackends, err := os.ReadDir(systemState.Backend.BackendsSystemPath); err == nil { for _, systemBackend := range systemBackends { if systemBackend.IsDir() { - systemBackendRunFile := filepath.Join(systemState.Backend.BackendsSystemPath, systemBackend.Name(), runFile) - if _, err := os.Stat(systemBackendRunFile); err == nil { + run := filepath.Join(systemState.Backend.BackendsSystemPath, systemBackend.Name(), runFile) + if _, err := os.Stat(run); err == nil { backends[systemBackend.Name()] = SystemBackend{ Name: systemBackend.Name(), - RunFile: filepath.Join(systemState.Backend.BackendsSystemPath, systemBackend.Name(), runFile), + RunFile: run, IsMeta: false, IsSystem: true, Metadata: nil, @@ -307,64 +311,104 @@ func ListSystemBackends(systemState *system.SystemState) (SystemBackends, error) } } } else { - log.Warn().Err(err).Msg("Failed to read system backends, but that's ok, we will just use the backends managed by LocalAI") + log.Warn().Err(err).Msg("Failed to read system backends, proceeding with user-managed backends") } - for _, potentialBackend := range potentialBackends { - if potentialBackend.IsDir() { - potentialBackendRunFile := filepath.Join(systemState.Backend.BackendsPath, potentialBackend.Name(), runFile) + // User-managed backends and alias collection + entries, err := os.ReadDir(systemState.Backend.BackendsPath) + if err != nil { + return nil, err + } - var metadata *BackendMetadata + aliasGroups := make(map[string][]backendCandidate) + metaMap := make(map[string]*BackendMetadata) - // If metadata file does not exist, we just use the directory name - // and we do not fill the other metadata (such as potential backend Aliases) - metadataFilePath := filepath.Join(systemState.Backend.BackendsPath, potentialBackend.Name(), metadataFile) - if _, err := os.Stat(metadataFilePath); os.IsNotExist(err) { - metadata = &BackendMetadata{ - Name: potentialBackend.Name(), - } + for _, e := range entries { + if !e.IsDir() { + continue + } + dir := e.Name() + run := filepath.Join(systemState.Backend.BackendsPath, dir, runFile) + + var metadata *BackendMetadata + metadataPath := filepath.Join(systemState.Backend.BackendsPath, dir, metadataFile) + if _, err := os.Stat(metadataPath); os.IsNotExist(err) { + metadata = &BackendMetadata{Name: dir} + } else { + m, rerr := readBackendMetadata(filepath.Join(systemState.Backend.BackendsPath, dir)) + if rerr != nil { + return nil, rerr + } + if m == nil { + metadata = &BackendMetadata{Name: dir} } else { - // Check for alias in metadata - metadata, err = readBackendMetadata(filepath.Join(systemState.Backend.BackendsPath, potentialBackend.Name())) - if err != nil { - return nil, err - } + metadata = m } + } - if !backends.Exists(potentialBackend.Name()) { - // We don't want to override aliases if already set, and if we are meta backend - if _, err := os.Stat(potentialBackendRunFile); err == nil { - backends[potentialBackend.Name()] = SystemBackend{ - Name: potentialBackend.Name(), - RunFile: potentialBackendRunFile, - IsMeta: false, - Metadata: metadata, - } - } + metaMap[dir] = metadata + + // Concrete backend entry + if _, err := os.Stat(run); err == nil { + backends[dir] = SystemBackend{ + Name: dir, + RunFile: run, + IsMeta: false, + Metadata: metadata, } + } - if metadata == nil { - continue + // Alias candidates + if metadata.Alias != "" { + aliasGroups[metadata.Alias] = append(aliasGroups[metadata.Alias], backendCandidate{name: dir, runFile: run}) + } + + // Meta backends indirection + if metadata.MetaBackendFor != "" { + backends[metadata.Name] = SystemBackend{ + Name: metadata.Name, + RunFile: filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor, runFile), + IsMeta: true, + Metadata: metadata, } + } + } - if metadata.Alias != "" { - backends[metadata.Alias] = SystemBackend{ - Name: metadata.Alias, - RunFile: potentialBackendRunFile, - IsMeta: false, - Metadata: metadata, + // Resolve aliases using system capability preferences + tokens := systemState.BackendPreferenceTokens() + for alias, cands := range aliasGroups { + chosen := backendCandidate{} + // Try preference tokens + for _, t := range tokens { + for _, c := range cands { + if strings.Contains(strings.ToLower(c.name), t) && c.runFile != "" { + chosen = c + break } } - - if metadata.MetaBackendFor != "" { - backends[metadata.Name] = SystemBackend{ - Name: metadata.Name, - RunFile: filepath.Join(systemState.Backend.BackendsPath, metadata.MetaBackendFor, runFile), - IsMeta: true, - Metadata: metadata, + if chosen.runFile != "" { + break + } + } + // Fallback: first runnable + if chosen.runFile == "" { + for _, c := range cands { + if c.runFile != "" { + chosen = c + break } } } + if chosen.runFile == "" { + continue + } + md := metaMap[chosen.name] + backends[alias] = SystemBackend{ + Name: alias, + RunFile: chosen.runFile, + IsMeta: false, + Metadata: md, + } } return backends, nil diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index d9cec0841c48..26652caadd05 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -18,6 +18,73 @@ const ( testImage = "quay.io/mudler/tests:localai-backend-test" ) +var _ = Describe("Runtime capability-based backend selection", func() { + var tempDir string + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "gallery-caps-*") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + It("ListSystemBackends prefers optimal alias candidate", func() { + // Arrange two installed backends sharing the same alias + must := func(err error) { Expect(err).NotTo(HaveOccurred()) } + + cpuDir := filepath.Join(tempDir, "cpu-llama-cpp") + must(os.MkdirAll(cpuDir, 0o750)) + cpuMeta := &BackendMetadata{Alias: "llama-cpp", Name: "cpu-llama-cpp"} + b, _ := json.Marshal(cpuMeta) + must(os.WriteFile(filepath.Join(cpuDir, "metadata.json"), b, 0o644)) + must(os.WriteFile(filepath.Join(cpuDir, "run.sh"), []byte(""), 0o755)) + + cudaDir := filepath.Join(tempDir, "cuda12-llama-cpp") + must(os.MkdirAll(cudaDir, 0o750)) + cudaMeta := &BackendMetadata{Alias: "llama-cpp", Name: "cuda12-llama-cpp"} + b, _ = json.Marshal(cudaMeta) + must(os.WriteFile(filepath.Join(cudaDir, "metadata.json"), b, 0o644)) + must(os.WriteFile(filepath.Join(cudaDir, "run.sh"), []byte(""), 0o755)) + + // Default system: alias should point to CPU + sysDefault, err := system.GetSystemState( + system.WithBackendPath(tempDir), + ) + must(err) + sysDefault.GPUVendor = "" // force default selection + backs, err := ListSystemBackends(sysDefault) + must(err) + aliasBack, ok := backs.Get("llama-cpp") + Expect(ok).To(BeTrue()) + Expect(aliasBack.RunFile).To(Equal(filepath.Join(cpuDir, "run.sh"))) + // concrete entries remain + _, ok = backs.Get("cpu-llama-cpp") + Expect(ok).To(BeTrue()) + _, ok = backs.Get("cuda12-llama-cpp") + Expect(ok).To(BeTrue()) + + // NVIDIA system: alias should point to CUDA + // Force capability to nvidia to make the test deterministic on platforms like darwin/arm64 (which default to metal) + os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia") + defer os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") + + sysNvidia, err := system.GetSystemState( + system.WithBackendPath(tempDir), + ) + must(err) + sysNvidia.GPUVendor = "nvidia" + sysNvidia.VRAM = 8 * 1024 * 1024 * 1024 + backs, err = ListSystemBackends(sysNvidia) + must(err) + aliasBack, ok = backs.Get("llama-cpp") + Expect(ok).To(BeTrue()) + Expect(aliasBack.RunFile).To(Equal(filepath.Join(cudaDir, "run.sh"))) + }) +}) + var _ = Describe("Gallery Backends", func() { var ( tempDir string diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index 6102cf1efd60..0ea60dfc81fe 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -1,3 +1,5 @@ +// Package system provides system detection utilities, including GPU/vendor detection +// and capability classification used to select optimal backends at runtime. package system import ( @@ -116,3 +118,25 @@ func detectGPUVendor(gpus []*gpu.GraphicsCard) (string, error) { return "", nil } + +// BackendPreferenceTokens returns a list of substrings that represent the preferred +// backend implementation order for the current system capability. Callers can use +// these tokens to select the most appropriate concrete backend among multiple +// candidates sharing the same alias (e.g., "llama-cpp"). +func (s *SystemState) BackendPreferenceTokens() []string { + capStr := strings.ToLower(s.getSystemCapabilities()) + switch { + case strings.HasPrefix(capStr, nvidia): + return []string{"cuda", "vulkan", "cpu"} + case strings.HasPrefix(capStr, amd): + return []string{"rocm", "hip", "vulkan", "cpu"} + case strings.HasPrefix(capStr, intel): + return []string{"sycl", "intel", "cpu"} + case strings.HasPrefix(capStr, metal): + return []string{"metal", "cpu"} + case strings.HasPrefix(capStr, darwinX86): + return []string{"darwin-x86", "cpu"} + default: + return []string{"cpu"} + } +}