feat(diffusers): add img2img and clip_skip, support more kernels schedulers (#906)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

api/backend/image.go

@@ -9,7 +9,7 @@ import (
 	model "github.com/go-skynet/LocalAI/pkg/model"
 )
 
-func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, dst string, loader *model.ModelLoader, c config.Config, o *options.Option) (func() error, error) {
+func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, c config.Config, o *options.Option) (func() error, error) {
 
 	opts := []model.Option{
 		model.WithBackendString(c.Backend),
@@ -22,6 +22,10 @@ func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negat
 			SchedulerType: c.Diffusers.SchedulerType,
 			PipelineType:  c.Diffusers.PipelineType,
 			CFGScale:      c.Diffusers.CFGScale,
+			IMG2IMG:       c.Diffusers.IMG2IMG,
+			CLIPModel:     c.Diffusers.ClipModel,
+			CLIPSubfolder: c.Diffusers.ClipSubFolder,
+			CLIPSkip:      int32(c.Diffusers.ClipSkip),
 		}),
 	}
 
@@ -53,9 +57,11 @@ func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negat
 				Mode:             int32(mode),
 				Step:             int32(step),
 				Seed:             int32(seed),
+				CLIPSkip:         int32(c.Diffusers.ClipSkip),
 				PositivePrompt:   positive_prompt,
 				NegativePrompt:   negative_prompt,
 				Dst:              dst,
+				Src:              src,
 				EnableParameters: c.Diffusers.EnableParameters,
 			})
 		return err

api/config/config.go

@@ -55,6 +55,10 @@ type Diffusers struct {
 	CUDA             bool    `yaml:"cuda"`
 	EnableParameters string  `yaml:"enable_parameters"` // A list of comma separated parameters to specify
 	CFGScale         float32 `yaml:"cfg_scale"`         // Classifier-Free Guidance Scale
+	IMG2IMG          bool    `yaml:"img2img"`           // Image to Image Diffuser
+	ClipSkip         int     `yaml:"clip_skip"`         // Skip every N frames
+	ClipModel        string  `yaml:"clip_model"`        // Clip model to use
+	ClipSubFolder    string  `yaml:"clip_subfolder"`    // Subfolder to use for clip model
 }
 
 type LLMConfig struct {

api/config/prediction.go

@@ -41,4 +41,7 @@ type PredictionOptions struct {
 	NegativePromptScale float32 `json:"negative_prompt_scale" yaml:"negative_prompt_scale"`
 	// AutoGPTQ
 	UseFastTokenizer bool `json:"use_fast_tokenizer" yaml:"use_fast_tokenizer"`
+
+	// Diffusers
+	ClipSkip int `json:"clip_skip" yaml:"clip_skip"`
 }

api/openai/image.go

@@ -4,7 +4,9 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -49,6 +51,37 @@ func ImageEndpoint(cm *config.ConfigLoader, o *options.Option) func(c *fiber.Ctx
 			return fmt.Errorf("failed reading parameters from request:%w", err)
 		}
 
+		src := ""
+		// retrieve the file data from the request
+		file, err := c.FormFile("src")
+		if err == nil {
+
+			f, err := file.Open()
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+
+			dir, err := os.MkdirTemp("", "img2img")
+
+			if err != nil {
+				return err
+			}
+			defer os.RemoveAll(dir)
+
+			dst := filepath.Join(dir, path.Base(file.Filename))
+			dstFile, err := os.Create(dst)
+			if err != nil {
+				return err
+			}
+
+			if _, err := io.Copy(dstFile, f); err != nil {
+				log.Debug().Msgf("Image file copying error %+v - %+v - err %+v", file.Filename, dst, err)
+				return err
+			}
+			src = dst
+		}
+
 		log.Debug().Msgf("Parameter Config: %+v", config)
 
 		// XXX: Only stablediffusion is supported for now
@@ -73,7 +106,7 @@ func ImageEndpoint(cm *config.ConfigLoader, o *options.Option) func(c *fiber.Ctx
 		if input.ResponseFormat == "b64_json" {
 			b64JSON = true
 		}
-
+		// src and clip_skip
 		var result []Item
 		for _, i := range config.PromptStrings {
 			n := input.N
@@ -121,7 +154,7 @@ func ImageEndpoint(cm *config.ConfigLoader, o *options.Option) func(c *fiber.Ctx
 
 				baseURL := c.BaseURL()
 
-				fn, err := backend.ImageGeneration(height, width, mode, step, input.Seed, positive_prompt, negative_prompt, output, o.Loader, *config, o)
+				fn, err := backend.ImageGeneration(height, width, mode, step, input.Seed, positive_prompt, negative_prompt, src, output, o.Loader, *config, o)
 				if err != nil {
 					return err
 				}

api/openai/request.go

@@ -75,6 +75,10 @@ func updateConfig(config *config.Config, input *OpenAIRequest) {
 		config.Backend = input.Backend
 	}
 
+	if input.ClipSkip != 0 {
+		config.Diffusers.ClipSkip = input.ClipSkip
+	}
+
 	if input.ModelBaseName != "" {
 		config.AutoGPTQ.ModelBaseName = input.ModelBaseName
 	}