merge sentencetransformers

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

backend/python/transformers/backend.py

@@ -25,6 +25,8 @@ from transformers import AutoTokenizer, AutoModel, set_seed, TextIteratorStreame
 from transformers import AutoProcessor, MusicgenForConditionalGeneration
 from scipy.io import wavfile
 import outetts
+from sentence_transformers import SentenceTransformer
+
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
@@ -88,6 +90,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         self.CUDA = torch.cuda.is_available()
         self.OV=False
         self.OuteTTS=False
+        self.SentenceTransformer = False
 
         device_map="cpu"
 
@@ -235,6 +238,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     self.speaker = self.interface.create_speaker(audio_path=self.AudioPath)
                 else:
                     self.speaker = self.interface.load_default_speaker(name=SPEAKER)               
+            elif request.Type == "SentenceTransformer":
+                self.model = SentenceTransformer(model_name, trust_remote_code=request.TrustRemoteCode)
+                self.SentenceTransformer = True
             else:
                 print("Automodel", file=sys.stderr)
                 self.model = AutoModel.from_pretrained(model_name, 
@@ -286,18 +292,26 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         max_length = 512
         if request.Tokens != 0:
             max_length = request.Tokens
-        encoded_input = self.tokenizer(request.Embeddings, padding=True, truncation=True, max_length=max_length, return_tensors="pt")    
 
-        # Create word embeddings
+        embeds = None
-        if self.CUDA:
-            encoded_input = encoded_input.to("cuda")
 
-        with torch.no_grad():    
+        if self.SentenceTransformer:
-            model_output = self.model(**encoded_input)
+            print("Calculated embeddings for: " + request.Embeddings, file=sys.stderr)
+            embeds = self.model.encode(request.Embeddings)
+        else:
+            encoded_input = self.tokenizer(request.Embeddings, padding=True, truncation=True, max_length=max_length, return_tensors="pt")    
 
-        # Pool to get sentence embeddings; i.e. generate one 1024 vector for the entire sentence
+            # Create word embeddings
-        sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+            if self.CUDA:
-        return backend_pb2.EmbeddingResult(embeddings=sentence_embeddings[0])
+                encoded_input = encoded_input.to("cuda")
+
+            with torch.no_grad():    
+                model_output = self.model(**encoded_input)
+
+            # Pool to get sentence embeddings; i.e. generate one 1024 vector for the entire sentence
+            sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+            embeds = sentence_embeddings[0]
+        return backend_pb2.EmbeddingResult(embeddings=embeds)
 
     async def _predict(self, request, context, streaming=False): 
         set_seed(request.Seed)

backend/python/transformers/requirements-cpu.txt

@@ -4,3 +4,4 @@ accelerate
 transformers
 bitsandbytes
 outetts
+sentence-transformers==3.3.1

backend/python/transformers/requirements-cublas11.txt

@@ -5,3 +5,4 @@ accelerate
 transformers
 bitsandbytes
 outetts
+sentence-transformers==3.3.1

backend/python/transformers/requirements-cublas12.txt

@@ -4,3 +4,4 @@ llvmlite==0.43.0
 transformers
 bitsandbytes
 outetts
+sentence-transformers==3.3.1

backend/python/transformers/requirements-hipblas.txt

@@ -5,3 +5,5 @@ transformers
 llvmlite==0.43.0
 bitsandbytes
 outetts
+bitsandbytes
+sentence-transformers==3.3.1

backend/python/transformers/requirements-intel.txt

@@ -7,3 +7,4 @@ llvmlite==0.43.0
 intel-extension-for-transformers
 bitsandbytes
 outetts
+sentence-transformers==3.3.1