Simplifying image handling

Author: zsogitbe

LLama.Examples/Examples/LlavaInteractiveModeExecute.cs

@@ -102,7 +102,7 @@ public static async Task Run()
                     //
                     foreach (var image in imagePaths)
                     {
-                        ex.Images.Add(new ImageData(ImageData.DataType.ImagePath, image));
+                        ex.Images.Add(File.ReadAllBytes(image));
                     }
                 }
 

LLama/Abstractions/ILLamaExecutor.cs

@@ -27,7 +27,7 @@ public interface ILLamaExecutor
         /// <summary>
         /// List of images: Image filen path, uri or image byte array. See ImageData.
         /// </summary>
-        public List<ImageData> Images { get; }
+        public List<byte[]> Images { get; }
 
         /// <summary>
         /// Asynchronously infers a response from the model.
@@ -38,46 +38,4 @@ public interface ILLamaExecutor
         /// <returns></returns>
         IAsyncEnumerable<string> InferAsync(string text, IInferenceParams? inferenceParams = null, CancellationToken token = default);
     }
-
-    /// <summary>
-    /// Holds image data
-    /// </summary>
-    public class ImageData
-    {
-        /// <summary>
-        /// constructor
-        /// </summary>
-        /// <param name="type"></param>
-        /// <param name="data"></param>
-        public ImageData(DataType type, object data) { Type = type; Data = data; }
-
-        /// <summary>
-        /// the possible types of image data
-        /// </summary>
-        public enum DataType 
-        { 
-            /// <summary>
-            /// file path
-            /// </summary>
-            ImagePath, 
-            /// <summary>
-            /// byte array
-            /// </summary>
-            ImageBytes, 
-            /// <summary>
-            /// uri
-            /// </summary>
-            ImageURL 
-        }
-
-        /// <summary>
-        /// the type of this image data
-        /// </summary>
-        public DataType Type { get; set; }
-
-        /// <summary>
-        /// the image data (string, byte array or uri)
-        /// </summary>
-        public object? Data { get; set; }
-    }
 }

LLama/LLamaExecutorBase.cs

@@ -79,7 +79,7 @@ public bool IsMultiModal
         public LLavaWeights? ClipModel { get;  }
 
         /// <inheritdoc />
-        public List<ImageData> Images { get; set; }
+        public List<byte[]> Images { get; set; }
 
         /// <summary>
         /// Current "mu" value for mirostat sampling
@@ -95,7 +95,7 @@ public bool IsMultiModal
         /// <param name="logger"></param>
         protected StatefulExecutorBase(LLamaContext context, ILogger? logger = null)
         {
-            Images = new List<ImageData>();
+            Images = new List<byte[]>();
             _logger = logger;
             Context = context;
             _pastTokensCount = 0;

LLama/LLamaInteractExecutor.cs

@@ -153,24 +153,7 @@ private Task PreprocessLlava(string text, InferStateArgs args, bool addBos = tru
             {
                 foreach (var image in Images)
                 {
-                    if (image.Type == ImageData.DataType.ImagePath && image.Data != null)
-                    {
-                        _imageEmbedHandles.Add(SafeLlavaImageEmbedHandle.CreateFromFileName(ClipModel.NativeHandle, Context, (string)image.Data));
-                    }
-                    else if (image.Type == ImageData.DataType.ImageBytes && image.Data != null)
-                    {
-                        _imageEmbedHandles.Add(SafeLlavaImageEmbedHandle.CreateFromMemory(ClipModel.NativeHandle, Context, (byte[])image.Data));
-                    }
-                    else if (image.Type == ImageData.DataType.ImageURL && image.Data != null)
-                    {
-                        using var httpClient = new HttpClient();
-                        var uri = new Uri((string)image.Data);
-                        var imageBytes = httpClient.GetByteArrayAsync(uri).Result;
-                        if (imageBytes != null && imageBytes.Length > 0)
-                        {
-                            _imageEmbedHandles.Add(SafeLlavaImageEmbedHandle.CreateFromMemory(ClipModel.NativeHandle, Context, imageBytes));
-                        }
-                    }
+                    _imageEmbedHandles.Add(SafeLlavaImageEmbedHandle.CreateFromMemory(ClipModel.NativeHandle, Context, image));
                 }
 
                 int imageIndex = text.IndexOf("<image>");

LLama/LLamaStatelessExecutor.cs

@@ -34,7 +34,7 @@ public class StatelessExecutor
         public LLavaWeights? ClipModel { get;  }
 
         /// <inheritdoc />
-        public List<ImageData> Images { get; set; }
+        public List<byte[]> Images { get; set; }
 
         /// <summary>
         /// The context used by the executor when running the inference.
@@ -49,7 +49,7 @@ public class StatelessExecutor
         /// <param name="logger"></param>
         public StatelessExecutor(LLamaWeights weights, IContextParams @params, ILogger? logger = null)
         {
-            Images = new List<ImageData>();
+            Images = new List<byte[]>();
             _weights = weights;
             _params = @params;
             _logger = logger;