finished code

Author: fbelderink

android/src/main/java/io/fynn/torch_mobile/Convert.java

@@ -0,0 +1,71 @@
+package io.fynn.torch_mobile;
+
+public class Convert {
+
+    public static String dtypeAsPrimitive(String dtype){
+        switch (dtype.toLowerCase()){
+            case "float32":
+                return "Float";
+            case "float64":
+                return "Double";
+            case "int32":
+                return "Integer";
+            case "int64":
+                return "Long";
+            case "int8":
+                return "Byte";
+            case "uint8":
+                return "Byte";
+            default:
+                return null;
+        }
+    }
+
+    public static long[] toPrimitives(Integer[] objects){
+        long[] primitives = new long[objects.length];
+        for(int i = 0; i < objects.length; i++){
+            primitives[i] = objects[i].longValue();
+        }
+        return primitives;
+    }
+
+    public static byte[] toBytePrimitives(Double[] objects){
+        byte[] primitives = new byte[objects.length];
+        for(int i = 0; i < objects.length; i++){
+            primitives[i] = objects[i].byteValue();
+        }
+        return primitives;
+    }
+
+    public static float[] toFloatPrimitives(Double[] objects){
+        float[] primitives = new float[objects.length];
+        for(int i = 0; i < objects.length; i++){
+            primitives[i] = objects[i].floatValue();
+        }
+        return primitives;
+    }
+
+    public static double[] toDoublePrimitives(Double[] objects){
+        double[] primitives = new double[objects.length];
+        for(int i = 0; i < objects.length; i++){
+            primitives[i] = objects[i].doubleValue();
+        }
+        return primitives;
+    }
+
+    public static long[] toLongPrimitives(Double[] objects){
+        long[] primitives = new long[objects.length];
+        for(int i = 0; i < objects.length; i++){
+            primitives[i] = objects[i].longValue();
+        }
+        return primitives;
+    }
+
+    public static int[] toIntegerPrimitives(Double[] objects){
+        int[] primitives = new int[objects.length];
+        for(int i = 0; i < objects.length; i++){
+            primitives[i] = objects[i].intValue();
+        }
+        return primitives;
+    }
+}

android/src/main/java/io/fynn/torch_mobile/TorchMobilePlugin.java

@@ -1,5 +1,7 @@
 package io.fynn.torch_mobile;
 
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
 import android.util.Log;
 
 import androidx.annotation.NonNull;
@@ -10,17 +12,19 @@
 import io.flutter.plugin.common.MethodChannel.Result;
 import io.flutter.plugin.common.PluginRegistry.Registrar;
 
+import org.pytorch.DType;
 import org.pytorch.IValue;
 import org.pytorch.Module;
 import org.pytorch.Tensor;
 import org.pytorch.torchvision.TensorImageUtils;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 
 /** TorchMobilePlugin */
-public class TorchMobilePlugin implements FlutterPlugin, MethodCallHandler {
+public class TorchMobilePlugin<T> implements FlutterPlugin, MethodCallHandler {
 
   ArrayList<Module> modules = new ArrayList<>();
 
@@ -50,31 +54,151 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull Result result) {
         }
         break;
       case "predict":
-        Module module;
+        Module module = null;
+        Integer[] shape = null;
+        Double[] data = null;
+        DType dtype = null;
+
         try{
           int index = call.argument("index");
           module = modules.get(index);
 
-          ArrayList<Long> shapeList = call.argument("shape");
-          Long[] shape = shapeList.toArray(new Long[shapeList.size()]);
-          Tensor.fromBlob(new int[]{1,2,3,4}, toPrimitives(shape));
+          dtype = DType.valueOf(call.argument("dtype").toString().toUpperCase());
+
+          ArrayList<Integer> shapeList = call.argument("shape");
+          shape = shapeList.toArray(new Integer[shapeList.size()]);
+
+          ArrayList<Double> dataList = call.argument("data");
+          data = dataList.toArray(new Double[dataList.size()]);
 
         }catch(Exception e){
-          Log.e("TorchMobile", "", e);
+          Log.e("TorchMobile", "error parsing arguments", e);
+        }
+
+        //prepare input tensor
+        final Tensor inputTensor = getInputTensor(dtype, data, shape);
+
+        //run model
+        Tensor outputTensor = null;
+        try {
+          outputTensor = module.forward(IValue.from(inputTensor)).toTensor();
+        }catch(RuntimeException e){
+          Log.e("TorchMobile", "Your input type " + dtype.toString().toLowerCase()  + " (" + Convert.dtypeAsPrimitive(dtype.toString()) +") " + "does not match with model input type",e);
+          result.success(null);
+        }
+
+        successResult(result, dtype, outputTensor);
+
+        break;
+      case "predictImage":
+        Module imageModule = null;
+        Bitmap bitmap = null;
+        try {
+          int index = call.argument("index");
+          byte[] imageData = call.argument("image");
+          int width = call.argument("width");
+          int height = call.argument("height");
+
+          imageModule = modules.get(index);
+
+          bitmap = BitmapFactory.decodeByteArray(imageData,0,imageData.length);
+
+          bitmap = Bitmap.createScaledBitmap(bitmap, width, height, false);
+
+        }catch (Exception e){
+          Log.e("TorchMobile", "error reading image", e);
+        }
+
+        final Tensor imageInputTensor = TensorImageUtils.bitmapToFloat32Tensor(bitmap,
+                TensorImageUtils.TORCHVISION_NORM_MEAN_RGB, TensorImageUtils.TORCHVISION_NORM_STD_RGB);
+
+        final Tensor imageOutputTensor = imageModule.forward(IValue.from(imageInputTensor)).toTensor();
+
+        float[] scores = imageOutputTensor.getDataAsFloatArray();
+
+        ArrayList<Float> out = new ArrayList<>();
+        for(float f : scores){
+          out.add(f);
         }
+
+        result.success(out);
+
         break;
       default:
         result.notImplemented();
         break;
     }
   }
 
-  public static long[] toPrimitives(Long[] objects){
-    long[] primitives = new long[objects.length];
-    for(int i = 0; i < objects.length; i++){
-      primitives[i] = objects[i];
+  //returns input tensor depending on dtype
+  private Tensor getInputTensor(DType dtype, Double[] data, Integer[] shape){
+    switch (dtype){
+      case FLOAT32:
+        return Tensor.fromBlob(Convert.toFloatPrimitives(data), Convert.toPrimitives(shape));
+      case FLOAT64:
+        return  Tensor.fromBlob(Convert.toPrimitives(shape), Convert.toDoublePrimitives(data));
+      case INT32:
+        return Tensor.fromBlob(Convert.toIntegerPrimitives(data), Convert.toPrimitives(shape));
+      case INT64:
+        return Tensor.fromBlob(Convert.toLongPrimitives(data), Convert.toPrimitives(shape));
+      case INT8:
+        return Tensor.fromBlob(Convert.toBytePrimitives(data), Convert.toPrimitives(shape));
+      case UINT8:
+        return Tensor.fromBlobUnsigned(Convert.toBytePrimitives(data), Convert.toPrimitives(shape));
+      default:
+        return null;
+    }
+  }
+
+  //gets tensor depending on dtype and creates list of it, which is being returned
+  private void successResult(Result result, DType dtype, Tensor outputTensor){
+    switch (dtype){
+      case FLOAT32:
+        ArrayList<Float> outputListFloat = new ArrayList<>();
+        for(float f : outputTensor.getDataAsFloatArray()){
+          outputListFloat.add(f);
+        }
+        result.success(outputListFloat);
+        break;
+      case FLOAT64:
+        ArrayList<Double> outputListDouble = new ArrayList<>();
+        for(double d : outputTensor.getDataAsDoubleArray()){
+          outputListDouble.add(d);
+        }
+        result.success(outputListDouble);
+        break;
+      case INT32:
+        ArrayList<Integer> outputListInteger = new ArrayList<>();
+        for(int i : outputTensor.getDataAsIntArray()){
+          outputListInteger.add(i);
+        }
+        result.success(outputListInteger);
+        break;
+      case INT64:
+        ArrayList<Long> outputListLong = new ArrayList<>();
+        for(long l : outputTensor.getDataAsLongArray()){
+          outputListLong.add(l);
+        }
+        result.success(outputListLong);
+        break;
+      case INT8:
+        ArrayList<Byte> outputListByte = new ArrayList<>();
+        for(byte b : outputTensor.getDataAsByteArray()){
+          outputListByte.add(b);
+        }
+        result.success(outputListByte);
+        break;
+      case UINT8:
+        ArrayList<Byte> outputListUByte = new ArrayList<>();
+        for(byte ub : outputTensor.getDataAsUnsignedByteArray()){
+          outputListUByte.add(ub);
+        }
+        result.success(outputListUByte);
+        break;
+      default:
+        result.success(null);
+        break;
     }
-    return primitives;
   }
 
   @Override

example/assets/labels/labels.csv

@@ -0,0 +1,2 @@
+trade,test,ride,leg,excitement,paper,boy,reach,lot,gulf,middle,row,white,citizen,stuck,product,mostly,toward,do,once,now,shown,leader,basic,person,zoo,under,easier,early,eat,girl,influence,camp,everybody,brief,herself,exercise,dark,ball,spend,come,anyway,pupil,great,winter,have,symbol,previous,mark,mysterious,wonder,law,available,sale,operation,stared,are,supper,trap,fallen,fighting,possible,most,picture,automobile,escape,sit,adjective,push,sound,pale,hay,pull,help,cent,discussion,break,taste,till,differ,period,felt,dirt,recently,own,quite,planned,clear,screen,upper,those,hope,definition,twelve,whatever,over,silence,rubber,mirror,goose,shelter,garage,throughout,husband,past,character,dress,forest,forget,disappear,happy,blanket,flat,design,whether,foot,noted,completely,track,using,hole,taught,go,chest,especially,slide,appearance,high,rod,newspaper,strength,everyone,please,careful,recent,magic,church,replace,they,late,rear,harbor,became,hill,unless,extra,driving,grass,finish,jump,gate,its,favorite,anyone,practical,provide,on,eleven,article,choose,safe,straight,stomach,shot,edge,warm,darkness,feel,composed,cave,bit,research,instance,breath,sat,leaf,dog,field,other,view,wonderful,prove,nation,horn,table,fight,farmer,cabin,trail,avoid,stay,lips,hat,vote,equipment,bite,curve,electricity,pound,easily,pot,happily,repeat,rather,production,cause,brain,finger,orange,stick,wise,wagon,lucky,by,bell,vegetable,freedom,library,number,correct,secret,exclaimed,explain,broad,among,won,seeing,men,easy,sail,satellites,came,safety,master,huge,kept,particularly,roll,heard,touch,value,blow,proper,wrong,pleasant,reason,this,date,prepare,garden,sky,cup,setting,contain,valuable,shells,took,stream,beneath,clean,pipe,store,write,primitive,team,wife,onto,work,acres,straw,science,short,later,feed,me,remove,point,practice,hurried,place,youth,yourself,mind,square,according,flight,pool,grade,held,cake,post,become,satisfied,layers,poem,remember,ants,ourselves,cook,sunlight,buffalo,answer,thrown,model,ring,perhaps,mad,central,every,slabs,lying,art,buried,tried,statement,essential,phrase,arrange,browserling,worry,river,syllable,studying,wealth,never,studied,market,chicken,that,nobody,limited,experiment,anything,fact,double,porch,fat,ice,highest,walk,original,silver,bottom,fireplace,here,oil,organization,duty,everywhere,manner,loose,since,drop,shade,future,combination,certain,home,may,fog,special,electric,slow,numeral,leaving,check,strike,thus,stand,putting,official,slowly,mean,factory,water,fort,mission,changing,low,enter,for,conversation,think,saddle,dead,nothing,nuts,volume,across,wild,muscle,simply,letter,sheep,hunt,swung,identity,leave,problem,clothing,create,did,thirty,deep,then,slipped,medicine,behavior,south,lovely,meal,without,any,information,author,larger,mine,guide,lay,coast,sand,brought,universe,record,popular,them,themselves,rough,small,seen,my,all,time,word,count,hair,speed,himself,book,brick,sun,fellow,occasionally,copper,finest,knowledge,twenty,necessary,useful,forward,front,shut,scared,blind,together,remain,courage,greatest,failed,ancient,examine,pleasure,frame,thank,frequently,vast,region,frozen,neck,climb,die,danger,tribe,famous,speak,well,port,sea,snake,birth,blue,red,carbon,plastic,draw,to,salmon,flow,us,quickly,however,coach,catch,boat,either,plate,fur,dream,ocean,herd,graph,amount,west,near,imagine,bet,involved,wheel,yes,vessels,round,mountain,introduced,somewhere,mighty,sudden,joined,football,until,half,image,typical,whom,fewer,back,desert,unknown,sport,drew,certainly,swimming,food,caught,swim,cowboy,paragraph,several,waste,title,consider,eight,four,say,line,meant,event,movie,alphabet,chance,rate,already,discover,string,willing,third,trip,station,smallest,task,instead,tune,bicycle,quick,cattle,except,pour,fall,allow,alike,audience,plant,instrument,care,various,ate,sing,transportation,fix,just,origin,wire,prevent,pass,part,east,death,biggest,fell,journey,grabbed,as,north,chapter,facing,luck,gift,explanation,damage,spread,mile,health,shadow,face,his,concerned,put,sell,loss,dot,brush,whistle,down,bean,sharp,bring,fire,highway,birthday,somehow,jungle,when,dozen,hang,anywhere,construction,fought,raise,complete,knew,effect,pack,list,free,label,police,development,teacher,struck,serve,fence,parent,bar,air,full,pattern,lose,cream,surface,bat,donkey,throw,sold,bare,hollow,how,signal,ship,arrive,were,tropical,eaten,wish,tube,iron,must,parts,angle,fed,noise,direction,had,fear,younger,board,pressure,smile,higher,more,aboard,guess,none,busy,yellow,cut,process,silk,stopped,mathematics,heat,itself,nails,monkey,bark,tie,milk,voyage,southern,settle,beautiful,we,chain,neighborhood,laid,pie,adult,clearly,inside,indicate,needle,also,education,atmosphere,change,chief,badly,include,rocky,regular,scientist,carefully,merely,exact,tool,hurry,driver,strong,tonight,shoulder,search,threw,national,consonant,basis,state,scene,buy,widely,series,slept,collect,married,offer,happened,way,dug,moving,balloon,sleep,excited,suit,sitting,faster,getting,pay,fierce,success,car,handsome,sense,paint,city,road,pet,another,calm,spring,stepped,carry,wrapped,instant,ahead,structure,large,rise,airplane,naturally,want,constantly,tank,successful,although,broken,trace,machinery,hard,lungs,box,season,mainly,fairly,him,vertical,done,suddenly,whose,train
+warn,thumb,almost,young,smell,doctor,year,bill,show,what,born,how,chosen,volume,solve,good,protection,characteristic,clothing,search,dry,told,fruit,blue,carbon,government,throw,fish,slip,common,those,information,enjoy,fog,sad,only,know,worse,wild,chart,empty,indeed,swimming,master,edge,it,across,spell,explore,immediately,program,bare,product,then,all,forth,leaving,although,orange,nuts,build,something,key,bee,able,fight,position,below,fat,ice,deeply,mixture,welcome,frighten,my,sale,flies,oxygen,enemy,value,street,blind,freedom,remain,speech,it,writer,gravity,skin,unusual,shut,service,beside,school,pale,running,discover,personal,donkey,possible,just,bridge,aloud,till,clear,vegetable,phrase,win,now,appropriate,lost,log,foot,invented,whether,consider,his,came,there,coast,must,fair,again,summer,out,first,work,broke,gain,community,beneath,correctly,shells,directly,smallest,dawn,variety,cheese,vote,belt,bush,law,five,colony,forward,rear,remarkable,root,pie,older,higher,compound,log,while,society,gun,reason,pipe,post,road,this,trick,cabin,gate,closer,scared,your,board,list,classroom,dull,minute,pipe,sand,chamber,nearby,nation,mass,nearby,battle,chose,does,smaller,lady,shelf,bowl,know,through,slabs,wooden,topic,add,amount,hurry,car,low,lying,dollar,saved,heat,curve,our,rule,mad,unknown,push,fuel,dance,evening,angle,hard,hundred,standard,flight,thou,aid,show,headed,wool,cent,attack,over,shade,door,especially,hung,themselves,cookies,grade,lower,capital,strike,contrast,draw,stand,grabbed,sight,balance,tune,ice,volume,exciting,pride,appearance,support,triangle,certainly,slabs,burst,up,certainly,drew,wire,hang,thirty,cloud,truck,golden,seat,moving,bat,voice,bone,leader,society,get,pleasure,soon,chief,blew,deal,feel,flame,meal,fireplace,facing,being,hurried,prepare,drew,enough,former,youth,wrong,contain,hardly,go,straw,full,essential,soil,century,way,such,out,nearly,principal,forest,pressure,then,grade,date,atom,park,noted,old,tobacco,organization,cannot,want,subject,flow,onto,active,allow,plate,represent,dropped,cream,directly,immediately,spider,afternoon,older,each,relationship,twice,put,drawn,anybody,jungle,sleep,halfway,produce,continued,temperature,still,he,change,motor,giant,community,wrote,wonder,attack,event,still,went,wife,art,us,root,into,short,against,additional,tomorrow,master,pen,influence,settle,key,remain,own,sky,post,massage,proper,clay,price,dish,hunter,nice,division,enter,parallel,terrible,talk,pound,lovely,open,create,notice,went,whatever,company,judge,doll,browserling,control,huge,fewer,wide,piano,opportunity,corn,prevent,actually,due,moment,fireplace,now,thumb,breathe,alphabet,shorter,person,driven,worry,surprise,basic,ball,shore,pond,last,claws,nearby,throughout,gentle,inside,house,break,deal,instant,blood,television,rabbit,continent,meat,everybody,sort,write,lose,claws,very,citizen,right,sheep,activity,nest,whatever,silence,both,sort,improve,station,unknown,variety,wave,trap,believed,exercise,death,knew,shorter,fully,worry,blew,specific,slabs,closer,official,came,chart,pound,getting,born,general,neighbor,know,such,board,hungry,poetry,chamber,told,merely,headed,that,sad,twice,nature,manner,monkey,spring,production,finest,raw,combination,weather,did,herself,kids,evidence,situation,eleven,use,replied,cheese,land,scale,store,ten,position,threw,twice,height,drove,function,excitement,dirty,harbor,announced,salmon,engineer,save,cook,verb,excitement,flies,look,whole,important,percent,forgotten,care,tried,doubt,straw,deal,longer,break,gain,window,magic,lot,shallow,organization,rising,lake,leather,adventure,weak,so,buy,listen,stick,create,pole,stage,dozen,underline,make,putting,aside,weigh,browserling,river,toy,gravity,run,stick,coach,newspaper,willing,pool,market,kill,angry,brain,hope,whose,daughter,everything,accident,race,husband,captured,cause,stiff,notice,flame,flag,straw,stiff,into,where,made,single,theory,flew,title,block,burst,meant,simply,clear,event,mathematics,two,drive,careful,relationship,bean,except,my,blow,charge,wind,finally,local,sale,distance,living,tonight,deep,have,situation,greater,statement,either,flower,wool,lose,meat,beginning,diagram,generally,front,gray,page,begun,your,helpful,cost,slight,sleep,behavior,powerful,row,brave,beside,bring,blood,belong,excitement,conversation,main,work,break,captain,highest,know,act,noun,triangle,return,setting,cast,goose,around,list,feathers,bridge,contrast,mail,colony,drawn,stay,machinery,picture,regular,community,driver,poem,cup,explore,stopped,meant,company,say,free,split,trip,be,gain,base,gasoline,pink,season,ship,fellow,monkey,wire,brush,ordinary,upper,inch,us,effect,aboard,alphabet,exactly,had,strip,goes,stage,milk,fastened,most,pool,nearest,four,stood,wooden,read,pony,field,wealth,movie,stone,luck,fifty,action,spirit,careful,scientist,blanket,food,tomorrow,stems,thousand,per,wrote,sheep,across,invented,suggest,book,away,wonderful,hardly,shadow,rapidly,finger,double,everyone,huge,particularly,promised,near,memory,military,sold,mix,rough,lady,muscle,contain,myself,couple,indicate,long,spirit,discover,myself,never,continent,picture,shot,basic,declared,time,honor,generally,fallen,shall,struggle,soldier,loud,cheese,string,wolf,for,hang,entirely,notice,can,fifth,easier,trip,sick,declared,college,follow,office