Merge pull request #23 from alan-turing-institute/fix-slow

Improve performance of scitype on arrays in the mlj convention (resolves #12)

Author: ablaom

src/ScientificTypes.jl

@@ -136,6 +136,10 @@ scitype_union(A) = reduce((a,b)->Union{a,b}, (scitype(el) for el in A))
 # ## SCITYPES OF TUPLES AND ARRAYS
 
 scitype(t::Tuple, ::Val) = Tuple{scitype.(t)...}
+
+# The following fallback can be quite slow. Individual conventions
+# will usually be able to find more perfomant overloadings of this
+# method:
 scitype(A::B, ::Val) where {T,N,B<:AbstractArray{T,N}} =
     AbstractArray{scitype_union(A),N}
 

src/conventions/mlj/finite.jl

@@ -18,3 +18,18 @@ function coerce(v, ::Type{T2}; verbosity=1) where T2 <: Union{Missing,Finite}
     end
     return categorical(v, true, ordered=T2 <: Union{Missing,OrderedFactor})
 end
+
+## PERFORMANT SCITYPES FOR ARRAYS
+
+function scitype(A::B, ::Val{:mlj}) where {T,N,B<:CategoricalArray{T,N}}
+    nlevels = length(levels(A))
+    if isordered(A)
+        S = OrderedFactor{nlevels}
+    else
+        S = Multiclass{nlevels}
+    end
+    if T isa Union && Missing <: T
+        S = Union{S,Missing}
+    end
+    return AbstractArray{S, N}
+end

src/conventions/mlj/images.jl

@@ -4,5 +4,3 @@ scitype(image::AbstractArray{<:Gray,2}, ::Val{:mlj}) =
     GrayImage{size(image)...}
 scitype(image::AbstractArray{<:AbstractRGB,2}, ::Val{:mlj}) =
     ColorImage{size(image)...}
-
-

src/conventions/mlj/mlj.jl

@@ -5,6 +5,21 @@ _coerce_missing_warn(T) =
     @warn "Missing values encountered coercing scitype to $T.\n"*
           "Coerced to Union{Missing,$T} instead. "
 
+
+## PERFORMANT SCITYPES FOR ARRAYS
+
+const A{T,N} = AbstractArray{T,N}
+
+scitype(::B, ::Val{:mlj}) where {N,B<:A{<:AbstractFloat,N}} =
+    A{Continuous,N}
+scitype(::B, ::Val{:mlj}) where {N,B<:A{Union{<:AbstractFloat,Missing},N}} =
+    A{Union{Continuous,Missing},N}
+scitype(::B, ::Val{:mlj}) where {N,B<:A{<:Integer,N}} =
+    A{Count,N}
+scitype(::B, ::Val{:mlj}) where {N,B<:A{Union{<:Integer,Missing},N}} =
+    A{Union{Count,Missing},N}
+
+
 ## COERCE VECTOR TO CONTINUOUS
 
 """

test/runtests.jl

@@ -37,15 +37,53 @@ A = Any[2 4.5;
 end
 
 @testset "Arrays" begin
-    @test scitype(A) == AbstractArray{Union{Count, Continuous}, 2}
-    @test scitype([1,2,3, missing]) == AbstractVector{Union{Missing, Count}}
+    @test scitype(A) ==
+        AbstractArray{Union{Count, Continuous}, 2}
+
+    @test scitype([1, 2, 3]) ==
+        AbstractVector{Count}
+    @test scitype([1, missing, 3]) ==
+        AbstractVector{Union{Missing,Count}}
+    @test scitype(Any[1, 2, 3]) ==
+        AbstractVector{Count}
+    @test scitype(Any[1, missing, 3]) ==
+        AbstractVector{Union{Missing,Count}}
+
+    @test scitype([1.0, 2.0, 3.0]) ==
+        AbstractVector{Continuous}
+    @test scitype(Any[1.0, missing, 3.0]) ==
+        AbstractVector{Union{Missing,Continuous}}
+    @test scitype(Any[1.0, 2.0, 3.0]) ==
+        AbstractVector{Continuous}
+    @test scitype(Any[1.0, missing, 3.0]) ==
+        AbstractVector{Union{Missing,Continuous}}
+
+    @test scitype(categorical(1:4)) ==
+        AbstractVector{Multiclass{4}}
+    @test scitype(Any[categorical(1:4)...]) ==
+        AbstractVector{Multiclass{4}}
+    @test scitype(categorical([1, missing, 3])) ==
+        AbstractVector{Union{Multiclass{2},Missing}}
+
+    @test scitype(categorical(1:4, ordered=true)) ==
+        AbstractVector{OrderedFactor{4}}
+    @test scitype(Any[categorical(1:4, ordered=true)...]) ==
+        AbstractVector{OrderedFactor{4}}
+    @test scitype(categorical([1, missing, 3], ordered=true)) ==
+        AbstractVector{Union{OrderedFactor{2},Missing}}
+
 end
 
 @testset "Images" begin
     black = RGB(0, 0, 0)
     color_image = fill(black, (10, 20))
     @test scitype(color_image) == ColorImage{10,20}
 
+    color_image2 = fill(black, (5, 3))
+    v = [color_image, color_image2, color_image2]
+    @test scitype(v) ==
+        AbstractVector{Union{ColorImage{10,20},ColorImage{5,3}}}
+
     white = Gray(1.0)
     gray_image = fill(white, (10, 20))
     @test scitype(gray_image) == GrayImage{10,20}