fix #14009; check for types growing due to recursion and widen them

Author: JeffBezanson

base/inference.jl

@@ -208,6 +208,18 @@ add_tfunc(typeof, 1, 1, typeof_tfunc)
 add_tfunc(typeassert, 2, 2,
     (A, v, t)->(isType(t) ? typeintersect(v,t.parameters[1]) : Any))
 
+function type_depth(t::ANY, d::Int=0)
+    if isa(t,Union)
+        t === Bottom && return d
+        return maximum(x->type_depth(x, d+1), t.types)
+    elseif isa(t,DataType)
+        P = t.parameters
+        isempty(P) && return d
+        return maximum(x->type_depth(x, d+1), P)
+    end
+    return d
+end
+
 function limit_type_depth(t::ANY, d::Int, cov::Bool, vars)
     if isa(t,TypeVar) || isa(t,TypeConstructor)
         return t
@@ -1508,6 +1520,20 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
     #print("typeinf ", ast0, " ", sparams, " ", atypes, "\n")
 
     global inference_stack, CYCLE_ID
+
+    f = inference_stack
+    while !isa(f,EmptyCallStack)
+        if is(f.ast,ast0)
+            # impose limit if we recur and the argument types grow beyond MAX_TYPE_DEPTH
+            td = type_depth(atypes)
+            if td > MAX_TYPE_DEPTH && td > type_depth(f.types)
+                atypes = limit_type_depth(atypes, 0, true, [])
+                break
+            end
+        end
+        f = f.prev
+    end
+
     # check for recursion
     f = inference_stack
     while !isa(f,EmptyCallStack)

test/core.jl

@@ -3586,3 +3586,13 @@ function __f_isa_arg_1()
     length(a)
 end
 @test __f_isa_arg_1() == 1
+
+# non-terminating inference, issue #14009
+type A14009{T}; end
+A14009{T}(a::T) = A14009{T}()
+f14009(a) = rand(Bool) ? f14009(A14009(a)) : a
+code_typed(f14009, (Int,))
+
+type B14009{T}; end
+g14009(a) = g14009(B14009{a})
+code_typed(g14009, (Type{Int},))