fail jl_specializations_lookup more conservatively in the presence of incorrect UnionAll types

previously, inserting a type and then looking up that exact type could fail,
since it would compute the UnionAll in the wrong position and thus not consider the types to be equivalent

Author: vtjnash

src/gf.c

@@ -119,15 +119,15 @@ JL_DLLEXPORT jl_typemap_entry_t *jl_specializations_insert(jl_method_t *m, jl_tu
 
 JL_DLLEXPORT jl_value_t *jl_specializations_lookup(jl_method_t *m, jl_tupletype_t *type)
 {
-    jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(m->specializations, type, NULL, 1, /*subtype*/0, /*offs*/0);
+    jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(m->specializations, type, NULL, 2, /*subtype*/0, /*offs*/0);
     if (!sf)
         return jl_nothing;
     return sf->func.value;
 }
 
 JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_tupletype_t *type)
 {
-    jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(mt->defs, type, NULL, 1, /*subtype*/0, /*offs*/0);
+    jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(mt->defs, type, NULL, 2, /*subtype*/0, /*offs*/0);
     if (!sf)
         return jl_nothing;
     return sf->func.value;

src/typemap.c

@@ -514,8 +514,11 @@ int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs,
 
 int sigs_eq(jl_value_t *a, jl_value_t *b, int useenv)
 {
-    if (jl_has_typevars(a) || jl_has_typevars(b)) {
-        return jl_types_equal_generic(a,b,useenv);
+    // useenv == 0 : subtyping + ensure typevars correspond
+    // useenv == 1 : subtyping + ensure typevars correspond + fail if bound != bound in some typevar match
+    // useenv == 2 : ignore typevars (because UnionAll getting lost in intersection can cause jl_types_equal to fail in the wrong direction for some purposes)
+    if (useenv != 2 && (jl_has_typevars(a) || jl_has_typevars(b))) {
+        return jl_types_equal_generic(a, b, useenv);
     }
     return jl_subtype(a, b, 0) && jl_subtype(b, a, 0);
 }