Sync changes to upstream

Author: jeffmo

auxiliary/lib/auxiliary.js

@@ -1 +1,7 @@
 declare var XXX: string;
+declare module Foo {
+    declare var x: string;
+}
+declare module Bar {
+    declare var y: string;
+}

auxiliary_tests/module_test.js

@@ -0,0 +1,6 @@
+/* @flow */
+var Foo = require('Foo');
+var Bar = require('Bar');
+
+var x:number = Foo.x;
+var y:number = Bar.y;

auxiliary_tests/node_modules/Bar.js

@@ -1 +1,2 @@
+/* @flow */
 var _:number = XXX;

auxiliary_tests/node_modules/Foo_.js

@@ -50,7 +50,10 @@ type ViewAction = ViewCreateMessageAction | ViewClickThreadAction;
 
 type PayloadType = {
   source: any;
-  action: any/*ServerAction | ViewAction*/;
+  action: ViewCreateMessageAction
+        | ViewClickThreadAction
+        | ServerReceiveRawMessagesAction
+        | ServerReceiveRawCreatedMessageAction;
 };
 
 var PayloadSources = ChatConstants.PayloadSources;

auxiliary_tests/type_test.js

@@ -145,7 +145,13 @@ MessageStore.dispatchToken = ChatAppDispatcher.register(function(payload: any) {
       break;
 
     case ActionTypes.CREATE_MESSAGE:
-      var message = MessageStore.getCreatedMessageData(action.text);
+      // WORKAROUND: getCreatedMessageData could create a message object
+      // with threadID being null. This seems like a potentially serious
+      // issue. However, fixing this issue would require a major refactoring
+      // on threadID assignment to be consistent. Since in the example setup
+      // null never appears in this spot, we simply silence flow's errors
+      // about it.
+      var message: any = MessageStore.getCreatedMessageData(action.text);
       _messages[message.id] = message;
       MessageStore.emitChange();
       break;

examples/flux-chat/js/dispatcher/ChatAppDispatcher.js

@@ -25,20 +25,27 @@
  *    read-only mode with all the workers.
  *    The master stores, the workers read.
  *    Only concurrent reads allowed. No concurrent write/read and write/write.
+ *    There are a few different OCaml modules that act as interfaces to this
+ *    global storage. They all use the same area of memory, so only one can be
+ *    active at any one time. The caller is responsible for zeroing out the
+ *    memory when it is done.
  *
  * II) The dependency table. It's a hashtable that contains all the
  *    dependencies between Hack objects. It is filled concurrently by
  *    the workers. The dependency table is made of 2 hashtables, one that
  *    can is used to quickly answer if a dependency exists. The other one
  *    to retrieve the list of dependencies associated with an object.
+ *    Only the hashes of the objects are stored, so this uses relatively
+ *    little memory. No dynamic allocation is required.
  *
- * III) The Hashtable.
+ * III) The Hashtable that maps string keys to string values. (The strings
+ *    are really serialized / marshalled representations of OCaml structures.)
  *    Key observation of the table is that data with the same key are
  *    considered equivalent, and so you can arbitrarily get any copy of it;
  *    furthermore if data is missing it can be recomputed, so incorrectly
  *    saying data is missing when it is being written is only a potential perf
  *    loss. Note that "equivalent" doesn't necessarily mean "identical", e.g.,
- *    two alpha-converted types are "equivalent" though not litterally byte-
+ *    two alpha-converted types are "equivalent" though not literally byte-
  *    identical. (That said, I'm pretty sure the Hack typechecker actually does
  *    always write identical data, but the hashtable doesn't need quite that
  *    strong of an invariant.)
@@ -60,6 +67,11 @@
  *    -) Concurrent removes: NOT SUPPORTED
  *       Only the master can remove, and can only do so if there are no other
  *       concurrent operations (reads or writes).
+ *
+ *    Since the values are variably sized and can get quite large, they are
+ *    stored separately from the hashes in a garbage-collected heap.
+ *
+ * Both II and III resolve hash collisions via linear probing.
  */
 /*****************************************************************************/
 
@@ -175,6 +187,12 @@ static char* heap_init;
 /* This should only be used by the master */
 static size_t heap_init_size = 0;
 
+/* For debugging */
+value hh_heap_size() {
+  CAMLparam0();
+  CAMLreturn(Val_long(*heap - heap_init));
+}
+
 /*****************************************************************************/
 /* Given a pointer to the shared memory address space, initializes all
  * the globals that live in shared memory.

examples/flux-chat/js/stores/MessageStore.js

@@ -40,6 +40,11 @@ external save: string -> unit = "hh_save"
 (*****************************************************************************)
 external load: string -> unit = "hh_load"
 
+(*****************************************************************************)
+(* The size of the dynamically allocated shared memory section *)
+(*****************************************************************************)
+external heap_size: unit -> int = "hh_heap_size"
+
 (*****************************************************************************)
 (* Module returning the MD5 of the key. It's because the code in C land
  * expects this format. I prefer to make it an abstract type to make sure

hack/heap/hh_shared.c

@@ -50,6 +50,11 @@ val save: string -> unit
 (*****************************************************************************)
 val load: string -> unit
 
+(*****************************************************************************)
+(* The size of the dynamically allocated shared memory section *)
+(*****************************************************************************)
+val heap_size : unit -> int
+
 (*****************************************************************************)
 (* Cache invalidation. *)
 (*****************************************************************************)

hack/heap/sharedMem.ml

@@ -272,9 +272,7 @@ let xhpname = ('%')? letter (alphanumeric | ':' [^':''>'] | '-')*
 let otag = '<' ['a'-'z''A'-'Z'] (alphanumeric | ':' | '-')*
 let ctag = '<' '/' (alphanumeric | ':' | '-')+ '>'
 let lvar = '$' varname
-let reflvar = '&' '$' varname
 let ws = [' ' '\t' '\r' '\x0c']
-let wsnl = [' ' '\t' '\r' '\x0c''\n']
 let hex = digit | ['a'-'f''A'-'F']
 let hex_number = '0' 'x' hex+
 let bin_number = '0' 'b' ['0'-'1']+
@@ -698,3 +696,11 @@ and format_xhptoken = parse
   | "<!--"             { Topen_xhp_comment  }
   | "-->"              { Tclose_xhp_comment }
   | _                  { Terror             }
+
+(* Normally you can just use "token" and get back Tlvar, but specifically for
+ * member variable accesses, the part to the right of the "->" isn't a word
+ * (cannot contain '-' for example) but doesn't start with '$' so isn't an lvar
+ * either. *)
+and varname = parse
+  | varname            { Tword  }
+  | _                  { Terror }

hack/heap/sharedMem.mli

@@ -2517,7 +2517,7 @@ and expr_binop env bop ast_bop e1 =
 and expr_arrow env e1 tok =
   reduce env e1 tok begin fun e1 env ->
     let e2 =
-      match L.token env.file env.lb with
+      match L.varname env.lb with
       | Tword ->
           let name = Lexing.lexeme env.lb in
           let pos = Pos.make env.file env.lb in

hack/parsing/lexer_hack.mll

@@ -11,20 +11,6 @@
 type 'a nextlist = 
   unit -> 'a list
 
-let rec make_bucket n acc l =
-  match l with
-  | _ when n <= 0 -> acc, l
-  | [] -> acc, l
-  | x :: rl ->
-      make_bucket (n-1) (x :: acc) rl
-
-let make_bucket l = 
-  let x = ref l in
-  fun _ ->
-    let res, next = make_bucket 1000 [] l in
-    x := next;
-    res
-
 let single_threaded_call job merge neutral next =
   let x = ref (next()) in
   let acc = ref neutral in

hack/parsing/parser_hack.ml

@@ -14,8 +14,6 @@
 type 'a nextlist = 
   unit -> 'a list
 
-val make_bucket: 'a list -> ('b -> 'a list)
-
 val call :
   Worker.t list option ->
   job:('b -> 'a list -> 'b) ->

hack/procs/multiWorker.ml

@@ -12,9 +12,16 @@ open ServerEnv
 
 (* Initialization of the server *)
 let init_hack genv env get_next =
+  let t = Unix.gettimeofday() in
+
   let files_info, errorl1, failed1 =
     Parsing_service.go genv.workers ~get_next in
 
+  let t2 = Unix.gettimeofday() in
+  Printf.printf "Parsing: %f\n%!" (t2 -. t);
+  Printf.printf "Heap size: %d\n%!" (SharedMem.heap_size ());
+  let t = t2 in
+
   let is_check_mode =
     ServerArgs.check_mode genv.options &&
     ServerArgs.convert genv.options = None &&
@@ -31,11 +38,24 @@ let init_hack genv env get_next =
     Relative_path.Map.fold
       Naming.ndecl_file files_info ([], Relative_path.Set.empty, nenv) in
 
+  let t2 = Unix.gettimeofday() in
+  Printf.printf "Naming: %f\n%!" (t2 -. t);
+  let t = t2 in
+
   let fast = FileInfo.simplify_fast files_info in
   let fast = Relative_path.Set.fold Relative_path.Map.remove failed2 fast in
   let errorl3, failed3 = Typing_decl_service.go genv.workers nenv fast in
+
+  let t2 = Unix.gettimeofday() in
+  Printf.printf "Type-decl: %f\n%!" (t2 -. t);
+  Printf.printf "Heap size: %d\n%!" (SharedMem.heap_size ());
+  let t = t2 in
+
   let errorl4, failed4 = Typing_check_service.go genv.workers fast in
 
+  let t2 = Unix.gettimeofday() in
+  Printf.printf "Type-check: %f\n%!" (t2 -. t);
+
   let failed =
     List.fold_right Relative_path.Set.union
       [failed1; failed2; failed3; failed4]

hack/procs/multiWorker.mli

@@ -217,7 +217,7 @@ let type_check genv env =
   let t = Unix.gettimeofday() in
   let fast_parsed, errorl, failed_parsing = parsing genv env in
   let t2 = Unix.gettimeofday() in
-  Printf.printf "Parsing: %f\n" (t2 -. t); flush stdout;
+  Printf.printf "Parsing: %f\n%!" (t2 -. t);
   let t = t2 in
 
   (* UPDATE FILE INFO *)
@@ -237,14 +237,14 @@ let type_check genv env =
   let fast = add_old_decls env.files_info fast in
 
   let t2 = Unix.gettimeofday() in
-  Printf.printf "Naming: %f\n" (t2 -. t); flush stdout;
+  Printf.printf "Naming: %f\n%!" (t2 -. t);
   let t = t2 in
 
   let _, _, to_redecl_phase2, to_recheck1 =
     Typing_redecl_service.redo_type_decl
       ~update_pos:true genv.workers nenv fast in
   let t2 = Unix.gettimeofday() in
-  Printf.printf "Determining changes: %f\n" (t2 -. t); flush stdout;
+  Printf.printf "Determining changes: %f\n%!" (t2 -. t);
   let t = t2 in
 
   let to_redecl_phase2 = Typing_deps.get_files to_redecl_phase2 in
@@ -258,7 +258,7 @@ let type_check genv env =
       ~update_pos:false genv.workers nenv fast_redecl_phase2 in
 
   let t2 = Unix.gettimeofday() in
-  Printf.printf "Type-decl: %f\n" (t2 -. t); flush stdout;
+  Printf.printf "Type-decl: %f\n%!" (t2 -. t);
   let t = t2 in
 
   let errorl = List.rev_append errorl' errorl in
@@ -279,7 +279,7 @@ let type_check genv env =
   let errorl = List.rev (List.rev_append errorl' errorl) in
 
   let t2 = Unix.gettimeofday() in
-  Printf.printf "Type-check: %f\n" (t2 -. t); flush stdout;
+  Printf.printf "Type-check: %f\n%!" (t2 -. t);
 
   (* Done, that's the new environment *)
   { files_info = files_info;

hack/server/serverInit.ml

@@ -32,7 +32,7 @@ module SN = Naming_special_names
  * a worker needs to know if the type of a class is ready or if it needs to
  * be recomputed. When ClassStatus.mem *class-name* returns true, we know
  * the class has already been recomputed by a different worker.
- * TypedefStatus is a similar mechanism for typedefs.
+ * TypedefHeap is a similar mechanism for typedefs.
  *)
 (*****************************************************************************)
 

hack/server/serverTypeCheck.ml

@@ -13,7 +13,6 @@
 (* Module declaring the types in parallel *)
 (*****************************************************************************)
 open Utils
-module PHeap = Parser_heap.ParserHeap
 
 (* filename => functions defined, classes defined *)
 type fast = (SSet.t * SSet.t * SSet.t * SSet.t) SMap.t

hack/typing/typing_decl.ml

@@ -76,7 +76,6 @@ let fresh_tvars (env:Env.env) (ts:tparam list) (ps:fun_params) (uniq:int) : Env.
   let ts' = List.map (fun (x, y, z) -> x, map_snd rename_str y, z) ts in
   env, ps', ts'
 
-
 let lookup_magic_type (env:Env.env) (class_:ty) (fname:string) (uniq:int) :
     Env.env * (fun_params * tparam list * ty option) option =
   match class_ with
@@ -132,7 +131,6 @@ let parse_printf_string (env:Env.env) (s:string) (pos:Pos.t) (class_:ty) : Env.e
   in
     read_text env 0
 
-
 type ('a, 'b) either = Left of 'a | Right of 'b
 
 let mapM (f:'s->'x->'s*'y) : 'st -> 'x list -> 's * 'y list =

hack/typing/typing_decl_service.ml

@@ -28,10 +28,8 @@ let compute_deps_neutral = ISet.empty, ISet.empty
 (* This is the place where we are going to put everything necessary for
  * the redeclaration. We could "pass" the values directly to the workers,
  * but it gives too much work to the master and slows things downn.
- * So what we do instead is:
- * Set the value of "data_storage", which means that one of the data nodes
- * has this value.
- * Let each worker pool the data from "data_storage".
+ * So what we do instead is pass the data through shared memory via
+ * OnTheFlyStore.
  * I tried replicating the data to speed things up but it had no effect.
  *)
 (*****************************************************************************)
@@ -46,7 +44,7 @@ end)
 (* Re-declaring the types in a file *)
 (*****************************************************************************)
 
-let on_the_fly_decl_file nenv all_classes fast (errors, failed) fn =
+let on_the_fly_decl_file nenv all_classes (errors, failed) fn =
   let decl_errors, () = Errors.do_
     begin fun () ->
     (* We start "recording" dependencies. 
@@ -170,16 +168,16 @@ let update_positions classes to_update =
  *)
 (*****************************************************************************)
 
-let redeclare_files nenv all_classes fast filel =
+let redeclare_files nenv all_classes filel =
   List.fold_left
-    (on_the_fly_decl_file nenv all_classes fast)
+    (on_the_fly_decl_file nenv all_classes)
     ([], Relative_path.Set.empty)
     filel
 
-let otf_decl_files nenv all_classes fast filel =
+let otf_decl_files nenv all_classes filel =
   SharedMem.invalidate_caches();
   (* Redeclaring the files *)
-  let errors, failed = redeclare_files nenv all_classes fast filel in
+  let errors, failed = redeclare_files nenv all_classes filel in
   errors, failed
 
 let compute_deps ~update_pos nenv fast filel =
@@ -215,7 +213,7 @@ let compute_deps ~update_pos nenv fast filel =
 let load_and_otf_decl_files acc filel =
   try
     let nenv, all_classes, fast = OnTheFlyStore.load() in
-    otf_decl_files nenv all_classes fast filel
+    otf_decl_files nenv all_classes filel
   with e ->
     Printf.printf "Error: %s\n" (Printexc.to_string e);
     flush stdout;
@@ -296,23 +294,19 @@ let get_defs fast =
   end fast FileInfo.empty_names
 
 (*****************************************************************************)
-(* The main entry points *)
+(* The main entry point *)
 (*****************************************************************************)
 
-let init workers fast =
-  let fnl = Relative_path.Map.fold (fun x _ y -> x :: y) fast [] in
-  let all_classes = Typing_decl_service.get_classes fast in
-  all_classes, fnl
-
 let redo_type_decl ~update_pos workers nenv fast =
-  let all_classes, fnl = init workers fast in
+  let fnl = Relative_path.Map.keys fast in
+  let all_classes = Typing_decl_service.get_classes fast in
   let defs = get_defs fast in
   invalidate_heap defs;
   (* If there aren't enough files, let's do this ourselves ... it's faster! *)
   let result =
     if List.length fnl < 10
     then
-      let errors, failed = otf_decl_files nenv all_classes fast fnl in
+      let errors, failed = otf_decl_files nenv all_classes fnl in
       let to_redecl, to_recheck = compute_deps ~update_pos nenv fast fnl in
       errors, failed, to_redecl, to_recheck
     else parallel_otf_decl ~update_pos workers nenv all_classes fast fnl