Merge pull request #1295 from alexcrichton/js-snippets

Implement the local JS snippets RFC

Author: alexcrichton

Cargo.toml

@@ -81,6 +81,7 @@ members = [
   "examples/webaudio",
   "examples/webgl",
   "examples/without-a-bundler",
+  "examples/without-a-bundler-no-modules",
   "tests/no-std",
 ]
 exclude = ['crates/typescript']

azure-pipelines.yml

@@ -40,10 +40,15 @@ jobs:
       - template: ci/azure-install-sccache.yml
       - script: cargo test --target wasm32-unknown-unknown
         displayName: "wasm-bindgen test suite"
+        env:
+          RUST_LOG: wasm_bindgen_test_runner
+          GECKODRIVER_ARGS: --log trace
       - script: cargo test --target wasm32-unknown-unknown -p js-sys
         displayName: "js-sys test suite"
       - script: cargo test --target wasm32-unknown-unknown -p webidl-tests
         displayName: "webidl-tests test suite"
+        env:
+          WBINDGEN_I_PROMISE_JS_SYNTAX_WORKS_IN_NODE: 1
       - script: cargo build --manifest-path crates/web-sys/Cargo.toml --target wasm32-unknown-unknown --features "Node Window Document"
         displayName: "web-sys build"
 
@@ -94,6 +99,8 @@ jobs:
       - template: ci/azure-install-sccache.yml
       - script: cargo test -p wasm-bindgen-webidl
       - script: cargo test -p webidl-tests --target wasm32-unknown-unknown
+        env:
+          WBINDGEN_I_PROMISE_JS_SYNTAX_WORKS_IN_NODE: 1
 
   - job: test_ui
     displayName: "Run UI tests"

ci/azure-install-geckodriver.yml

@@ -12,3 +12,14 @@ steps:
       Write-Host "##vso[task.setvariable variable=GECKODRIVER;]$pwd\geckodriver.exe"
     displayName: "Download Geckodriver (Windows)"
     condition: eq( variables['Agent.OS'], 'Windows_NT' )
+
+  # It turns out that geckodriver.exe will fail if firefox takes too long to
+  # start, and for whatever reason the first execution of `firefox.exe` can
+  # take upwards of a mimute. It seems that subsequent executions are much
+  # faster, so have a dedicated step to run Firefox once which should I
+  # guess warm some cache somewhere so the headless tests later on all
+  # finish successfully
+  - script: |
+      "C:\Program Files\Mozilla Firefox\firefox.exe" --version
+    displayName: "Load firefox.exe into cache (presumably?)"
+    condition: eq( variables['Agent.OS'], 'Windows_NT' )

crates/backend/Cargo.toml

@@ -15,6 +15,7 @@ spans = []
 extra-traits = ["syn/extra-traits"]
 
 [dependencies]
+bumpalo = "2.1"
 lazy_static = "1.0.0"
 log = "0.4"
 proc-macro2 = "0.4.8"

crates/backend/src/ast.rs

@@ -2,6 +2,7 @@ use proc_macro2::{Ident, Span};
 use shared;
 use syn;
 use Diagnostic;
+use std::hash::{Hash, Hasher};
 
 /// An abstract syntax tree representing a rust program. Contains
 /// extra information for joining up this rust code with javascript.
@@ -24,6 +25,8 @@ pub struct Program {
     pub dictionaries: Vec<Dictionary>,
     /// custom typescript sections to be included in the definition file
     pub typescript_custom_sections: Vec<String>,
+    /// Inline JS snippets
+    pub inline_js: Vec<String>,
 }
 
 /// A rust to js interface. Allows interaction with rust objects/functions
@@ -66,11 +69,37 @@ pub enum MethodSelf {
 #[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq, Eq))]
 #[derive(Clone)]
 pub struct Import {
-    pub module: Option<String>,
+    pub module: ImportModule,
     pub js_namespace: Option<Ident>,
     pub kind: ImportKind,
 }
 
+#[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq, Eq))]
+#[derive(Clone)]
+pub enum ImportModule {
+    None,
+    Named(String, Span),
+    Inline(usize, Span),
+}
+
+impl Hash for ImportModule {
+    fn hash<H: Hasher>(&self, h: &mut H) {
+        match self {
+            ImportModule::None => {
+                0u8.hash(h);
+            }
+            ImportModule::Named(name, _) => {
+                1u8.hash(h);
+                name.hash(h);
+            }
+            ImportModule::Inline(idx, _) => {
+                2u8.hash(h);
+                idx.hash(h);
+            }
+        }
+    }
+}
+
 #[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq, Eq))]
 #[derive(Clone)]
 pub enum ImportKind {

crates/backend/src/codegen.rs

@@ -94,25 +94,45 @@ impl TryToTokens for ast::Program {
             shared::SCHEMA_VERSION,
             shared::version()
         );
+        let encoded = encode::encode(self)?;
         let mut bytes = Vec::new();
         bytes.push((prefix_json.len() >> 0) as u8);
         bytes.push((prefix_json.len() >> 8) as u8);
         bytes.push((prefix_json.len() >> 16) as u8);
         bytes.push((prefix_json.len() >> 24) as u8);
         bytes.extend_from_slice(prefix_json.as_bytes());
-        bytes.extend_from_slice(&encode::encode(self)?);
+        bytes.extend_from_slice(&encoded.custom_section);
 
         let generated_static_length = bytes.len();
         let generated_static_value = syn::LitByteStr::new(&bytes, Span::call_site());
 
+        // We already consumed the contents of included files when generating
+        // the custom section, but we want to make sure that updates to the
+        // generated files will cause this macro to rerun incrementally. To do
+        // that we use `include_str!` to force rustc to think it has a
+        // dependency on these files. That way when the file changes Cargo will
+        // automatically rerun rustc which will rerun this macro. Other than
+        // this we don't actually need the results of the `include_str!`, so
+        // it's just shoved into an anonymous static.
+        let file_dependencies = encoded.included_files
+            .iter()
+            .map(|file| {
+                let file = file.to_str().unwrap();
+                quote! { include_str!(#file) }
+            });
+
         (quote! {
             #[allow(non_upper_case_globals)]
             #[cfg(target_arch = "wasm32")]
             #[link_section = "__wasm_bindgen_unstable"]
             #[doc(hidden)]
             #[allow(clippy::all)]
-            pub static #generated_static_name: [u8; #generated_static_length] =
-                *#generated_static_value;
+            pub static #generated_static_name: [u8; #generated_static_length] = {
+                static _INCLUDED_FILES: &[&str] = &[#(#file_dependencies),*];
+
+                *#generated_static_value
+            };
+
         })
         .to_tokens(tokens);
 

crates/backend/src/encode.rs

@@ -1,40 +1,96 @@
+use proc_macro2::{Ident, Span};
 use std::cell::RefCell;
 use std::collections::HashMap;
-
-use proc_macro2::{Ident, Span};
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+use util::ShortHash;
 
 use ast;
 use Diagnostic;
 
-pub fn encode(program: &ast::Program) -> Result<Vec<u8>, Diagnostic> {
+pub struct EncodeResult {
+    pub custom_section: Vec<u8>,
+    pub included_files: Vec<PathBuf>,
+}
+
+pub fn encode(program: &ast::Program) -> Result<EncodeResult, Diagnostic> {
     let mut e = Encoder::new();
     let i = Interner::new();
     shared_program(program, &i)?.encode(&mut e);
-    Ok(e.finish())
+    let custom_section = e.finish();
+    let included_files = i.files.borrow().values().map(|p| &p.path).cloned().collect();
+    Ok(EncodeResult { custom_section, included_files })
 }
 
 struct Interner {
-    map: RefCell<HashMap<Ident, String>>,
+    bump: bumpalo::Bump,
+    files: RefCell<HashMap<String, LocalFile>>,
+    root: PathBuf,
+    crate_name: String,
+}
+
+struct LocalFile {
+    path: PathBuf,
+    definition: Span,
+    new_identifier: String,
 }
 
 impl Interner {
     fn new() -> Interner {
         Interner {
-            map: RefCell::new(HashMap::new()),
+            bump: bumpalo::Bump::new(),
+            files: RefCell::new(HashMap::new()),
+            root: env::var_os("CARGO_MANIFEST_DIR").unwrap().into(),
+            crate_name: env::var("CARGO_PKG_NAME").unwrap(),
         }
     }
 
     fn intern(&self, s: &Ident) -> &str {
-        let mut map = self.map.borrow_mut();
-        if let Some(s) = map.get(s) {
-            return unsafe { &*(&**s as *const str) };
-        }
-        map.insert(s.clone(), s.to_string());
-        unsafe { &*(&*map[s] as *const str) }
+        self.intern_str(&s.to_string())
     }
 
     fn intern_str(&self, s: &str) -> &str {
-        self.intern(&Ident::new(s, Span::call_site()))
+        // NB: eventually this could be used to intern `s` to only allocate one
+        // copy, but for now let's just "transmute" `s` to have the same
+        // lifetmie as this struct itself (which is our main goal here)
+        bumpalo::collections::String::from_str_in(s, &self.bump).into_bump_str()
+    }
+
+    /// Given an import to a local module `id` this generates a unique module id
+    /// to assign to the contents of `id`.
+    ///
+    /// Note that repeated invocations of this function will be memoized, so the
+    /// same `id` will always return the same resulting unique `id`.
+    fn resolve_import_module(&self, id: &str, span: Span) -> Result<&str, Diagnostic> {
+        let mut files = self.files.borrow_mut();
+        if let Some(file) = files.get(id) {
+            return Ok(self.intern_str(&file.new_identifier))
+        }
+        let path = if id.starts_with("/") {
+            self.root.join(&id[1..])
+        } else if id.starts_with("./") || id.starts_with("../") {
+            let msg = "relative module paths aren't supported yet";
+            return Err(Diagnostic::span_error(span, msg))
+        } else {
+            return Ok(self.intern_str(&id))
+        };
+
+        // Generate a unique ID which is somewhat readable as well, so mix in
+        // the crate name, hash to make it unique, and then the original path.
+        let new_identifier = format!("{}{}", self.unique_crate_identifier(), id);
+        let file = LocalFile {
+            path,
+            definition: span,
+            new_identifier,
+        };
+        files.insert(id.to_string(), file);
+        drop(files);
+        self.resolve_import_module(id, span)
+    }
+
+    fn unique_crate_identifier(&self) -> String {
+        format!("{}-{}", self.crate_name, ShortHash(0))
     }
 }
 
@@ -64,8 +120,30 @@ fn shared_program<'a>(
             .iter()
             .map(|x| -> &'a str { &x })
             .collect(),
-        // version: shared::version(),
-        // schema_version: shared::SCHEMA_VERSION.to_string(),
+        local_modules: intern
+            .files
+            .borrow()
+            .values()
+            .map(|file| {
+                fs::read_to_string(&file.path)
+                    .map(|s| {
+                        LocalModule {
+                            identifier: intern.intern_str(&file.new_identifier),
+                            contents: intern.intern_str(&s),
+                        }
+                    })
+                    .map_err(|e| {
+                        let msg = format!("failed to read file `{}`: {}", file.path.display(), e);
+                        Diagnostic::span_error(file.definition, msg)
+                    })
+            })
+            .collect::<Result<Vec<_>, _>>()?,
+        inline_js: prog
+            .inline_js
+            .iter()
+            .map(|js| intern.intern_str(js))
+            .collect(),
+        unique_crate_identifier: intern.intern_str(&intern.unique_crate_identifier()),
     })
 }
 
@@ -111,7 +189,13 @@ fn shared_variant<'a>(v: &'a ast::Variant, intern: &'a Interner) -> EnumVariant<
 
 fn shared_import<'a>(i: &'a ast::Import, intern: &'a Interner) -> Result<Import<'a>, Diagnostic> {
     Ok(Import {
-        module: i.module.as_ref().map(|s| &**s),
+        module: match &i.module {
+            ast::ImportModule::Named(m, span) => {
+                ImportModule::Named(intern.resolve_import_module(m, *span)?)
+            }
+            ast::ImportModule::Inline(idx, _) => ImportModule::Inline(*idx as u32),
+            ast::ImportModule::None => ImportModule::None,
+        },
         js_namespace: i.js_namespace.as_ref().map(|s| intern.intern(s)),
         kind: shared_import_kind(&i.kind, intern)?,
     })

crates/backend/src/lib.rs

@@ -2,6 +2,7 @@
 #![cfg_attr(feature = "extra-traits", deny(missing_debug_implementations))]
 #![doc(html_root_url = "https://docs.rs/wasm-bindgen-backend/0.2")]
 
+extern crate bumpalo;
 #[macro_use]
 extern crate log;
 extern crate proc_macro2;

crates/backend/src/util.rs

@@ -94,7 +94,7 @@ pub fn ident_ty(ident: Ident) -> syn::Type {
 
 pub fn wrap_import_function(function: ast::ImportFunction) -> ast::Import {
     ast::Import {
-        module: None,
+        module: ast::ImportModule::None,
         js_namespace: None,
         kind: ast::ImportKind::Function(function),
     }