Auto merge of #1536 - Xanewok:ipc-everything, r=Xanewok

Implement support for out-of-process compilation

This is quite a lengthy patch, but the gist of it is as follows:
- `rls-ipc` crate is introduced which acts as the IPC interface along with a server/client implementation
- `rls-rustc` is enhanced with optional support for the IPC
- RLS can optionally support it via setting `RLS_OUT_OF_PROCESS` env var (like `rls-rustc` it needs to be compiled `ipc` feature)

The IPC is async JSON-RPC running on Tokio using `parity-tokio-ipc` (UDS on unices and named pipes on Windows)
  - Tokio because I wanted to more or less easily come up with a PoC
  - RPC because I needed a request->response model for VFS IPC function calls
  - uds/pipes because it's somewhat cross-platform and we don't have to worry about `rustc` potentially polluting stdio (maybe just capturing the output in `run_compiler` would be enough?)

However, the implementation is far from efficient - it currently starts a thread per requested compilation, which in turn starts a single-threaded async runtime to drive the IPC server for a given compilation.

I imagine we could either just initiate the runtime globally and spawn the servers on it and drive them to completion on each compilation to reduce the thread spawn/coordination overhead.

While this gets rid of the global environment lock on each (previously) in-process crate compilation, what still needs to be addressed is the [sequential compilation](https://github.com/rust-lang/rls/blob/35eba227650eee482bedac7d691a69a8487b2135/rls/src/build/plan.rs#L122-L124) of cached build plan for this implementation to truly benefit from the unlocked parallelization potential.

I did some rough test runs (~5) and on a warm cache had the following results:
- integration test suite (release) 3.6 +- 0.2s (in-process) vs 3.8 +- 0.3s (out-of-process)
- rustfmt master whitespace change (release) 6.4 +- 0.2s (in-process) vs 6.6 +- 0.3s (out-of-process)

which at least initially confirms that the performance overhead is somewhat negligible if we can really parallelize the work and leverage process isolation for increased stability.

cc #1307

(I'll squash the commits in the final merge, 30+ commits is a tad too much 😅 )

If possible I'd like to get more eyes on the patch to see if it's a good approach and what might be directly improved:
- @matklad for potentially shared rustc-with-patched-filesystem
- @alexheretic for the RLS/implementation itself
- @alexcrichton @nrc do you have thoughts on if we can share the parallel graph compilation logic with Cargo somehow? For now we just rolled our own linear queue here because we didn't need much more but maybe it might be worthwhile to extract the pure execution bits somehow?

Author: bors

Cargo.lock

@@ -27,15 +27,17 @@ rls-data = "0.19"
 rls-rustc = { version = "0.6.0", path = "rls-rustc" }
 rls-span = "0.5"
 rls-vfs = "0.8"
+rls-ipc = { version = "0.1.0", path = "rls-ipc", optional = true }
 
 cargo = { git = "https://github.com/rust-lang/cargo", rev = "1f74bdf4494f4d51dbe3a6af5474e39c8d194ad6" }
 cargo_metadata = "0.8"
 clippy_lints = { git = "https://github.com/rust-lang/rust-clippy", rev = "72da1015d6d918fe1b29170acbf486d30e0c2695", optional = true }
 env_logger = "0.6"
 failure = "0.1.1"
+futures = { version = "0.1", optional = true }
 home = "0.5"
 itertools = "0.8"
-jsonrpc-core = "12"
+jsonrpc-core = "13"
 lsp-types = { version = "0.60", features = ["proposed"] }
 lazy_static = "1"
 log = "0.4"
@@ -50,6 +52,7 @@ serde = "1.0"
 serde_json = "1.0"
 serde_derive = "1.0"
 serde_ignored = "0.1"
+tokio = { version = "0.1", optional = true }
 url = "2"
 walkdir = "2"
 regex = "1"
@@ -76,4 +79,6 @@ tokio-timer = "0.2"
 rustc_tools_util = "0.2"
 
 [features]
-clippy = ["clippy_lints"]
+clippy = ["clippy_lints", "rls-rustc/clippy"]
+ipc = ["tokio", "futures", "rls-rustc/ipc", "rls-ipc/server"]
+default = []

Cargo.toml

@@ -0,0 +1,3 @@
+/target/
+**/*.rs.bk
+Cargo.lock

rls-ipc/.gitignore

@@ -0,0 +1,21 @@
+[package]
+name = "rls-ipc"
+version = "0.1.0"
+authors = ["Igor Matuszewski <[email protected]>"]
+edition = "2018"
+description = "Inter-process communication (IPC) layer between RLS and rustc"
+license = "Apache-2.0/MIT"
+repository = "https://github.com/rust-lang/rls"
+categories = ["development-tools"]
+
+[dependencies]
+jsonrpc-core = "13"
+jsonrpc-core-client = "13"
+jsonrpc-derive = "13"
+jsonrpc-ipc-server = { version = "13", optional = true }
+rls-data = "0.19"
+serde = { version = "1.0", features = ["derive"] }
+
+[features]
+client = ["jsonrpc-core-client/ipc"]
+server = ["jsonrpc-ipc-server"]

rls-ipc/Cargo.toml

@@ -0,0 +1,25 @@
+//! Allows to connect to an IPC server.
+
+use crate::rpc::callbacks::gen_client::Client as CallbacksClient;
+use crate::rpc::file_loader::gen_client::Client as FileLoaderClient;
+
+pub use jsonrpc_core_client::transports::ipc::connect;
+pub use jsonrpc_core_client::{RpcChannel, RpcError};
+
+/// Joint IPC client.
+#[derive(Clone)]
+pub struct Client {
+    /// File loader interface
+    pub file_loader: FileLoaderClient,
+    /// Callbacks interface
+    pub callbacks: CallbacksClient,
+}
+
+impl From<RpcChannel> for Client {
+    fn from(channel: RpcChannel) -> Self {
+        Client {
+            file_loader: FileLoaderClient::from(channel.clone()),
+            callbacks: CallbacksClient::from(channel),
+        }
+    }
+}

rls-ipc/src/client.rs

@@ -0,0 +1,9 @@
+//! Inter-process communication (IPC) layer between RLS and rustc.
+
+#![deny(missing_docs)]
+
+#[cfg(feature = "client")]
+pub mod client;
+pub mod rpc;
+#[cfg(feature = "server")]
+pub mod server;

rls-ipc/src/lib.rs

@@ -0,0 +1,80 @@
+//! Available remote procedure call (RPC) interfaces.
+
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+
+use jsonrpc_derive::rpc;
+use serde::{Deserialize, Serialize};
+
+pub use jsonrpc_core::{Error, Result};
+
+// Separated because #[rpc] macro generated a `gen_client` mod and so two
+// interfaces cannot be derived in the same scope due to a generated name clash
+/// RPC interface for an overriden file loader to be used inside `rustc`.
+pub mod file_loader {
+    use super::*;
+    // Expanded via #[rpc]
+    pub use gen_client::Client;
+    pub use rpc_impl_Rpc::gen_server::Rpc as Server;
+
+    #[rpc]
+    /// RPC interface for an overriden file loader to be used inside `rustc`.
+    pub trait Rpc {
+        /// Query the existence of a file.
+        #[rpc(name = "file_exists")]
+        fn file_exists(&self, path: PathBuf) -> Result<bool>;
+
+        /// Returns an absolute path to a file, if possible.
+        #[rpc(name = "abs_path")]
+        fn abs_path(&self, path: PathBuf) -> Result<Option<PathBuf>>;
+
+        /// Read the contents of an UTF-8 file into memory.
+        #[rpc(name = "read_file")]
+        fn read_file(&self, path: PathBuf) -> Result<String>;
+    }
+}
+
+// Separated because #[rpc] macro generated a `gen_client` mod and so two
+// interfaces cannot be derived in the same scope due to a generated name clash
+/// RPC interface to feed back data from `rustc` instances.
+pub mod callbacks {
+    use super::*;
+    // Expanded via #[rpc]
+    pub use gen_client::Client;
+    pub use rpc_impl_Rpc::gen_server::Rpc as Server;
+
+    #[rpc]
+    /// RPC interface to feed back data from `rustc` instances.
+    pub trait Rpc {
+        /// Hands back computed analysis data for the compiled crate
+        #[rpc(name = "complete_analysis")]
+        fn complete_analysis(&self, analysis: rls_data::Analysis) -> Result<()>;
+
+        /// Hands back computed input files for the compiled crate
+        #[rpc(name = "input_files")]
+        fn input_files(&self, input_files: HashMap<PathBuf, HashSet<Crate>>) -> Result<()>;
+    }
+}
+
+/// Build system-agnostic, basic compilation unit
+#[derive(PartialEq, Eq, Hash, Debug, Clone, Deserialize, Serialize)]
+pub struct Crate {
+    /// Crate name
+    pub name: String,
+    /// Optional path to a crate root
+    pub src_path: Option<PathBuf>,
+    /// Edition in which a given crate is compiled
+    pub edition: Edition,
+    /// From rustc; mainly used to group other properties used to disambiguate a
+    /// given compilation unit.
+    pub disambiguator: (u64, u64),
+}
+
+/// Rust edition
+#[derive(PartialEq, Eq, Hash, Debug, PartialOrd, Ord, Copy, Clone, Deserialize, Serialize)]
+pub enum Edition {
+    /// Rust 2015
+    Edition2015,
+    /// Rust 2018
+    Edition2018,
+}

rls-ipc/src/rpc.rs

@@ -0,0 +1,3 @@
+//! Includes facility functions to start an IPC server.
+
+pub use jsonrpc_ipc_server::{CloseHandle, ServerBuilder};

rls-ipc/src/server.rs

@@ -9,3 +9,18 @@ repository = "https://github.com/rust-lang/rls"
 categories = ["development-tools"]
 
 [dependencies]
+env_logger = "0.6"
+log = "0.4"
+failure = "0.1"
+rand = "0.6"
+clippy_lints = { git = "https://github.com/rust-lang/rust-clippy", rev = "72da1015d6d918fe1b29170acbf486d30e0c2695", optional = true }
+tokio = { version = "0.1", optional = true }
+futures = { version = "0.1", optional = true }
+serde = { version = "1", features = ["derive"], optional = true }
+rls-data = { version = "0.19", optional = true }
+rls-ipc = { path = "../rls-ipc", optional = true }
+
+[features]
+clippy = ["clippy_lints"]
+ipc = ["tokio", "futures", "serde", "rls-data", "rls-ipc/client"]
+default = []

rls-rustc/Cargo.toml

@@ -1,3 +1,3 @@
-fn main() {
-    rls_rustc::run();
+fn main() -> Result<(), ()> {
+    rls_rustc::run()
 }

rls-rustc/src/bin/rustc.rs

@@ -0,0 +1,90 @@
+//! Copied from rls/src/config.rs
+
+use std::str::FromStr;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum ClippyPreference {
+    /// Disable clippy.
+    Off,
+    /// Enable clippy, but `allow` clippy lints (i.e., require `warn` override).
+    OptIn,
+    /// Enable clippy.
+    On,
+}
+
+pub fn preference() -> Option<ClippyPreference> {
+    std::env::var("RLS_CLIPPY_PREFERENCE").ok().and_then(|pref| FromStr::from_str(&pref).ok())
+}
+
+/// Permissive deserialization for `ClippyPreference`
+/// "opt-in", "Optin" -> `ClippyPreference::OptIn`
+impl FromStr for ClippyPreference {
+    type Err = ();
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.to_lowercase().as_str() {
+            "off" => Ok(ClippyPreference::Off),
+            "optin" | "opt-in" => Ok(ClippyPreference::OptIn),
+            "on" => Ok(ClippyPreference::On),
+            _ => Err(()),
+        }
+    }
+}
+
+pub fn adjust_args(args: Vec<String>, preference: ClippyPreference) -> Vec<String> {
+    if preference != ClippyPreference::Off {
+        // Allow feature gating in the same way as `cargo clippy`
+        let mut clippy_args = vec!["--cfg".to_owned(), r#"feature="cargo-clippy""#.to_owned()];
+
+        if preference == ClippyPreference::OptIn {
+            // `OptIn`: Require explicit `#![warn(clippy::all)]` annotation in each workspace crate
+            clippy_args.push("-A".to_owned());
+            clippy_args.push("clippy::all".to_owned());
+        }
+
+        args.iter().map(ToOwned::to_owned).chain(clippy_args).collect()
+    } else {
+        args.to_owned()
+    }
+}
+
+#[cfg(feature = "clippy")]
+pub fn after_parse_callback(compiler: &rustc_interface::interface::Compiler) {
+    use rustc_plugin::registry::Registry;
+
+    let sess = compiler.session();
+    let mut registry = Registry::new(
+        sess,
+        compiler
+            .parse()
+            .expect(
+                "at this compilation stage \
+                 the crate must be parsed",
+            )
+            .peek()
+            .span,
+    );
+    registry.args_hidden = Some(Vec::new());
+
+    let conf = clippy_lints::read_conf(&registry);
+    clippy_lints::register_plugins(&mut registry, &conf);
+
+    let Registry {
+        early_lint_passes, late_lint_passes, lint_groups, llvm_passes, attributes, ..
+    } = registry;
+    let mut ls = sess.lint_store.borrow_mut();
+    for pass in early_lint_passes {
+        ls.register_early_pass(Some(sess), true, false, pass);
+    }
+    for pass in late_lint_passes {
+        ls.register_late_pass(Some(sess), true, false, false, pass);
+    }
+
+    for (name, (to, deprecated_name)) in lint_groups {
+        ls.register_group(Some(sess), true, name, deprecated_name, to);
+    }
+    clippy_lints::register_pre_expansion_lints(sess, &mut ls, &conf);
+    clippy_lints::register_renamed(&mut ls);
+
+    sess.plugin_llvm_passes.borrow_mut().extend(llvm_passes);
+    sess.plugin_attributes.borrow_mut().extend(attributes);
+}

rls-rustc/src/clippy.rs

@@ -0,0 +1,73 @@
+use std::collections::{HashMap, HashSet};
+use std::io;
+use std::path::{Path, PathBuf};
+
+use failure::Fail;
+use futures::Future;
+
+use rls_ipc::client::{Client as JointClient, RpcChannel, RpcError};
+use rls_ipc::rpc::callbacks::Client as CallbacksClient;
+use rls_ipc::rpc::file_loader::Client as FileLoaderClient;
+
+pub use rls_ipc::client::connect;
+
+#[derive(Clone)]
+pub struct Client(JointClient);
+
+impl From<RpcChannel> for Client {
+    fn from(channel: RpcChannel) -> Self {
+        Client(channel.into())
+    }
+}
+
+#[derive(Clone)]
+pub struct IpcFileLoader(FileLoaderClient);
+
+impl IpcFileLoader {
+    pub fn into_boxed(self) -> Option<Box<dyn syntax::source_map::FileLoader + Send + Sync>> {
+        Some(Box::new(self))
+    }
+}
+
+impl syntax::source_map::FileLoader for IpcFileLoader {
+    fn file_exists(&self, path: &Path) -> bool {
+        self.0.file_exists(path.to_owned()).wait().unwrap()
+    }
+
+    fn abs_path(&self, path: &Path) -> Option<PathBuf> {
+        self.0.abs_path(path.to_owned()).wait().ok()?
+    }
+
+    fn read_file(&self, path: &Path) -> io::Result<String> {
+        self.0
+            .read_file(path.to_owned())
+            .wait()
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.compat()))
+    }
+}
+
+#[derive(Clone)]
+pub struct IpcCallbacks(CallbacksClient);
+
+impl IpcCallbacks {
+    pub fn complete_analysis(
+        &self,
+        analysis: rls_data::Analysis,
+    ) -> impl Future<Item = (), Error = RpcError> {
+        self.0.complete_analysis(analysis)
+    }
+
+    pub fn input_files(
+        &self,
+        input_files: HashMap<PathBuf, HashSet<rls_ipc::rpc::Crate>>,
+    ) -> impl Future<Item = (), Error = RpcError> {
+        self.0.input_files(input_files)
+    }
+}
+
+impl Client {
+    pub fn split(self) -> (IpcFileLoader, IpcCallbacks) {
+        let JointClient { file_loader, callbacks } = self.0;
+        (IpcFileLoader(file_loader), IpcCallbacks(callbacks))
+    }
+}