Skip to content

perf(turbopack): Use rayon for CPU-heavy tasks #79561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: canary
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion turbopack/crates/turbo-tasks-backend/src/backend/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1887,7 +1887,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
}

let this = self.clone();
let snapshot = turbo_tasks::spawn_blocking(move || this.snapshot()).await;
let snapshot = turbo_tasks::spawn_blocking_rayon(move || this.snapshot()).await;
if let Some((snapshot_start, new_data)) = snapshot {
last_snapshot = snapshot_start;
if new_data {
Expand Down
2 changes: 1 addition & 1 deletion turbopack/crates/turbo-tasks-fs/src/retry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ where
{
let path = path.as_ref().to_owned();

turbo_tasks::spawn_blocking(move || {
turbo_tasks::spawn_blocking_tokio(move || {
let mut attempt = 1;

loop {
Expand Down
4 changes: 2 additions & 2 deletions turbopack/crates/turbo-tasks/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ pub use manager::{
CurrentCellRef, ReadConsistency, TaskPersistence, TurboTasks, TurboTasksApi,
TurboTasksBackendApi, TurboTasksBackendApiExt, TurboTasksCallApi, Unused, UpdateInfo,
dynamic_call, emit, mark_finished, mark_root, mark_session_dependent, mark_stateful,
prevent_gc, run_once, run_once_with_reason, spawn_blocking, spawn_thread, trait_call,
turbo_tasks, turbo_tasks_scope,
prevent_gc, run_once, run_once_with_reason, spawn_blocking_rayon, spawn_blocking_tokio,
spawn_thread, trait_call, turbo_tasks, turbo_tasks_scope,
};
pub use output::OutputContent;
pub use raw_vc::{CellId, RawVc, ReadRawVcFuture, ResolveTypeError};
Expand Down
25 changes: 24 additions & 1 deletion turbopack/crates/turbo-tasks/src/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1784,7 +1784,9 @@ pub fn emit<T: VcValueTrait + ?Sized>(collectible: ResolvedVc<T>) {
})
}

pub async fn spawn_blocking<T: Send + 'static>(func: impl FnOnce() -> T + Send + 'static) -> T {
pub async fn spawn_blocking_tokio<T: Send + 'static>(
func: impl FnOnce() -> T + Send + 'static,
) -> T {
let turbo_tasks = turbo_tasks();
let span = Span::current();
let (result, duration, alloc_info) = tokio::task::spawn_blocking(|| {
Expand All @@ -1801,6 +1803,27 @@ pub async fn spawn_blocking<T: Send + 'static>(func: impl FnOnce() -> T + Send +
result
}

pub async fn spawn_blocking_rayon<T: Send + 'static>(
func: impl FnOnce() -> T + Send + 'static,
) -> T {
let (tx, rx) = tokio::sync::oneshot::channel();

let turbo_tasks = turbo_tasks();
let span = Span::current();
rayon::spawn(|| {
let _guard = span.entered();
let start = Instant::now();
let start_allocations = TurboMalloc::allocation_counters();
let r = turbo_tasks_scope(turbo_tasks, func);
let _ = tx.send((r, start.elapsed(), start_allocations.until_now()));
});

let (result, duration, alloc_info) = rx.await.unwrap();
capture_future::add_duration(duration);
capture_future::add_allocation_info(alloc_info);
result
}

pub fn spawn_thread(func: impl FnOnce() + Send + 'static) {
let handle = Handle::current();
let span = info_span!("thread").or_current();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ impl EcmascriptBrowserChunkContent {
let mut code = code.build();

if let MinifyType::Minify { mangle } = this.chunking_context.await?.minify_type() {
code = minify(&code, source_maps, mangle)?;
code = minify(code, source_maps, mangle).await?;
}

Ok(code.cell())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ impl EcmascriptBrowserEvaluateChunk {
let mut code = code.build();

if let MinifyType::Minify { mangle } = this.chunking_context.await?.minify_type() {
code = minify(&code, source_maps, mangle)?;
code = minify(code, source_maps, mangle).await?;
}

Ok(code.cell())
Expand Down
10 changes: 7 additions & 3 deletions turbopack/crates/turbopack-ecmascript/src/minify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,11 @@ use turbopack_core::{
use crate::parse::generate_js_source_map;

#[instrument(level = Level::INFO, skip_all)]
pub fn minify(code: &Code, source_maps: bool, mangle: Option<MangleType>) -> Result<Code> {
pub async fn minify(code: Code, source_maps: bool, mangle: Option<MangleType>) -> Result<Code> {
turbo_tasks::spawn_blocking_rayon(move || minify_inner(code, source_maps, mangle)).await
}

fn minify_inner(code: Code, source_maps: bool, mangle: Option<MangleType>) -> Result<Code> {
let source_maps = source_maps
.then(|| code.generate_source_map_ref())
.transpose()?;
Expand Down Expand Up @@ -139,8 +143,8 @@ pub fn minify(code: &Code, source_maps: bool, mangle: Option<MangleType>) -> Res
src_map_buf,
Some(original_map),
// We do not inline source contents.
// We provide a synthesized value to `cm.new_source_file` above, so it cannot be
// the value user expect anyway.
// We provide a synthesized value to `cm.new_source_file` above, so it cannot
// be the value user expect anyway.
false,
)?),
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ impl EcmascriptBuildNodeChunkContent {
let mut code = code.build();

if let MinifyType::Minify { mangle } = this.chunking_context.await?.minify_type() {
code = minify(&code, source_maps, mangle)?;
code = minify(code, source_maps, mangle).await?;
}

Ok(code.cell())
Expand Down
Loading