Refactor derive macros and add HasSpaces trait (#934)

The main part of this PR is adding a `HasSpaces` trait that enumerates
spaces in a plan. This trait can be automatically generated using derive
macro and attributes, saving the effort to enumerate spaces by hand.

- This PR adds a `HasSpaces` trait that enumerates spaces of a plan.
Plans (including NoGC) are now required to implement this trait.
-   This PR implements derived macro for `HasSpaces`
- The `#[trace]` and `#[fallback_trace]` attributes are replaced by the
`#[space]` and `#[parent]` attributes for generality. A dedicated
`#[copy_semantics(xxxxx)]` attribute is added to specify copy semantics
for `trace_object`.
- The manually written `Plan::get_spaces()` method is removed in favor
for the automatically generated `HasSpace::for_each_space` method.
- Added a `Plan::verify_side_metadata_sanity` that calls
`Space::verify_side_metadata_sanity` for all spaces in the plan. This
replaces the manual invocations of `Space::verify_side_metadata_sanity`
in the constructors of plans.

This PR also slightly refactors the mmtk-macros crate.
- The `syn` dependency is one major version behind the latest. This PR
bumps the version and refactors the code to adapt to this change.
- `mmtk-core/macros/src/lib.rs` is cleaned up so that it only contains
functions with minimum function bodies. Those functions for derive
macros are required to be in the root module of the crate. Their
function bodies are off-loaded to other modules.

This PR is a stepping stone for
#925. This PR makes it possible to
enumerate spaces of a plan without resorting to unsafe global maps.

Author: wks

docs/userguide/src/tutorial/code/mygc_semispace/gc_work.rs

@@ -23,7 +23,7 @@ impl<VM: VMBinding> crate::scheduler::GCWorkContext for MyGCWorkContext2<VM> {
     type PlanType = MyGC<VM>;
     type ProcessEdgesWorkType = PlanProcessEdges<Self::VM, MyGC<VM>, DEFAULT_TRACE>;
 }
-// ANCHOR: workcontext_plan
+// ANCHOR_END: workcontext_plan
 
 use crate::util::ObjectReference;
 use crate::util::copy::CopySemantics;

docs/userguide/src/tutorial/code/mygc_semispace/global.rs

@@ -14,7 +14,7 @@ use crate::scheduler::*; // Modify
 use crate::util::alloc::allocators::AllocatorSelector;
 use crate::util::copy::*;
 use crate::util::heap::VMRequest;
-use crate::util::metadata::side_metadata::{SideMetadataSanity, SideMetadataContext};
+use crate::util::metadata::side_metadata::SideMetadataContext;
 use crate::util::opaque_pointer::*;
 use crate::vm::VMBinding;
 use enum_map::EnumMap;
@@ -24,18 +24,20 @@ use std::sync::atomic::{AtomicBool, Ordering}; // Add
 // Remove #[allow(unused_imports)].
 // Remove handle_user_collection_request().
 
-use mmtk_macros::PlanTraceObject;
+use mmtk_macros::{HasSpaces, PlanTraceObject};
 
 // Modify
 // ANCHOR: plan_def
-#[derive(PlanTraceObject)]
+#[derive(HasSpaces, PlanTraceObject)]
 pub struct MyGC<VM: VMBinding> {
     pub hi: AtomicBool,
-    #[trace(CopySemantics::DefaultCopy)]
+    #[space]
+    #[copy_semantics(CopySemantics::DefaultCopy)]
     pub copyspace0: CopySpace<VM>,
-    #[trace(CopySemantics::DefaultCopy)]
+    #[space]
+    #[copy_semantics(CopySemantics::DefaultCopy)]
     pub copyspace1: CopySpace<VM>,
-    #[fallback_trace]
+    #[parent]
     pub common: CommonPlan<VM>,
 }
 // ANCHOR_END: plan_def
@@ -51,8 +53,6 @@ pub const MYGC_CONSTRAINTS: PlanConstraints = PlanConstraints {
 // ANCHOR_END: constraints
 
 impl<VM: VMBinding> Plan for MyGC<VM> {
-    type VM = VM;
-
     fn constraints(&self) -> &'static PlanConstraints {
         &MYGC_CONSTRAINTS
     }
@@ -74,15 +74,6 @@ impl<VM: VMBinding> Plan for MyGC<VM> {
     }
     // ANCHOR_END: create_copy_config
 
-    // ANCHOR: get_spaces
-    fn get_spaces(&self) -> Vec<&dyn Space<Self::VM>> {
-        let mut ret = self.common.get_spaces();
-        ret.push(&self.copyspace0);
-        ret.push(&self.copyspace1);
-        ret
-    }
-    // ANCHOR_EN: get_spaces
-
     // Modify
     // ANCHOR: schedule_collection
     fn schedule_collection(&'static self, scheduler: &GCWorkScheduler<VM>) {
@@ -188,12 +179,7 @@ impl<VM: VMBinding> MyGC<VM> {
             common: CommonPlan::new(plan_args),
         };
 
-        // Use SideMetadataSanity to check if each spec is valid. This is also needed for check
-        // side metadata in extreme_assertions.
-        let mut side_metadata_sanity_checker = SideMetadataSanity::new();
-        res.common.verify_side_metadata_sanity(&mut side_metadata_sanity_checker);
-        res.copyspace0.verify_side_metadata_sanity(&mut side_metadata_sanity_checker);
-        res.copyspace1.verify_side_metadata_sanity(&mut side_metadata_sanity_checker);
+        res.verify_side_metadata_sanity();
 
         res
     }

docs/userguide/src/tutorial/mygc/ss/alloc.md

@@ -66,13 +66,16 @@ Finished code (step 2):
 {{#include ../../code/mygc_semispace/global.rs:plan_def}}
 ```
 
-Note that we have attributes on some fields. These attributes tell MMTk's macros on
-how to generate code to trace objects in this plan. Although there are other approaches that
-you can implement object tracing, in this tutorial we use the macros, as it is the simplest.
-Make sure you import the macros. We will discuss on what those attributes mean in later sections.
+Note that `MyGC` now also derives `PlanTraceObject` besides `HasSpaces`, and we
+have attributes on some fields. These attributes tell MMTk's macros how to
+generate code to visit each space of this plan as well as trace objects in this
+plan.  Although there are other approaches that you can implement object
+tracing, in this tutorial we use the macros, as it is the simplest.  Make sure
+you import the macros. We will discuss on what those attributes mean in later
+sections.
 
 ```rust
-use mmtk_macros::PlanTraceObject;
+use mmtk_macros::{HasSpaces, PlanTraceObject};
 ```
 
 ### Implement the Plan trait for MyGC
@@ -230,4 +233,4 @@ by the common plan:
 
 With this, you should have the allocation working, but not garbage collection. 
 Try building again. If you run HelloWorld or Fannkunchredux, they should
-work. DaCapo's lusearch should fail, as it requires garbage to be collected. 
+work. DaCapo's lusearch should fail, as it requires garbage to be collected. 

docs/userguide/src/tutorial/mygc/ss/collection.md

@@ -181,11 +181,19 @@ it can use `PlanProcessEdges`.
 
 You can manually provide an implementation of `PlanTraceObject` for `MyGC`. But you can also use the derive macro MMTK provides,
 and the macro will generate an implementation of `PlanTraceObject`:
-* add `#[derive(PlanTraceObject)]` for `MyGC` (import the macro properly: `use mmtk_macros::PlanTraceObject`)
-* add `#[trace(CopySemantics::Default)]` to both copy space fields, `copyspace0` and `copyspace1`. This tells the macro to generate
-  trace code for both spaces, and for any copying in the spaces, use `CopySemantics::DefaultCopy` that we have configured early.
-* add `#[fallback_trace]` to `common`. This tells the macro that if an object is not found in any space with `#[trace]` in ths plan,
-  try find the space for the object in the 'parent' plan. In our case, we fall back to the `CommonPlan`, as the object may be
+
+* Make sure `MyGC` already has the `#[derive(HasSpaces)]` attribute because all plans need to
+  implement the `HasSpaces` trait anyway.  (import the macro properly: `use mmtk_macros::HasSpaces`)
+* Add `#[derive(PlanTraceObject)]` for `MyGC` (import the macro properly: `use mmtk_macros::PlanTraceObject`)
+* Add both `#[space]` and `#[copy_semantics(CopySemantics::Default)]` to both copy space fields,
+  `copyspace0` and `copyspace1`. `#[space]` tells the macro that both `copyspace0` and `copyspace1`
+  are spaces in the `MyGC` plan, and the generated trace code will check both spaces.
+  `#[copy_semantics(CopySemantics::DefaultCopy)]` specifies the copy semantics to use when tracing
+  objects in the corresponding space.
+* Add `#[parent]` to `common`. This tells the macro that there are more spaces defined in `common`
+  and its nested structs.  If an object is not found in any space with `#[space]` in this plan,
+  the trace code will try to find the space for the object in the 'parent' plan.  In our case, the
+  trace code will proceed by checking spaces in the `CommonPlan`, as the object may be
   in large object space or immortal space in the common plan. `CommonPlan` also implements `PlanTraceObject`, so it knows how to
   find a space for the object and trace it in the same way.
 
@@ -238,7 +246,7 @@ In the end, use `MyGCProcessEdges` as `ProcessEdgesWorkType` in the `GCWorkConte
 ## Summary
 
 You should now have MyGC working and able to collect garbage. All three
-benchmarks should be able to pass now. 
+benchmarks should be able to pass now.
 
 If the benchmarks pass - good job! You have built a functional copying
 collector!

macros/Cargo.toml

@@ -13,6 +13,6 @@ proc-macro = true
 
 [dependencies]
 proc-macro2 = "1.0.37"
-syn = { version = "1.0.91", features = ["extra-traits"] }
+syn = { version = "2.0.29", features = ["extra-traits"] }
 quote = "1.0.18"
 proc-macro-error = "1.0.4"

macros/src/has_spaces_impl.rs

@@ -0,0 +1,82 @@
+use proc_macro2::TokenStream as TokenStream2;
+use proc_macro_error::abort_call_site;
+use quote::quote;
+use syn::{DeriveInput, Field};
+
+use crate::util;
+
+pub(crate) fn derive(input: DeriveInput) -> TokenStream2 {
+    let ident = input.ident;
+    let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
+
+    let syn::Data::Struct(syn::DataStruct {
+        fields: syn::Fields::Named(ref fields),
+        ..
+    }) = input.data else {
+        abort_call_site!("`#[derive(HasSpaces)]` only supports structs with named fields.");
+    };
+
+    let spaces = util::get_fields_with_attribute(fields, "space");
+    let parent = util::get_unique_field_with_attribute(fields, "parent");
+
+    let items = generate_impl_items(&spaces, &parent);
+
+    quote! {
+        impl #impl_generics crate::plan::HasSpaces for #ident #ty_generics #where_clause {
+            type VM = VM;
+
+            #items
+        }
+    }
+}
+
+pub(crate) fn generate_impl_items<'a>(
+    space_fields: &[&'a Field],
+    parent_field: &Option<&'a Field>,
+) -> TokenStream2 {
+    // Currently we implement callback-style visitor methods.
+    // Iterators should be more powerful, but is more difficult to implement.
+
+    let mut space_visitors = vec![];
+    let mut space_visitors_mut = vec![];
+
+    for f in space_fields {
+        let f_ident = f.ident.as_ref().unwrap();
+
+        let visitor = quote! {
+            __func(&self.#f_ident);
+        };
+
+        let visitor_mut = quote! {
+            __func(&mut self.#f_ident);
+        };
+
+        space_visitors.push(visitor);
+        space_visitors_mut.push(visitor_mut);
+    }
+
+    let (parent_visitor, parent_visitor_mut) = if let Some(f) = parent_field {
+        let f_ident = f.ident.as_ref().unwrap();
+        let visitor = quote! {
+            self.#f_ident.for_each_space(__func)
+        };
+        let visitor_mut = quote! {
+            self.#f_ident.for_each_space_mut(__func)
+        };
+        (visitor, visitor_mut)
+    } else {
+        (quote! {}, quote! {})
+    };
+
+    quote! {
+        fn for_each_space(&self, __func: &mut dyn FnMut(&dyn Space<VM>)) {
+            #(#space_visitors)*
+            #parent_visitor
+        }
+
+        fn for_each_space_mut(&mut self, __func: &mut dyn FnMut(&mut dyn Space<VM>)) {
+            #(#space_visitors_mut)*
+            #parent_visitor_mut
+        }
+    }
+}

macros/src/lib.rs

@@ -4,66 +4,60 @@ extern crate quote;
 extern crate syn;
 
 use proc_macro::TokenStream;
-use proc_macro_error::abort_call_site;
 use proc_macro_error::proc_macro_error;
-use quote::quote;
 use syn::parse_macro_input;
 use syn::DeriveInput;
 
+mod has_spaces_impl;
 mod plan_trace_object_impl;
 mod util;
 
 const DEBUG_MACRO_OUTPUT: bool = false;
 
-/// Generally a plan needs to add these attributes in order for the macro to work. The macro will
-/// generate an implementation of `PlanTraceObject` for the plan. With `PlanTraceObject`, the plan use
-/// `PlanProcessEdges` for GC tracing. The attributes only affects code generation in the macro, thus
-/// only affects the generated `PlanTraceObject` implementation.
-/// * add `#[derive(PlanTraceObject)]` to the plan struct.
-/// * add `#[trace]` to each space field the plan struct has. If the policy is a copying policy,
-///   it needs to further specify the copy semantic (`#[trace(CopySemantics::X)]`)
-/// * add `#[fallback_trace]` to the parent plan if the plan is composed with other plans (or parent plans).
-///   For example, `GenImmix` is composed with `Gen`, `Gen` is composed with `CommonPlan`, `CommonPlan` is composed
-///   with `BasePlan`.
-/// * add `#[post_scan]` to any space field that has some policy-specific post_scan_object(). For objects in those spaces,
-///   `post_scan_object()` in the policy will be called after `VM::VMScanning::scan_object()`.
+/// This macro will generate an implementation of `HasSpaces` for a plan or any structs that
+/// contain spaces, including `Gen`, `CommonPlan` and `BasePlan`.
+///
+/// The `HasSpaces` trait is responsible for enumerating spaces in a struct.  When using this
+/// derive macro, the user should do the following.
+///
+/// * Make sure the struct has a generic type parameter named `VM` which requires `VMBinding`.
+///   For example, `struct MyPlan<VM: VMBinding>` will work.
+/// * Add `#[space]` for each space field in the struct.
+/// * Add `#[parent]` to the field that contain more space fields.  This attribute is usually
+///   added to `Gen`, `CommonPlan` or `BasePlan` fields.  There can be at most one parent in
+///   a struct.
 #[proc_macro_error]
-#[proc_macro_derive(PlanTraceObject, attributes(trace, post_scan, fallback_trace))]
-pub fn derive_plan_trace_object(input: TokenStream) -> TokenStream {
+#[proc_macro_derive(HasSpaces, attributes(space, parent))]
+pub fn derive_has_spaces(input: TokenStream) -> TokenStream {
     let input = parse_macro_input!(input as DeriveInput);
-    let ident = input.ident;
-    let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
-
-    let output = if let syn::Data::Struct(syn::DataStruct {
-        fields: syn::Fields::Named(ref fields),
-        ..
-    }) = input.data
-    {
-        let spaces = util::get_fields_with_attribute(fields, "trace");
-        let post_scan_spaces = util::get_fields_with_attribute(fields, "post_scan");
-        let fallback = util::get_unique_field_with_attribute(fields, "fallback_trace");
+    let output = has_spaces_impl::derive(input);
 
-        let trace_object_function =
-            plan_trace_object_impl::generate_trace_object(&spaces, &fallback, &ty_generics);
-        let post_scan_object_function = plan_trace_object_impl::generate_post_scan_object(
-            &post_scan_spaces,
-            &fallback,
-            &ty_generics,
-        );
-        let may_move_objects_function =
-            plan_trace_object_impl::generate_may_move_objects(&spaces, &fallback, &ty_generics);
-        quote! {
-            impl #impl_generics crate::plan::PlanTraceObject #ty_generics for #ident #ty_generics #where_clause {
-                #trace_object_function
-
-                #post_scan_object_function
+    output.into()
+}
 
-                #may_move_objects_function
-            }
-        }
-    } else {
-        abort_call_site!("`#[derive(PlanTraceObject)]` only supports structs with named fields.")
-    };
+/// The macro will generate an implementation of `PlanTraceObject` for the plan. With
+/// `PlanTraceObject`, the plan will be able to use `PlanProcessEdges` for GC tracing.
+///
+/// The user should add `#[space]` and `#[parent]` attributes to fields as specified by the
+/// `HasSpaces` trait.  When using this derive macro, all spaces must implement the
+/// `PolicyTraceObject` trait.  The generated `trace_object` method will check for spaces in the
+/// current plan and, if the object is not in any of them, check for plans in the parent struct.
+/// The parent struct must also implement the `PlanTraceObject` trait.
+///
+/// In addition, the user can add the following attributes to fields in order to control the
+/// behavior of the generated `trace_object` method.
+///
+/// * Add `#[copy_semantics(CopySemantics::X)]` to a space field to specify that when tracing
+///   objects in that space, `Some(CopySemantics::X)` will be passed to the `Space::trace_object`
+///   method as the `copy` argument.
+/// * Add `#[post_scan]` to any space field that has some policy-specific `post_scan_object()`. For
+///   objects in those spaces, `post_scan_object()` in the policy will be called after
+///   `VM::VMScanning::scan_object()`.
+#[proc_macro_error]
+#[proc_macro_derive(PlanTraceObject, attributes(space, parent, copy_semantics, post_scan))]
+pub fn derive_plan_trace_object(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+    let output = plan_trace_object_impl::derive(input);
 
     // Debug the output - use the following code to debug the generated code (when cargo exapand is not working)
     if DEBUG_MACRO_OUTPUT {