mmtk · wks · Dec 11, 2021 · Dec 14, 2021 · Dec 23, 2021 · Jan 6, 2022
diff --git a/.github/configs/base.yml b/.github/configs/base.yml
@@ -2,13 +2,16 @@ includes:
   - "$RUNNING_NG_PACKAGE_DATA/base/runbms.yml"
 
 suites:
-  dacapo-23.9-RC3-chopin-ci:
+  # We use the same suites name "dacapochopin" as defined in "running-ng/src/running/config/base/dacapo.yml".
+  # This will combine this definition with the existing "dacapochopin".
+  # This will also allow us to use modifiers such as "dacapochopin_jdk21" which expect the suites name "dacapochopin".
+  dacapochopin:
     type: DaCapo
-    # Need running-ng to support 23.9
-    release: evaluation
-    path: "DACAPO_PATH/dacapo-23.9-RC3-chopin.jar"
+    release: "23.11"
+    path: "DACAPO_PATH/dacapo-23.11-MR2-chopin.jar"
     minheap: mmtk-openjdk-11-MarkCompact
     # Min heap values are from dacapo-evaluation-git-04132797
+    # TODO: Use the values provided by the released version, and use G1 instead of MarkCompact.
     minheap_values:
       mmtk-openjdk-11-MarkCompact:
         avrora: 8
@@ -83,19 +86,19 @@ plugins:
       - "scratch/stderr.log"
 
 runtimes:
-  jdk11-master:
+  jdk21-master:
     type: OpenJDK
-    release: 11
+    release: 21
     home: "/home/runner/work/mmtk-openjdk/mmtk-openjdk/bundles/jdk"
 
 configs:
-  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-SemiSpace"
-  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-GenCopy"
-  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-Immix"
-  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-GenImmix"
-  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-StickyImmix"
-  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkSweep"
-  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkCompact"
+  - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-SemiSpace"
+  - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-GenCopy"
+  - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-Immix"
+  - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-GenImmix"
+  - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-StickyImmix"
+  - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkSweep"
+  - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkCompact"
 
 benchmarks:
-  dacapo-23.9-RC3-chopin-ci:
+  dacapochopin:
diff --git a/.github/scripts/ci-build.sh b/.github/scripts/ci-build.sh
@@ -13,4 +13,4 @@ build_target=$1
 # Build product bundle
 cd $OPENJDK_PATH
 sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL
-make CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$BINDING_PATH/openjdk $OPENJDK_BUILD_TARGET
+make CONF=linux-x86_64-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$BINDING_PATH/openjdk $OPENJDK_BUILD_TARGET
diff --git a/.github/scripts/ci-check-lineends.sh b/.github/scripts/ci-check-lineends.sh
@@ -2,12 +2,20 @@
 
 FN=$1
 
+ANY_ERRORS=0
+
 LAST_CHAR="$(tail -c 1 $FN | od -A n -t x1)"
 if test "$LAST_CHAR" != " 0a"; then
     echo "File does not end with newline:" $FN
+    ANY_ERRORS=1
 fi
 
 LINEEND="$(dos2unix -i $FN | awk '{print $1 " " $3}')"
 if test "$LINEEND" != "0 0"; then
     echo "File contains non-UNIX line endings:" $FN
+    ANY_ERRORS=1
+fi
+
+if [[ $ANY_ERRORS -ne 0 ]]; then
+    exit 1
 fi
diff --git a/.github/scripts/ci-checkout.sh b/.github/scripts/ci-checkout.sh
@@ -1,14 +1,10 @@
-# ci-checkout.sh
 set -ex
 
 . $(dirname "$0")/common.sh
 
 OPENJDK_URL=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python3 -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["openjdk"]["openjdk_repo"])'`
 OPENJDK_VERSION=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python3 -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["openjdk"]["openjdk_version"])'`
 
-# Use default openjdk path if it is not set (this is only need when MMTk core calls the script)
-OPENJDK_PATH=${OPENJDK_PATH:="$BINDING_PATH/repos/openjdk"}
-
 rm -rf $OPENJDK_PATH
 git clone $OPENJDK_URL $OPENJDK_PATH
 git -C $OPENJDK_PATH checkout $OPENJDK_VERSION
diff --git a/.github/scripts/ci-setup.sh b/.github/scripts/ci-setup.sh
@@ -2,7 +2,7 @@ set -xe
 
 . $(dirname "$0")/common.sh
 
-# Install nightly rust
+# Install the appropriate Rust toolchain
 rustup toolchain install $RUSTUP_TOOLCHAIN
 rustup target add i686-unknown-linux-gnu --toolchain $RUSTUP_TOOLCHAIN
 rustup component add clippy --toolchain $RUSTUP_TOOLCHAIN

diff --git a/.github/scripts/ci-style.sh b/.github/scripts/ci-style.sh
@@ -11,11 +11,26 @@ cargo clippy --release
 cargo fmt -- --check
 popd
 
-find $BINDING_PATH/openjdk \
+# Allow all files to be checked, and exit if any went wrong.
+set +xe
+
+ANY_LINE_END_ERRORS=0
+for FN in $(find $BINDING_PATH/openjdk \
     $BINDING_PATH/mmtk \
+    '(' \
     -name '*.hpp' \
     -o -name '*.cpp' \
     -o -name '*.rs' \
     -o -name '*.toml' \
     -o -name '*.gmk' \
-    -exec ./ci-check-lineends.sh '{}' \;
+    ')')
+do
+    $(dirname "$0")/ci-check-lineends.sh "$FN"
+    if [[ $? -ne 0 ]]; then
+        ANY_LINE_END_ERRORS=1
+    fi
+done
+
+if [[ $ANY_LINE_END_ERRORS -ne 0 ]]; then
+    exit 1
+fi
diff --git a/.github/scripts/ci-test-extended.sh b/.github/scripts/ci-test-extended.sh
@@ -1,21 +1,10 @@
 set -ex
 
-. $(dirname "$0")/common.sh
-cur=$BINDING_PATH/.github/scripts
-
-# This script is only used by MMTk core.
-# OPENJDK_PATH is the default path set in ci-checkout.sh
-export OPENJDK_PATH=$BINDING_PATH/repos/openjdk
-export DEBUG_LEVEL=fastdebug
-export TEST_JAVA_BIN=$OPENJDK_PATH/build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java
-
-# Download dacapo
-export DACAPO_PATH=$BINDING_PATH/dacapo
-mkdir -p $DACAPO_PATH
-wget https://downloads.sourceforge.net/project/dacapobench/archive/2006-10-MR2/dacapo-2006-10-MR2.jar -O $DACAPO_PATH/dacapo-2006-10-MR2.jar
+. $(dirname "$0")/common-binding-test.sh
 
 # Normal build
 $cur/ci-build.sh
+
 # Test
 $cur/ci-test-only-normal.sh
 $cur/ci-test-only-normal-no-compressed-oops.sh

diff --git a/.github/scripts/ci-test-minimal.sh b/.github/scripts/ci-test-minimal.sh
@@ -1,21 +1,10 @@
 set -ex
 
-. $(dirname "$0")/common.sh
-cur=$BINDING_PATH/.github/scripts
-
-# This script is only used by MMTk core.
-# OPENJDK_PATH is the default path set in ci-checkout.sh
-export OPENJDK_PATH=$BINDING_PATH/repos/openjdk
-export DEBUG_LEVEL=fastdebug
-export TEST_JAVA_BIN=$OPENJDK_PATH/build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java
-
-# Download dacapo
-export DACAPO_PATH=$BINDING_PATH/dacapo
-mkdir -p $DACAPO_PATH
-wget https://downloads.sourceforge.net/project/dacapobench/archive/2006-10-MR2/dacapo-2006-10-MR2.jar -O $DACAPO_PATH/dacapo-2006-10-MR2.jar
+. $(dirname "$0")/common-binding-test.sh
 
 # Normal build
 $cur/ci-build.sh
+
 # Test
 MMTK_PLAN=SemiSpace runbms_dacapo2006_with_heap_multiplier fop 4
 MMTK_PLAN=Immix runbms_dacapo2006_with_heap_multiplier fop 4

diff --git a/.github/scripts/ci-test-only-normal-no-compressed-oops.sh b/.github/scripts/ci-test-only-normal-no-compressed-oops.sh
@@ -4,6 +4,10 @@ set -xe
 
 unset JAVA_TOOL_OPTIONS
 
+# This script runs against a specific pre-built java binary, not OpenJDK source code.
+ensure_env TEST_JAVA_BIN
+unset OPENJDK_PATH
+
 run_all_no_compressed_oop() {
     heap_multiplier=$1
 

diff --git a/.github/scripts/ci-test-only-normal.sh b/.github/scripts/ci-test-only-normal.sh
@@ -4,6 +4,10 @@ set -xe
 
 unset JAVA_TOOL_OPTIONS
 
+# This script runs against a specific pre-built java binary, not OpenJDK source code.
+ensure_env TEST_JAVA_BIN
+unset OPENJDK_PATH
+
 run_all() {
     heap_multiplier=$1
 

diff --git a/.github/scripts/ci-test-only-weak-ref.sh b/.github/scripts/ci-test-only-weak-ref.sh
@@ -3,7 +3,10 @@ set -xe
 . $(dirname "$0")/common.sh
 
 unset JAVA_TOOL_OPTIONS
-cd $OPENJDK_PATH
+
+# This script runs against a specific pre-built java binary, not OpenJDK source code.
+ensure_env TEST_JAVA_BIN
+unset OPENJDK_PATH
 
 export MMTK_NO_REFERENCE_TYPES=false
 # Just test Immix and MarkCompact

diff --git a/.github/scripts/common-binding-test.sh b/.github/scripts/common-binding-test.sh
@@ -0,0 +1,19 @@
+# This is the common part of binding tests (minimal and extended) invoked from mmtk-core.
+
+set -xe
+
+. $(dirname "$0")/common.sh
+cur=$BINDING_PATH/.github/scripts
+
+# Use fastdebug for binding tests
+export DEBUG_LEVEL=fastdebug
+
+# We build OpenJDK with the "default" target, and this is the location of the Java binary.
+# It's equivalent to invoking "make" without an explicit target, but it's better to be explicit.
+export OPENJDK_BUILD_TARGET=default
+export TEST_JAVA_BIN=$OPENJDK_PATH/build/linux-x86_64-server-$DEBUG_LEVEL/jdk/bin/java
+
+# Download dacapo
+export DACAPO_PATH=$BINDING_PATH/dacapo
+mkdir -p $DACAPO_PATH
+wget https://downloads.sourceforge.net/project/dacapobench/archive/2006-10-MR2/dacapo-2006-10-MR2.jar -O $DACAPO_PATH/dacapo-2006-10-MR2.jar
diff --git a/.github/scripts/common.sh b/.github/scripts/common.sh
@@ -1,6 +1,10 @@
 BINDING_PATH=$(realpath $(dirname "$0"))/../..
 RUSTUP_TOOLCHAIN=`cat $BINDING_PATH/mmtk/rust-toolchain`
 
+# build.yml specifies the OPENJDK_PATH env var when calling ci-build.sh
+# But other scripts expect a default path for OpenJDK.
+OPENJDK_PATH=${OPENJDK_PATH:="$BINDING_PATH/repos/jdk"}
+
 # dacapo2006 min heap for mark compact
 MINHEAP_ANTLR=5
 MINHEAP_FOP=13

diff --git a/.github/scripts/pgo-build.sh b/.github/scripts/pgo-build.sh
@@ -11,20 +11,20 @@ cargo clean
 popd
 
 # Compile with profiling support
-RUSTFLAGS="-Cprofile-generate=/tmp/$USER/pgo-data" make CONF=linux-x86_64-normal-server-release THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk images
+RUSTFLAGS="-Cprofile-generate=/tmp/$USER/pgo-data" make CONF=linux-x86_64-server-release THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk images
 
 # Remove extraneous profiling data
 rm -rf /tmp/$USER/pgo-data/*
 
 # Profile using fop
-MMTK_PLAN=GenImmix MMTK_STRESS_FACTOR=16777216 MMTK_PRECISE_STRESS=false ./build/linux-x86_64-normal-server-release/images/jdk/bin/java -XX:MetaspaceSize=500M -XX:+DisableExplicitGC -XX:-TieredCompilation -Xcomp -XX:+UseThirdPartyHeap -Xms60M -Xmx60M -jar /usr/share/benchmarks/dacapo/dacapo-23.9-RC3-chopin.jar -n 5 fop
+MMTK_PLAN=GenImmix MMTK_STRESS_FACTOR=4194304 MMTK_PRECISE_STRESS=false ./build/linux-x86_64-server-release/images/jdk/bin/java -XX:MetaspaceSize=500M -XX:+DisableExplicitGC -XX:-TieredCompilation -Xcomp -XX:+UseThirdPartyHeap -Xms60M -Xmx60M -jar /usr/share/benchmarks/dacapo/dacapo-23.11-chopin.jar -n 5 fop
 
 # Merge profiling data
-/opt/rust/toolchains/1.71.1-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o /tmp/$USER/pgo-data/merged.profdata /tmp/$USER/pgo-data
+/opt/rust/toolchains/1.83.0-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o /tmp/$USER/pgo-data/merged.profdata /tmp/$USER/pgo-data
 
 pushd ../mmtk-openjdk/mmtk
 cargo clean
 popd
 
 # Compile using profiling data
-RUSTFLAGS="-Cprofile-use=/tmp/$USER/pgo-data/merged.profdata -Cllvm-args=-pgo-warn-missing-function" make CONF=linux-x86_64-normal-server-release THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk images
+RUSTFLAGS="-Cprofile-use=/tmp/$USER/pgo-data/merged.profdata -Cllvm-args=-pgo-warn-missing-function" make CONF=linux-x86_64-server-release THIRD_PARTY_HEAP=$PWD/../mmtk-openjdk/openjdk images
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -20,24 +20,19 @@ jobs:
   build-linux-x64:
     name: linux-x64
     runs-on: ubuntu-22.04
+    env:
+      # This overrides OPENJDK_PATH in ci-common.sh
+      # Note: We cannot use default environment variables (GITHUB_*) in the "env" context.  Use github.* instead.
+      # See: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#default-environment-variables
+      OPENJDK_PATH: ${{ github.workspace }}/git/openjdk
     steps:
     - name: Checkout MMTk OpenJDK binding
       uses: actions/checkout@v4
       with:
         path: ./git/mmtk-openjdk
-    - name: Extract OpenJDK revision
-      id: extract-openjdk-revision
-      run: |
-        OPENJDK_VERSION=`sed -n 's/^openjdk_version.=."\(.*\)"$/\1/p' < git/mmtk-openjdk/mmtk/Cargo.toml`
-        echo "openjdk-revision=$OPENJDK_VERSION" >> $GITHUB_OUTPUT
     - name: Checkout OpenJDK
-      uses: actions/checkout@v4
-      with:
-        repository: mmtk/openjdk
-        path: ./git/openjdk
-        ref: ${{ steps.extract-openjdk-revision.outputs.openjdk-revision }}
-    # Patch mmtk-core version if needed
-    - name: Patch mmtk-core version
+      run: ./git/mmtk-openjdk/.github/scripts/ci-checkout.sh
+    - name: Patch mmtk-core version if needed
       if: ${{ inputs.mmtk-core-repo != '' || inputs.mmtk-core-ref != '' }}
       run: |
         pip3 install tomlkit
@@ -60,14 +55,16 @@ jobs:
           echo "BUILD_SUFFIX=$escaped_build" >> $GITHUB_ENV
         fi
     - name: Build MMTk OpenJDK ${{ inputs.debug-level }}
-      run: |
-        OPENJDK_PATH=$GITHUB_WORKSPACE/git/openjdk DEBUG_LEVEL=${{ inputs.debug-level }} OPENJDK_BUILD_TARGET=product-bundles ./.github/scripts/ci-build.sh
+      env:
+        DEBUG_LEVEL: ${{ inputs.debug-level }}
+        OPENJDK_BUILD_TARGET: product-bundles
+      run: ./.github/scripts/ci-build.sh
       working-directory: ./git/mmtk-openjdk
     - name: Upload bundles
       uses: actions/upload-artifact@v4
       with:
         name: linux-x86_64-server-${{ inputs.debug-level }}-bundles-${{ env.BUILD_SUFFIX }}
         path: |
-          ./git/openjdk/build/linux-x86_64-normal-server-${{ inputs.debug-level }}/bundles/*_bin.tar.gz
-          ./git/openjdk/build/linux-x86_64-normal-server-${{ inputs.debug-level }}/bundles/*_bin-debug.tar.gz
+          ./git/openjdk/build/linux-x86_64-server-${{ inputs.debug-level }}/bundles/*_bin.tar.gz
+          ./git/openjdk/build/linux-x86_64-server-${{ inputs.debug-level }}/bundles/*_bin-debug.tar.gz
         retention-days: 2
diff --git a/.github/workflows/run-dacapo-chopin.yml b/.github/workflows/run-dacapo-chopin.yml
@@ -4,9 +4,9 @@ on:
   workflow_call:
 
 env:
-  DACAPO_VERSION: dacapo-23.9-RC3-chopin
-  DACAPO_FILE: dacapo-23.9-RC3-chopin.zip
-  DACAPO_DOWNLOAD_URL: https://download.dacapobench.org/chopin/dacapo-23.9-RC3-chopin.zip
+  DACAPO_VERSION: dacapo-23.11-MR2-chopin
+  DACAPO_FILE: dacapo-23.11-MR2-chopin.zip
+  DACAPO_DOWNLOAD_URL: https://download.dacapobench.org/chopin/dacapo-23.11-MR2-chopin.zip
 
 jobs:
   cache-dacapo: