diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a6dc28189..25878e7083 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -396,11 +396,23 @@ jobs: if: matrix.build_playground run: yarn workspace playground test + - name: Setup Rclone + if: ${{ matrix.build_playground && startsWith(github.ref, 'refs/tags/v') }} + uses: cometkim/rclone-actions/setup-rclone@89de8311dd0ca0b43143329d3daeb1041852ad9f + + - name: Configure Rclone remote + if: ${{ matrix.build_playground && startsWith(github.ref, 'refs/tags/v') }} + uses: cometkim/rclone-actions/configure-remote/s3-provider@89de8311dd0ca0b43143329d3daeb1041852ad9f + with: + name: rescript + provider: Cloudflare + endpoint: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com + access-key-id: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + acl: private + - name: Upload playground compiler to CDN if: ${{ matrix.build_playground && startsWith(github.ref, 'refs/tags/v') }} - env: - KEYCDN_USER: ${{ secrets.KEYCDN_USER }} - KEYCDN_PASSWORD: ${{ secrets.KEYCDN_PASSWORD }} run: yarn workspace playground upload-bundle - name: "Upload artifacts: binaries" diff --git a/Makefile b/Makefile index 99dd18476b..96a166ce9e 100644 --- a/Makefile +++ b/Makefile @@ -69,14 +69,14 @@ artifacts: lib # Builds the core playground bundle (without the relevant cmijs files for the runtime) playground: dune build --profile browser - cp ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js packages/playground/compiler.js + cp -f ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js packages/playground/compiler.js # Creates all the relevant core and third party cmij files to side-load together with the playground bundle playground-cmijs: artifacts yarn workspace playground build # Builds the playground, runs some e2e tests and releases the playground to the -# CDN (requires KEYCDN_USER and KEYCDN_PASSWORD set in the env variables) +# Cloudflare R2 (requires Rclone `rescript:` remote) playground-release: playground playground-cmijs yarn workspace playground test yarn workspace playground upload-bundle diff --git a/package.json b/package.json index acb466fad3..ca1418e676 100644 --- a/package.json +++ b/package.json @@ -84,9 +84,11 @@ "devDependencies": { "@biomejs/biome": "1.9.4", "@types/node": "^20.14.9", + "@types/semver": "^7.7.0", "@yarnpkg/types": "^4.0.1", "mocha": "10.8.2", "nyc": "15.0.0", + "semver": "^7.7.2", "typescript": "5.8.2" }, "workspaces": [ diff --git a/packages/playground/.gitignore b/packages/playground/.gitignore index f20e8d25e1..e979f5b774 100644 --- a/packages/playground/.gitignore +++ b/packages/playground/.gitignore @@ -32,3 +32,5 @@ yarn.lock /packages/ /compiler.js + +.tmp/ diff --git a/packages/playground/package.json b/packages/playground/package.json index 0e43a5a4d3..3131ae618f 100644 --- a/packages/playground/package.json +++ b/packages/playground/package.json @@ -5,8 +5,8 @@ "scripts": { "clean": "rescript clean", "test": "node ./playground_test.cjs", - "build": "rescript clean && rescript build && node ./scripts/generate_cmijs.mjs && rollup -c", - "upload-bundle": "./scripts/upload_bundle.sh", + "build": "rescript clean && rescript build && node scripts/generate_cmijs.mjs && rollup -c", + "upload-bundle": "node scripts/upload_bundle.mjs", "revalidate": "./scripts/website_update_playground.sh" }, "dependencies": { diff --git a/packages/playground/scripts/common.mjs b/packages/playground/scripts/common.mjs new file mode 100644 index 0000000000..f45ea3a51b --- /dev/null +++ b/packages/playground/scripts/common.mjs @@ -0,0 +1,33 @@ +// @ts-check + +import * as child_process from "node:child_process"; +import * as path from "node:path"; + +export const compilerRootDir = path.join( + import.meta.dirname, + "..", + "..", + "..", +); + +// The playground-bundling root dir +export const playgroundDir = path.join(import.meta.dirname, ".."); + +// Final target output directory where all the cmijs will be stored +export const playgroundPackagesDir = path.join(playgroundDir, "packages"); + +/** + * @param {string} cmd + * @param {child_process.ExecSyncOptions} [opts={}] + */ +export function exec(cmd, opts = {}) { + console.log(`>>>>>> running command: ${cmd}`); + const result = child_process.execSync(cmd, { + cwd: playgroundDir, + stdio: "inherit", + ...opts, + encoding: "utf8", + }); + console.log("<<<<<<"); + return result || ""; +} diff --git a/packages/playground/scripts/generate_cmijs.mjs b/packages/playground/scripts/generate_cmijs.mjs index 2fb44d8f05..56b29f50f0 100644 --- a/packages/playground/scripts/generate_cmijs.mjs +++ b/packages/playground/scripts/generate_cmijs.mjs @@ -13,92 +13,54 @@ * playground bundle. */ -import * as child_process from "node:child_process"; import * as fs from "node:fs"; import * as path from "node:path"; import resConfig from "../rescript.json" with { type: "json" }; +import { + exec, + compilerRootDir, + playgroundPackagesDir, +} from "./common.mjs"; -const RESCRIPT_COMPILER_ROOT_DIR = path.join( - import.meta.dirname, - "..", - "..", - "..", -); +exec("yarn rescript clean"); +exec("yarn rescript"); -// The playground-bundling root dir -const PLAYGROUND_DIR = path.join(import.meta.dirname, ".."); - -// Final target output directory where all the cmijs will be stored -const PACKAGES_DIR = path.join(PLAYGROUND_DIR, "packages"); +// We need to build the compiler's builtin modules as a separate cmij. +// Otherwise we can't use them for compilation within the playground. +buildCmij(compilerRootDir, "compiler-builtins"); -// Making sure this directory exists, since it's not checked in to git -fs.mkdirSync(PACKAGES_DIR, { recursive: true }); +const packages = resConfig["bs-dependencies"]; +for (const pkgName of packages) { + buildCmij( + path.join(compilerRootDir, "node_modules", pkgName), + pkgName, + ); +} /** - * @param {string} cmd + * @param {string} pkgDir + * @param {string} pkgName */ -function e(cmd) { - console.log(`>>>>>> running command: ${cmd}`); - child_process.execSync(cmd, { - cwd: PLAYGROUND_DIR, - encoding: "utf8", - stdio: [0, 1, 2], - }); - console.log("<<<<<<"); -} - -e("yarn rescript clean"); -e("yarn rescript"); - -const packages = resConfig["bs-dependencies"]; - -// We need to build the compiler's builtin modules as a separate cmij. -// Otherwise we can't use them for compilation within the playground. -function buildCompilerCmij() { - const rescriptLibOcamlFolder = path.join( - RESCRIPT_COMPILER_ROOT_DIR, +function buildCmij(pkgDir, pkgName) { + const libOcamlFolder = path.join( + pkgDir, "lib", "ocaml", ); - const outputFolder = path.join(PACKAGES_DIR, "compiler-builtins"); + const outputFolder = path.join(playgroundPackagesDir, pkgName); fs.mkdirSync(outputFolder, { recursive: true }); const cmijFile = path.join(outputFolder, "cmij.js"); - - e( - `find ${rescriptLibOcamlFolder} -name "*.cmi" -or -name "*.cmj" | xargs -n1 basename | xargs js_of_ocaml build-fs -o ${cmijFile} -I ${rescriptLibOcamlFolder}`, - ); + const inputFiles = fs.readdirSync(libOcamlFolder).filter(isCmij).join(" "); + exec(`js_of_ocaml build-fs -o ${cmijFile} -I ${libOcamlFolder} ${inputFiles}`); } -function buildThirdPartyCmijs() { - for (const pkg of packages) { - const libOcamlFolder = path.join( - RESCRIPT_COMPILER_ROOT_DIR, - "node_modules", - pkg, - "lib", - "ocaml", - ); - const libEs6Folder = path.join( - RESCRIPT_COMPILER_ROOT_DIR, - "node_modules", - pkg, - "lib", - "es6", - ); - const outputFolder = path.join(PACKAGES_DIR, pkg); - fs.mkdirSync(outputFolder, { recursive: true }); - - const cmijFile = path.join(outputFolder, "cmij.js"); - - e(`find ${libEs6Folder} -name '*.js' -exec cp {} ${outputFolder} \\;`); - e( - `find ${libOcamlFolder} -name "*.cmi" -or -name "*.cmj" | xargs -n1 basename | xargs js_of_ocaml build-fs -o ${cmijFile} -I ${libOcamlFolder}`, - ); - } +/** + * @param {string} basename + * @return {boolean} + */ +function isCmij(basename) { + return /\.cm(i|j)$/.test(basename); } - -buildCompilerCmij(); -buildThirdPartyCmijs(); diff --git a/packages/playground/scripts/upload_bundle.mjs b/packages/playground/scripts/upload_bundle.mjs new file mode 100644 index 0000000000..265e08837c --- /dev/null +++ b/packages/playground/scripts/upload_bundle.mjs @@ -0,0 +1,129 @@ +#!/usr/bin/env node + +// @ts-check + +// This script will publish the compiler.js bundle / packages cmij.js files to our Cloudclare R2. +// The target folder on R2 bucket will be the compiler.js' version number. +// This script requires `rclone` and `zstd` to be installed. + +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as readline from "node:readline/promises"; +import { createRequire } from "node:module"; +import semver from "semver"; + +import { + exec, + playgroundDir, + playgroundPackagesDir, +} from "./common.mjs"; + +const require = createRequire(import.meta.url); +// @ts-ignore +const { rescript_compiler } = require('../compiler.js'); + +/** + * @type {number} + */ +const version = rescript_compiler.make().rescript.version; +const tag = "v" + version; + +console.log("Uploading playground artifacts for %s", tag); +if (!process.env.CI) { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + const answer = await rl.question("Do you want to proceed? [y/N]: "); + rl.close(); + + if (answer.toLowerCase() !== "y") { + console.log("Cancelled"); + process.exit(1); + } +} + +const rcloneOpts = (process.env.CI + ? [ + "--stats 5", + "--checkers 5000", + "--transfers 8", + "--buffer-size 128M", + "--s3-chunk-size 128M", + "--s3-upload-concurrency 8", + ] + : [ + "--progress", + "--checkers 5000", + "--transfers 16", + "--buffer-size 128M", + "--s3-chunk-size 128M", + "--s3-upload-concurrency 16", + ]).join(" "); + +const remote = process.env.RCLONE_REMOTE || "rescript"; +const bucket = "cdn-assets"; + +// Create a temporary directory for bundling +const tmpDir = path.join(playgroundDir, ".tmp"); +const artifactsDir = path.join(tmpDir, tag); +const archivePath = path.join(tmpDir, `${tag}.tar.zst`); +fs.rmSync(tmpDir, { recursive: true, force: true }); +fs.mkdirSync(artifactsDir, { recursive: true }); + +console.log("Copying compiler.js"); +fs.copyFileSync( + path.join(playgroundDir, "compiler.js"), + path.join(artifactsDir, "compiler.js"), +); + +console.log("Copying packages"); +fs.cpSync(playgroundPackagesDir, artifactsDir, { recursive: true }); + +// Create tar.zst archive +console.log("Creating archive..."); +exec(`tar \\ + --use-compress-program="zstd -T0 --adapt --exclude-compressed" \\ + -cf "${archivePath}" \\ + -C "${artifactsDir}" . +`); + +// TODO: This will no longer be needed after the KeyCDN to Cloudflare transition is complete. +console.log(`Uploading v${version} artifacts...`); +exec(`rclone sync ${rcloneOpts} --fast-list \\ + "${artifactsDir}" \\ + "${remote}:${bucket}/${tag}" +`); + +console.log("Uploading archive..."); +exec(`rclone copyto ${rcloneOpts} \\ + "${archivePath}" \\ + "${remote}:${bucket}/playground-bundles/${tag}.tar.zst" +`); + +console.log("Update versions.json"); +{ + const bundles = exec( + `rclone lsf --include="*.tar.zst" rescript:cdn-assets/playground-bundles`, + { stdio: ['ignore', 'pipe', 'pipe'] }, + ).trimEnd().split("\n"); + + /** @type {string[]} */ + let playgroundVersions = []; + for (const bundle of bundles) { + playgroundVersions.push(bundle.replace(".tar.zst", "")); + } + playgroundVersions = semver.sort(playgroundVersions); + + console.log('Versions: ', playgroundVersions); + + const versionsPath = path.join(tmpDir, "versions.json"); + fs.writeFileSync( + versionsPath, + JSON.stringify(playgroundVersions), + ); + exec(`rclone copyto ${rcloneOpts} \\ + "${versionsPath}" \\ + "${remote}:${bucket}/playground-bundles/versions.json" + `); +} diff --git a/packages/playground/scripts/upload_bundle.sh b/packages/playground/scripts/upload_bundle.sh deleted file mode 100755 index 28c61ab55e..0000000000 --- a/packages/playground/scripts/upload_bundle.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -# This script will publish the compiler.js bundle / packages cmij.js files to our KeyCDN server. -# The target folder on KeyCDN will be the compiler.js' version number. -# This script requires `curl` / `openssl` to be installed. - -SCRIPT_PATH="${BASH_SOURCE[0]}" -PLAYGROUND_DIR="$(dirname "$(dirname "$SCRIPT_PATH")")" - -# Get the actual version from the compiled playground bundle -VERSION="$(node -e "console.log(require('$PLAYGROUND_DIR/compiler.js').rescript_compiler.make().rescript.version)")" - -if [ -z "${KEYCDN_USER}" ]; then - echo "KEYCDN_USER environment variable not set. Make sure to set the environment accordingly." - exit 1 -fi - -if [ -z "${KEYCDN_PASSWORD}" ]; then - echo "KEYCDN_PASSWORD environment variable not set. Make sure to set the environment accordingly." - exit 1 -fi - -KEYCDN_SRV="ftp.keycdn.com" -NETRC_FILE="${PLAYGROUND_DIR}/.netrc" - -# To make sure to not leak any secrets in the bash history, we create a NETRC_FILE -# with the credentials provided via ENV variables. -if [ ! -f "${NETRC_FILE}" ]; then - echo "No .netrc file found. Creating file '${NETRC_FILE}'" - echo "machine ${KEYCDN_SRV} login $KEYCDN_USER password $KEYCDN_PASSWORD" > "${NETRC_FILE}" -fi - -PACKAGES=("compiler-builtins" "@rescript/react") - -echo "Uploading compiler.js file..." -curl --ftp-create-dirs -T "${PLAYGROUND_DIR}/compiler.js" --ssl --tls-max 1.2 --netrc-file $NETRC_FILE ftp://${KEYCDN_SRV}/v${VERSION}/compiler.js - -echo "---" -echo "Uploading packages cmij files..." -for dir in ${PACKAGES[@]}; -do - SOURCE="${PLAYGROUND_DIR}/packages/${dir}" - TARGET="ftp://${KEYCDN_SRV}/v${VERSION}/${dir}" - - echo "Uploading '$SOURCE/cmij.js' to '$TARGET/cmij.js'..." - - curl --ftp-create-dirs -T "${SOURCE}/cmij.js" --ssl --tls-max 1.2 --netrc-file $NETRC_FILE "${TARGET}/cmij.js" -done - -# we now upload the bundled stdlib runtime files - -DIR="compiler-builtins/stdlib" -SOURCE="${PLAYGROUND_DIR}/packages/${DIR}" -TARGET="ftp://${KEYCDN_SRV}/v${VERSION}/${DIR}" - -echo "Uploading '$SOURCE/*.js' to '$TARGET/*.js'..." - -# we use TLS 1.2 because 1.3 sometimes causes data losses (files capped at 16384B) -# https://github.com/curl/curl/issues/6149#issuecomment-1618591420 -find "${SOURCE}" -type f -name "*.js" -exec sh -c 'curl --ftp-create-dirs --ssl --tls-max 1.2 --netrc-file "$0" -T "$1" "${2}/$(basename "$1")"' "$NETRC_FILE" {} "$TARGET" \; diff --git a/yarn.lock b/yarn.lock index 3749dcd9a2..a0ea540067 100644 --- a/yarn.lock +++ b/yarn.lock @@ -747,6 +747,13 @@ __metadata: languageName: node linkType: hard +"@types/semver@npm:^7.7.0": + version: 7.7.0 + resolution: "@types/semver@npm:7.7.0" + checksum: 10c0/6b5f65f647474338abbd6ee91a6bbab434662ddb8fe39464edcbcfc96484d388baad9eb506dff217b6fc1727a88894930eb1f308617161ac0f376fe06be4e1ee + languageName: node + linkType: hard + "@yarnpkg/types@npm:^4.0.1": version: 4.0.1 resolution: "@yarnpkg/types@npm:4.0.1" @@ -2483,9 +2490,11 @@ __metadata: "@rescript/linux-x64": "workspace:packages/@rescript/linux-x64" "@rescript/win32-x64": "workspace:packages/@rescript/win32-x64" "@types/node": "npm:^20.14.9" + "@types/semver": "npm:^7.7.0" "@yarnpkg/types": "npm:^4.0.1" mocha: "npm:10.8.2" nyc: "npm:15.0.0" + semver: "npm:^7.7.2" typescript: "npm:5.8.2" dependenciesMeta: "@rescript/darwin-arm64": @@ -2685,6 +2694,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:^7.7.2": + version: 7.7.2 + resolution: "semver@npm:7.7.2" + bin: + semver: bin/semver.js + checksum: 10c0/aca305edfbf2383c22571cb7714f48cadc7ac95371b4b52362fb8eeffdfbc0de0669368b82b2b15978f8848f01d7114da65697e56cd8c37b0dab8c58e543f9ea + languageName: node + linkType: hard + "serialize-javascript@npm:^6.0.2": version: 6.0.2 resolution: "serialize-javascript@npm:6.0.2"