Skip to content

Commit 740237d

Browse files
cwillisfcwillisf
committed
build: create fresh monorepo as a subdirectory
1 parent 8e8b60a commit 740237d

File tree

2 files changed

+84
-36
lines changed

2 files changed

+84
-36
lines changed

‎.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
# build-monorepo.sh
2+
/monorepo.out
3+
/monorepo.tmp
4+
/monorepo.cache
5+
16
# Logs
27
logs
38
*.log

‎build-monorepo.sh

+79-36
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,10 @@
22

33
### Configuration ###
44

5-
# This script will use local copies of the repositories to reduce network traffic
6-
MY_REPOS="$HOME/GitHub"
7-
85
# All repositories are assumed to be hosted in this GitHub org
96
GITHUB_ORG="scratchfoundation"
107

8+
# This is the list of repositories to merge into the monorepo
119
ALL_REPOS="
1210
scratch-audio \
1311
scratch-blocks \
@@ -27,10 +25,43 @@ ALL_REPOS="
2725
paper.js \
2826
"
2927

28+
# This is the directory where you have a copy of all the repositories you want to merge.
29+
# This script will run `git fetch` on these repos, but otherwise will not modify them.
30+
BUILD_CACHE="./monorepo.cache"
31+
32+
# The monorepo will be built here. Delete it to start over.
33+
BUILD_OUT="./monorepo.out"
34+
35+
# Temporary clones will be placed here. If the script completes successfully, this directory will be deleted.
36+
BUILD_TMP="./monorepo.tmp"
37+
38+
# Use ${BASE_COMMIT} from ${BASE_REPO} as the starting point for the monorepo.
39+
BASE_COMMIT="$(git rev-parse develop)"
40+
BASE_REPO="scratch-editor"
41+
3042
### End configuration ###
3143

3244
set -e
3345

46+
if [ ! -d "$BUILD_CACHE" ]; then
47+
echo "Please link $BUILD_CACHE to a directory with a copy of all the repositories you want to merge."
48+
echo "For example, if you have ~/GitHub/scratch-audio, ~/GitHub/scratch-blocks, etc., then run:"
49+
echo "ln -s ~/GitHub $BUILD_CACHE"
50+
exit 1
51+
fi
52+
53+
if [ -d "$BUILD_TMP" ]; then
54+
echo "Please remove $BUILD_TMP before running this script."
55+
echo "You may want: rm -rf $BUILD_TMP $BUILD_OUT"
56+
exit 1
57+
fi
58+
59+
if [ -d "$BUILD_OUT" ]; then
60+
echo "Please remove $BUILD_OUT before running this script."
61+
echo "You may want: rm -rf $BUILD_TMP $BUILD_OUT"
62+
exit 1
63+
fi
64+
3465
# Thanks to https://stackoverflow.com/a/17841619
3566
join_args () {
3667
local d=${1-} f=${2-}
@@ -39,43 +70,49 @@ join_args () {
3970
fi
4071
}
4172

73+
init_monorepo () {
74+
git init "$BUILD_OUT"
75+
git -C "$BUILD_OUT" remote add origin "[email protected]:${GITHUB_ORG}/${BASE_REPO}.git"
76+
git -C "$BUILD_OUT" fetch --all
77+
git -C "$BUILD_OUT" checkout -b develop "$BASE_COMMIT"
78+
}
79+
4280
add_repo_to_monorepo () {
4381
REPO_NAME="$1"
4482
ORG_AND_REPO_NAME="${GITHUB_ORG}/${REPO_NAME}"
4583
echo "Working on $ORG_AND_REPO_NAME"
46-
mkdir -p tmp
4784

4885
#
4986
# Clone
5087
#
5188

5289
# refresh the cache
53-
git -C "${MY_REPOS}/${REPO_NAME}" fetch --all
90+
git -C "${BUILD_CACHE}/${REPO_NAME}" fetch --all
5491
# reference = go faster
55-
git -C tmp clone --single-branch --reference "$MY_REPOS"/"$REPO_NAME" "[email protected]:${ORG_AND_REPO_NAME}.git"
92+
git -C "$BUILD_TMP" clone --single-branch --reference "$(realpath "$BUILD_CACHE")/${REPO_NAME}" "[email protected]:${ORG_AND_REPO_NAME}.git"
5693
# get ready to disconnect reference repo
57-
git -C "tmp/${REPO_NAME}" repack -a
94+
git -C "${BUILD_TMP}/${REPO_NAME}" repack -a
5895
# actually disconnect the reference repo
59-
rm -f "tmp/${REPO_NAME}/.git/objects/info/alternates"
96+
rm -f "${BUILD_TMP}/${REPO_NAME}/.git/objects/info/alternates"
6097

6198
#
6299
# Move to subdirectory
63100
#
64101

65102
# make filter-repo accept this as a fresh clone
66-
git -C "tmp/${REPO_NAME}" gc
103+
git -C "${BUILD_TMP}/${REPO_NAME}" gc
67104

68105
# rewrite history as if all this work happened in a subdirectory
69106
# "git mv" is simpler but makes history less visible unless you explicitly use "--follow"
70-
if [ ! -f "tmp/${REPO_NAME}/.gitmodules" ]; then
107+
if [ ! -f "${BUILD_TMP}/${REPO_NAME}/.gitmodules" ]; then
71108
# this is significantly faster than the special case below
72-
git -C "tmp/${REPO_NAME}" filter-repo --to-subdirectory-filter "workspaces/$REPO_NAME"
109+
git -C "${BUILD_TMP}/${REPO_NAME}" filter-repo --to-subdirectory-filter "workspaces/$REPO_NAME"
73110
else
74111
# the .gitmodules file must stay in the repository root, but the paths inside it must be rewritten
75112
# this is complicated for the reasons described here: https://github.com/newren/git-filter-repo/issues/158
76113
# this is also slower, so we only do it for repositories that have submodules
77114
# if we have more than one, this will cause merge conflicts
78-
git -C "tmp/${REPO_NAME}" filter-repo \
115+
git -C "${BUILD_TMP}/${REPO_NAME}" filter-repo \
79116
--filename-callback "return filename if filename == b'.gitmodules' else b'workspaces/${REPO_NAME}/'+filename" \
80117
--blob-callback "if blob.data.startswith(b'[submodule '): blob.data = blob.data.replace(b'path = ', b'path = workspaces/${REPO_NAME}/')"
81118
fi
@@ -84,26 +121,26 @@ add_repo_to_monorepo () {
84121
# Merge it in
85122
#
86123

87-
BRANCH="$(git -C "tmp/${REPO_NAME}" branch --format="%(refname:short)")"
124+
BRANCH="$(git -C "${BUILD_TMP}/${REPO_NAME}" branch --format="%(refname:short)")"
88125

89-
git remote add "temp-$REPO_NAME" "tmp/${REPO_NAME}"
90-
git fetch --no-tags "temp-$REPO_NAME"
91-
git merge --allow-unrelated-histories --no-verify -m "chore(deps): add workspaces/$REPO_NAME" "temp-$REPO_NAME/$BRANCH"
92-
git remote remove "temp-$REPO_NAME"
93-
rm -rf "tmp/${REPO_NAME}"
126+
git -C "$BUILD_OUT" remote add "temp-$REPO_NAME" "$(realpath "${BUILD_TMP}")/${REPO_NAME}"
127+
git -C "$BUILD_OUT" fetch --no-tags "temp-$REPO_NAME"
128+
git -C "$BUILD_OUT" merge --allow-unrelated-histories --no-verify -m "chore(deps): add workspaces/$REPO_NAME" "temp-$REPO_NAME/$BRANCH"
129+
git -C "$BUILD_OUT" remote remove "temp-$REPO_NAME"
130+
rm -rf "${BUILD_TMP}/${REPO_NAME}"
94131
}
95132

96133
fixup_current_branch () {
97134
# submodules could be necessary for build/test scripts
98-
git submodule update --init --recursive
135+
git -C "$BUILD_OUT" submodule update --init --recursive
99136

100137
# remove repository-level configuration and dependencies, like commitlint
101138
# do not remove configuration and dependencies that could vary between packages, like semantic-release
102139
# some of these files only make sense in the root of the repository
103140
# others could be in subdirectories, like .editorconfig, but centralizing them makes consistency easier
104141
# it would be nice to merge all the package-lock.json files into one but it's not clear how to do that
105142
# just remove the package-lock.json files for now, and build a new one with "npm i" later
106-
rm -rf workspaces/*/{.circleci,.editorconfig,.gitattributes,.github,.husky,package-lock.json,renovate.json*}
143+
rm -rf "$BUILD_OUT"/workspaces/*/{.circleci,.editorconfig,.gitattributes,.github,.husky,package-lock.json,renovate.json*}
107144
for REPO in $ALL_REPOS; do
108145
jq -f <(join_args ' | ' \
109146
'if .scripts.prepare == "husky install" then del(.scripts.prepare) else . end' \
@@ -116,14 +153,14 @@ fixup_current_branch () {
116153
'del(.devDependencies."cz-conventional-changelog")' \
117154
'del(.devDependencies."husky")' \
118155
'if .devDependencies == {} then del(.devDependencies) else . end' \
119-
) "workspaces/${REPO}/package.json" | sponge "workspaces/${REPO}/package.json"
156+
) "${BUILD_OUT}/workspaces/${REPO}/package.json" | sponge "${BUILD_OUT}/workspaces/${REPO}/package.json"
120157
done
121-
git commit -m "chore: remove repo-level configuration and deps from workspaces/*" \
158+
git -C "$BUILD_OUT" commit -m "chore: remove repo-level configuration and deps from workspaces/*" \
122159
workspaces
123160

124-
npm i
125-
npm i --package-lock-only # sometimes this is necessary to get a consistent package-lock.json
126-
git commit -m "chore(deps): build initial real package-lock.json" \
161+
npm -C "$BUILD_OUT" i
162+
npm -C "$BUILD_OUT" i --package-lock-only # sometimes this is necessary to get a consistent package-lock.json
163+
git -C "$BUILD_OUT" commit -m "chore(deps): build initial real package-lock.json" \
127164
package.json package-lock.json
128165

129166
for REPO in $ALL_REPOS; do
@@ -133,50 +170,56 @@ fixup_current_branch () {
133170
OPTDEPS=""
134171
PEERDEPS=""
135172
for DEP in $ALL_REPOS; do
136-
if jq -e .dependencies.\"$DEP\" "workspaces/${REPO}/package.json" > /dev/null; then
173+
if jq -e .dependencies.\"$DEP\" "${BUILD_OUT}/workspaces/${REPO}/package.json" > /dev/null; then
137174
REMOVEDEPS="$REMOVEDEPS $DEP"
138175
DEPS="$DEPS $DEP@*"
139176
fi
140-
if jq -e .devDependencies.\"$DEP\" "workspaces/${REPO}/package.json" > /dev/null; then
177+
if jq -e .devDependencies.\"$DEP\" "${BUILD_OUT}/workspaces/${REPO}/package.json" > /dev/null; then
141178
REMOVEDEPS="$REMOVEDEPS $DEP"
142179
DEVDEPS="$DEVDEPS $DEP@*"
143180
fi
144-
if jq -e .optionalDependencies.\"$DEP\" "workspaces/${REPO}/package.json" > /dev/null; then
181+
if jq -e .optionalDependencies.\"$DEP\" "${BUILD_OUT}/workspaces/${REPO}/package.json" > /dev/null; then
145182
REMOVEDEPS="$REMOVEDEPS $DEP"
146183
OPTDEPS="$OPTDEPS $DEP@*"
147184
fi
148-
if jq -e .peerDependencies.\"$DEP\" "workspaces/${REPO}/package.json" > /dev/null; then
185+
if jq -e .peerDependencies.\"$DEP\" "${BUILD_OUT}/workspaces/${REPO}/package.json" > /dev/null; then
149186
REMOVEDEPS="$REMOVEDEPS $DEP"
150187
PEERDEPS="$PEERDEPS $DEP@*"
151188
fi
152189
done
153190
if [ -n "$REMOVEDEPS" ]; then
154-
npm uninstall $REMOVEDEPS -w $REPO
191+
npm -C $BUILD_OUT uninstall $REMOVEDEPS -w "$REPO"
155192
if [ -n "$DEPS" ]; then
156-
npm i --save --save-exact $DEPS -w $REPO
193+
npm -C "$BUILD_OUT" i --save --save-exact $DEPS -w "$REPO"
157194
fi
158195
if [ -n "$DEVDEPS" ]; then
159-
npm i --save-dev --save-exact $DEVDEPS -w $REPO
196+
npm -C "$BUILD_OUT" i --save-dev --save-exact $DEVDEPS -w "$REPO"
160197
fi
161198
if [ -n "$OPTDEPS" ]; then
162-
npm i --save-optional --save-exact $OPTDEPS -w $REPO
199+
npm -C "$BUILD_OUT" i --save-optional --save-exact $OPTDEPS -w "$REPO"
163200
fi
164201
if [ -n "$PEERDEPS" ]; then
165-
npm i --save-peer --save-exact $PEERDEPS -w $REPO
202+
npm -C "$BUILD_OUT" i --save-peer --save-exact $PEERDEPS -w "$REPO"
166203
fi
167204
fi
168205
done
169206

170-
git commit -m "chore(deps): use workspace versions of all local packages" \
207+
git -C "$BUILD_OUT" commit -m "chore(deps): use workspace versions of all local packages" \
171208
package.json package-lock.json workspaces/*/package.json
172209
}
173210

174211
### Do the things! ###
175212

213+
init_monorepo
214+
215+
mkdir -p "$BUILD_TMP"
216+
176217
for REPO in $ALL_REPOS; do
177218
add_repo_to_monorepo "$REPO"
178219
done
179220

180-
(rmdir tmp || true) 2> /dev/null
221+
rmdir "$BUILD_TMP"
181222

182223
fixup_current_branch
224+
225+
echo "All done!"

0 commit comments

Comments
 (0)