Skip to content

Commit f677787

Browse files
adrians5jadrians5j
authored
fix: direct download of extensions (#4206)
1 parent a034764 commit f677787

35 files changed

+783
-348
lines changed

‎.adiorc.js

+2
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ module.exports = {
3636
"https",
3737
"inspector",
3838
"node:fs",
39+
"node:timers",
40+
"node:path",
3941
"os",
4042
"path",
4143
"readline",

‎packages/aws-sdk/src/client-s3/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ export {
1212
HeadObjectCommand,
1313
HeadObjectOutput,
1414
ListObjectsOutput,
15+
ListObjectsV2Command,
1516
ListPartsCommand,
1617
ListPartsCommandOutput,
1718
ListPartsOutput,

‎packages/cli-plugin-deploy-pulumi/commands/deploy.js

+7
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const { PackagesBuilder } = require("./buildPackages/PackagesBuilder");
44
const pulumiLoginSelectStack = require("./deploy/pulumiLoginSelectStack");
55
const executeDeploy = require("./deploy/executeDeploy");
66
const executePreview = require("./deploy/executePreview");
7+
const { setTimeout } = require("node:timers/promises");
78

89
module.exports = (params, context) => {
910
const command = createPulumiCommand({
@@ -16,6 +17,12 @@ module.exports = (params, context) => {
1617

1718
const hookArgs = { context, env, inputs, projectApplication };
1819

20+
context.info("Webiny version: %s", context.version);
21+
console.log();
22+
23+
// Just so the version stays on the screen for a second, before the process continues.
24+
await setTimeout(1000);
25+
1926
if (build) {
2027
await runHook({
2128
hook: "hook-before-build",

‎packages/cli-plugin-scaffold-extensions/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,16 @@
1919
},
2020
"homepage": "https://github.com/webiny/webiny-js#readme",
2121
"dependencies": {
22+
"@webiny/aws-sdk": "0.0.0",
2223
"@webiny/cli": "0.0.0",
2324
"@webiny/cli-plugin-scaffold": "0.0.0",
2425
"@webiny/error": "0.0.0",
2526
"case": "^1.6.3",
27+
"chalk": "^4.1.0",
2628
"execa": "^5.0.0",
2729
"glob": "^7.1.2",
2830
"load-json-file": "^6.2.0",
2931
"lodash": "^4.17.21",
30-
"ncp": "^2.0.0",
3132
"replace-in-path": "^1.1.0",
3233
"ts-morph": "^11.0.0",
3334
"validate-npm-package-name": "^3.0.0",

‎packages/cli-plugin-scaffold-extensions/src/downloadAndLinkExtension.ts

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
import os from "os";
2+
import path from "path";
3+
import fs from "node:fs";
4+
import fsAsync from "node:fs/promises";
5+
import { CliCommandScaffoldCallableArgs } from "@webiny/cli-plugin-scaffold/types";
6+
import { setTimeout } from "node:timers/promises";
7+
import { WEBINY_DEV_VERSION } from "~/utils/constants";
8+
import { linkAllExtensions } from "./utils/linkAllExtensions";
9+
import { Input } from "./types";
10+
import { downloadFolderFromS3 } from "./downloadAndLinkExtension/downloadFolderFromS3";
11+
import { setWebinyPackageVersions } from "~/utils/setWebinyPackageVersions";
12+
import { runYarnInstall } from "@webiny/cli-plugin-scaffold/utils";
13+
import { getDownloadedExtensionType } from "~/downloadAndLinkExtension/getDownloadedExtensionType";
14+
import chalk from "chalk";
15+
import { Extension } from "./extensions/Extension";
16+
17+
const EXTENSIONS_ROOT_FOLDER = "extensions";
18+
19+
const S3_BUCKET_NAME = "webiny-examples";
20+
const S3_BUCKET_REGION = "us-east-1";
21+
22+
const getVersionFromVersionFolders = async (
23+
versionFoldersList: string[],
24+
currentWebinyVersion: string
25+
) => {
26+
const availableVersions = versionFoldersList.map(v => v.replace(".x", ".0")).sort();
27+
28+
let versionToUse = "";
29+
30+
// When developing Webiny, we want to use the latest version.
31+
if (currentWebinyVersion === WEBINY_DEV_VERSION) {
32+
versionToUse = availableVersions[availableVersions.length - 1];
33+
} else {
34+
for (const availableVersion of availableVersions) {
35+
if (currentWebinyVersion >= availableVersion) {
36+
versionToUse = availableVersion;
37+
} else {
38+
break;
39+
}
40+
}
41+
}
42+
43+
return versionToUse.replace(".0", ".x");
44+
};
45+
46+
export const downloadAndLinkExtension = async ({
47+
input,
48+
ora,
49+
context
50+
}: CliCommandScaffoldCallableArgs<Input>) => {
51+
const currentWebinyVersion = context.version;
52+
53+
const downloadExtensionSource = input.templateArgs!;
54+
55+
try {
56+
ora.start(`Downloading extension...`);
57+
58+
const randomId = String(Date.now());
59+
const downloadFolderPath = path.join(os.tmpdir(), `wby-ext-${randomId}`);
60+
61+
await downloadFolderFromS3({
62+
bucketName: S3_BUCKET_NAME,
63+
bucketRegion: S3_BUCKET_REGION,
64+
bucketFolderKey: downloadExtensionSource,
65+
downloadFolderPath
66+
});
67+
68+
ora.text = `Copying extension...`;
69+
await setTimeout(1000);
70+
71+
let extensionsFolderToCopyPath = path.join(downloadFolderPath, "extensions");
72+
73+
// If we have `extensions` folder in the root of the downloaded extension.
74+
// it means the example extension is not versioned, and we can just copy it.
75+
const extensionsFolderExistsInRoot = fs.existsSync(extensionsFolderToCopyPath);
76+
const versionedExtension = !extensionsFolderExistsInRoot;
77+
78+
if (versionedExtension) {
79+
// If we have `x.x.x` folders in the root of the downloaded
80+
// extension, we need to find the right version to use.
81+
82+
// This can be `5.40.x`, `5.41.x`, etc.
83+
const versionFolders = await fsAsync.readdir(downloadFolderPath);
84+
85+
const versionToUse = await getVersionFromVersionFolders(
86+
versionFolders,
87+
currentWebinyVersion
88+
);
89+
90+
extensionsFolderToCopyPath = path.join(downloadFolderPath, versionToUse, "extensions");
91+
}
92+
93+
await fsAsync.cp(extensionsFolderToCopyPath, EXTENSIONS_ROOT_FOLDER, {
94+
recursive: true
95+
});
96+
97+
ora.text = `Linking extension...`;
98+
99+
// Retrieve extensions folders in the root of the downloaded extension. We use this
100+
// later to run additional setup tasks on each extension.
101+
const extensionsFolderNames = await fsAsync.readdir(extensionsFolderToCopyPath);
102+
const downloadedExtensions: Extension[] = [];
103+
104+
for (const extensionsFolderName of extensionsFolderNames) {
105+
const folderPath = path.join(EXTENSIONS_ROOT_FOLDER, extensionsFolderName);
106+
const extensionType = await getDownloadedExtensionType(folderPath);
107+
108+
downloadedExtensions.push(
109+
new Extension({
110+
name: extensionsFolderName,
111+
type: extensionType,
112+
location: folderPath,
113+
114+
// We don't care about the package name here.
115+
packageName: extensionsFolderName
116+
})
117+
);
118+
}
119+
120+
for (const downloadedExtension of downloadedExtensions) {
121+
await setWebinyPackageVersions(downloadedExtension, currentWebinyVersion);
122+
}
123+
124+
await linkAllExtensions();
125+
await runYarnInstall();
126+
127+
if (downloadedExtensions.length === 1) {
128+
const [downloadedExtension] = downloadedExtensions;
129+
ora.succeed(
130+
`Extension downloaded in ${context.success.hl(downloadedExtension.getLocation())}.`
131+
);
132+
133+
const nextSteps = downloadedExtension.getNextSteps();
134+
135+
console.log();
136+
console.log(chalk.bold("Next Steps"));
137+
nextSteps.forEach(message => {
138+
console.log(`‣ ${message}`);
139+
});
140+
} else {
141+
const paths = downloadedExtensions.map(ext => ext.getLocation());
142+
ora.succeed(`Extensions downloaded in ${context.success.hl(paths.join(", "))}.`);
143+
}
144+
} catch (e) {
145+
switch (e.code) {
146+
case "NO_OBJECTS_FOUND":
147+
ora.fail("Could not download extension. Looks like the extension does not exist.");
148+
break;
149+
default:
150+
ora.fail("Could not create extension. Please check the logs below.");
151+
console.log();
152+
console.log(e);
153+
}
154+
}
155+
};

‎packages/cli-plugin-scaffold-extensions/src/downloadAndLinkExtension/downloadFolderFromS3.ts

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import { S3Client, ListObjectsV2Command, GetObjectCommand } from "@webiny/aws-sdk/client-s3";
2+
import fs from "fs";
3+
import path from "path";
4+
import { WebinyError } from "@webiny/error";
5+
6+
interface DownloadFolderFromS3Params {
7+
bucketName: string;
8+
bucketRegion: string;
9+
bucketFolderKey: string;
10+
downloadFolderPath: string;
11+
}
12+
13+
export const downloadFolderFromS3 = async (params: DownloadFolderFromS3Params) => {
14+
const { bucketName, bucketRegion, bucketFolderKey, downloadFolderPath } = params;
15+
16+
// Configure the S3 client
17+
const s3Client = new S3Client({ region: bucketRegion });
18+
19+
// List all objects in the specified S3 folder
20+
const listObjects = async (bucket: string, folderKey: string) => {
21+
const command = new ListObjectsV2Command({
22+
Bucket: bucket,
23+
Prefix: folderKey
24+
});
25+
const response = await s3Client.send(command);
26+
return response.Contents;
27+
};
28+
29+
// Download an individual file from S3
30+
const downloadFile = async (bucket: string, key: string, localPath: string) => {
31+
const command = new GetObjectCommand({
32+
Bucket: bucket,
33+
Key: key
34+
});
35+
36+
const response = await s3Client.send(command);
37+
38+
return new Promise((resolve, reject) => {
39+
const fileStream = fs.createWriteStream(localPath);
40+
// @ts-expect-error
41+
response.Body.pipe(fileStream);
42+
// @ts-expect-error
43+
response.Body.on("error", reject);
44+
fileStream.on("finish", resolve);
45+
});
46+
};
47+
48+
const objects = (await listObjects(bucketName, bucketFolderKey)) || [];
49+
if (!objects.length) {
50+
throw new WebinyError(`No objects found in the specified S3 folder.`, "NO_OBJECTS_FOUND");
51+
}
52+
53+
for (const object of objects) {
54+
const s3Key = object.Key!;
55+
const relativePath = path.relative(bucketFolderKey, s3Key);
56+
const localFilePath = path.join(downloadFolderPath, relativePath);
57+
58+
if (s3Key.endsWith("/")) {
59+
// It's a directory, create it if it doesn't exist.
60+
if (!fs.existsSync(localFilePath)) {
61+
fs.mkdirSync(localFilePath, { recursive: true });
62+
}
63+
} else {
64+
// It's a file, download it.
65+
const localDirPath = path.dirname(localFilePath);
66+
if (!fs.existsSync(localDirPath)) {
67+
fs.mkdirSync(localDirPath, { recursive: true });
68+
}
69+
70+
await downloadFile(bucketName, s3Key, localFilePath);
71+
}
72+
}
73+
};

‎packages/cli-plugin-scaffold-extensions/src/downloadAndLinkExtension/getDownloadedExtensionType.ts

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import loadJson from "load-json-file";
2+
import { PackageJson } from "@webiny/cli-plugin-scaffold/types";
3+
import path from "node:path";
4+
5+
export const getDownloadedExtensionType = async (downloadedExtensionRootPath: string) => {
6+
const pkgJsonPath = path.join(downloadedExtensionRootPath, "package.json");
7+
const pkgJson = await loadJson<PackageJson>(pkgJsonPath);
8+
9+
const keywords = pkgJson.keywords;
10+
if (Array.isArray(keywords)) {
11+
for (const keyword of keywords) {
12+
if (keyword.startsWith("webiny-extension-type:")) {
13+
return keyword.replace("webiny-extension-type:", "");
14+
}
15+
}
16+
}
17+
18+
throw new Error(`Could not determine the extension type from the downloaded extension.`);
19+
};

‎packages/cli-plugin-scaffold-extensions/src/extensions/AbstractExtension.ts

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
export interface ExtensionTypeConstructorParams {
2+
name: string;
3+
type: string;
4+
location: string;
5+
packageName: string;
6+
}
7+
8+
export abstract class AbstractExtension {
9+
protected params: ExtensionTypeConstructorParams;
10+
11+
constructor(params: ExtensionTypeConstructorParams) {
12+
this.params = params;
13+
}
14+
15+
abstract generate(): Promise<void>;
16+
17+
abstract getNextSteps(): string[];
18+
19+
getPackageJsonPath(): string {
20+
return `${this.params.location}/package.json`;
21+
}
22+
23+
getLocation(): string {
24+
return this.params.location;
25+
}
26+
27+
getPackageName(): string {
28+
return this.params.packageName;
29+
}
30+
31+
getName(): string {
32+
return this.params.name;
33+
}
34+
}

0 commit comments

Comments
 (0)