[Blueprints] Preserve the first char of all filenames sourced from GitDirectoryReference (#2070)

Adjust GitDirectoryReference to stop cutting off the first character
from each filename when the repository path starts with a slash.

Closes #2061
Closes #1999

 ## Implementation

The problem was passing the wring prefix length to
`fileName.substring()` (user input vs path without the leading slash).
This PR adjusts that.

In addition, it updates git-sparse-checkout.ts to use `commitHash` and
not ref name for fetching files. This way you can checkout a specific
commit, and that's convenient for testing and developers consuming the
API.

It also modified the CORS proxy to allow 'git-protocol' in headers.

## Testing 

Go to
http://127.0.0.1:5400/website-server/#{%22landingPage%22:%22/wp-admin/%22,%22preferredVersions%22:{%22php%22:%227.4%22,%22wp%22:%225.9%22},%22steps%22:[{%22step%22:%22login%22,%22username%22:%22admin%22},{%22step%22:%22installPlugin%22,%22pluginData%22:{%22resource%22:%22git:directory%22,%22url%22:%22https://wordpress-playground-cors-proxy.net/cors-proxy.php?https://github.com/Automattic/page-optimize.git%22,%22ref%22:%22master%22,%22path%22:%22/%22},%22options%22:{%22activate%22:true,%22targetFolderName%22:%22page-optimize-from-git%22}}]}
and confirm you see wp-admin (without CSS).

Author: adamziel

packages/playground/blueprints/src/lib/resources.spec.ts

@@ -1,4 +1,4 @@
-import { UrlResource } from './resources';
+import { UrlResource, GitDirectoryResource } from './resources';
 
 describe('UrlResource', () => {
 	it('should create a new instance of UrlResource', () => {
@@ -21,3 +21,29 @@ describe('UrlResource', () => {
 		);
 	});
 });
+
+describe('GitDirectoryResource', () => {
+	describe('resolve', () => {
+		it.each([
+			'packages/docs/site/docs/blueprints/tutorial',
+			'/packages/docs/site/docs/blueprints/tutorial',
+		])(
+			'should return a list of files in the directory (path: %s)',
+			async (path) => {
+				const resource = new GitDirectoryResource({
+					resource: 'git:directory',
+					url: 'https://github.com/WordPress/wordpress-playground',
+					ref: '05138293dd39e25a9fa8e43a9cc775d6fb780e37',
+					path,
+				});
+				const { files } = await resource.resolve();
+				expect(Object.keys(files)).toEqual([
+					'01-what-are-blueprints-what-you-can-do-with-them.md',
+					'02-how-to-load-run-blueprints.md',
+					'03-build-your-first-blueprint.md',
+					'index.md',
+				]);
+			}
+		);
+	});
+});

packages/playground/blueprints/src/lib/resources.ts

@@ -7,6 +7,7 @@ import { dirname, Semaphore } from '@php-wasm/util';
 import {
 	listDescendantFiles,
 	listGitFiles,
+	resolveCommitHash,
 	sparseCheckout,
 } from '@wp-playground/storage';
 import { zipNameToHumanName } from './utils/zip-name-to-human-name';
@@ -461,21 +462,20 @@ export class GitDirectoryResource extends Resource<Directory> {
 		const repoUrl = this.options?.corsProxy
 			? `${this.options.corsProxy}${this.reference.url}`
 			: this.reference.url;
-		const ref = ['', 'HEAD'].includes(this.reference.ref)
-			? 'HEAD'
-			: `refs/heads/${this.reference.ref}`;
-		const allFiles = await listGitFiles(repoUrl, ref);
+
+		const commitHash = await resolveCommitHash(repoUrl, {
+			value: this.reference.ref,
+			type: 'infer',
+		});
+		const allFiles = await listGitFiles(repoUrl, commitHash);
 
 		const requestedPath = this.reference.path.replace(/^\/+/, '');
 		const filesToClone = listDescendantFiles(allFiles, requestedPath);
-		let files = await sparseCheckout(repoUrl, ref, filesToClone);
+		let files = await sparseCheckout(repoUrl, commitHash, filesToClone);
+
 		// Remove the path prefix from the cloned file names.
-		files = Object.fromEntries(
-			Object.entries(files).map(([name, contents]) => {
-				name = name.substring(this.reference.path.length);
-				name = name.replace(/^\/+/, '');
-				return [name, contents];
-			})
+		files = mapKeys(files, (name) =>
+			name.substring(requestedPath.length).replace(/^\/+/, '')
 		);
 		return {
 			name:
@@ -493,6 +493,12 @@ export class GitDirectoryResource extends Resource<Directory> {
 	}
 }
 
+function mapKeys(obj: Record<string, any>, fn: (key: string) => string) {
+	return Object.fromEntries(
+		Object.entries(obj).map(([key, value]) => [fn(key), value])
+	);
+}
+
 /**
  * A `Resource` that represents a git directory.
  */

packages/playground/php-cors-proxy/cors-proxy.php

@@ -19,7 +19,7 @@
     header('Access-Control-Allow-Origin: ' . $origin);
     header('Access-Control-Allow-Credentials: true');
     header('Access-Control-Allow-Methods: GET, POST, OPTIONS');
-    header('Access-Control-Allow-Headers: Accept, Authorization, Content-Type');
+    header('Access-Control-Allow-Headers: Accept, Authorization, Content-Type, git-protocol');
 }
 if ($_SERVER['REQUEST_METHOD'] === 'OPTIONS') {
     header("Allow: GET, POST, OPTIONS");

packages/playground/storage/src/lib/git-sparse-checkout.spec.ts

@@ -2,6 +2,7 @@ import {
 	listGitRefs,
 	sparseCheckout,
 	listGitFiles,
+	resolveCommitHash,
 } from './git-sparse-checkout';
 
 describe('listRefs', () => {
@@ -18,9 +19,16 @@ describe('listRefs', () => {
 
 describe('sparseCheckout', () => {
 	it('should retrieve the requested files from a git repo', async () => {
+		const commitHash = await resolveCommitHash(
+			'https://github.com/WordPress/wordpress-playground.git',
+			{
+				value: 'trunk',
+				type: 'branch',
+			}
+		);
 		const files = await sparseCheckout(
 			'https://github.com/WordPress/wordpress-playground.git',
-			'refs/heads/trunk',
+			commitHash,
 			['README.md']
 		);
 		expect(files).toEqual({
@@ -32,9 +40,16 @@ describe('sparseCheckout', () => {
 
 describe('listGitFiles', () => {
 	it('should list the files in a git repo', async () => {
+		const commitHash = await resolveCommitHash(
+			'https://github.com/WordPress/wordpress-playground.git',
+			{
+				value: 'trunk',
+				type: 'branch',
+			}
+		);
 		const files = await listGitFiles(
 			'https://github.com/WordPress/wordpress-playground.git',
-			'refs/heads/trunk'
+			commitHash
 		);
 		expect(files).toEqual(
 			expect.arrayContaining([

packages/playground/storage/src/lib/git-sparse-checkout.ts

@@ -39,11 +39,9 @@ if (typeof window !== 'undefined') {
  */
 export async function sparseCheckout(
 	repoUrl: string,
-	fullyQualifiedBranchName: string,
+	commitHash: string,
 	filesPaths: string[]
 ) {
-	const refs = await listGitRefs(repoUrl, fullyQualifiedBranchName);
-	const commitHash = refs[fullyQualifiedBranchName];
 	const treesIdx = await fetchWithoutBlobs(repoUrl, commitHash);
 	const objects = await resolveObjects(treesIdx, commitHash, filesPaths);
 
@@ -75,24 +73,24 @@ export type FileTreeFolder = {
 };
 export type FileTree = FileTreeFile | FileTreeFolder;
 
+export type GitRef = {
+	value: string;
+	type?: 'branch' | 'commit' | 'refname' | 'infer';
+};
+
 /**
  * Lists all files in a git repository.
  *
  * See https://git-scm.com/book/en/v2/Git-Internals-Git-Objects for more information.
  *
  * @param repoUrl The URL of the git repository.
- * @param fullyQualifiedBranchName The full name of the branch to fetch from (e.g., 'refs/heads/main').
+ * @param commitHash The commit hash to fetch from.
  * @returns A list of all files in the repository.
  */
 export async function listGitFiles(
 	repoUrl: string,
-	fullyQualifiedBranchName: string
+	commitHash: string
 ): Promise<FileTree[]> {
-	const refs = await listGitRefs(repoUrl, fullyQualifiedBranchName);
-	if (!(fullyQualifiedBranchName in refs)) {
-		throw new Error(`Branch ${fullyQualifiedBranchName} not found`);
-	}
-	const commitHash = refs[fullyQualifiedBranchName];
 	const treesIdx = await fetchWithoutBlobs(repoUrl, commitHash);
 	const rootTree = await resolveAllObjects(treesIdx, commitHash);
 	if (!rootTree?.object) {
@@ -102,6 +100,48 @@ export async function listGitFiles(
 	return gitTreeToFileTree(rootTree);
 }
 
+/**
+ * Resolves a ref description, e.g. a branch name, to a commit hash.
+ *
+ * @param repoUrl The URL of the git repository.
+ * @param ref The branch name or commit hash.
+ * @returns The commit hash.
+ */
+export async function resolveCommitHash(repoUrl: string, ref: GitRef) {
+	if (ref.type === 'infer' || ref.type === undefined) {
+		if (['', 'HEAD'].includes(ref.value)) {
+			ref = {
+				value: ref.value,
+				type: 'refname',
+			};
+		} else if (typeof ref.value === 'string' && ref.value.length === 40) {
+			ref = {
+				value: ref.value,
+				type: 'commit',
+			};
+		}
+	}
+	if (ref.type === 'branch') {
+		ref = {
+			value: `refs/heads/${ref.value}`,
+			type: 'refname',
+		};
+	}
+	switch (ref.type) {
+		case 'commit':
+			return ref.value;
+		case 'refname': {
+			const refs = await listGitRefs(repoUrl, ref.value);
+			if (!(ref.value in refs)) {
+				throw new Error(`Branch ${ref.value} not found`);
+			}
+			return refs[ref.value];
+		}
+		default:
+			throw new Error(`Invalid ref type: ${ref.type}`);
+	}
+}
+
 function gitTreeToFileTree(tree: GitTree): FileTree[] {
 	return tree.object
 		.map((branch) => {

packages/playground/storage/src/lib/paths.ts

@@ -46,3 +46,10 @@ export function listDescendantFiles(files: FileTree[], selectedPath: string) {
 	}
 	return descendants;
 }
+
+export function removePathPrefix(path: string, prefix: string) {
+	if (path.startsWith(prefix)) {
+		return path.substring(prefix.length);
+	}
+	return path;
+}