Keep track of unnamed chunk files

Author: xPaw

composer.json

@@ -1,4 +1,7 @@
 {
+    "scripts": {
+        "test": "phpstan analyse"
+    },
     "require": {
         "phpstan/phpstan": "^1.4"
     }

generate_manifest_urls.mjs

@@ -45,8 +45,26 @@ for (const { file, cdn } of files) {
 		const code = await readFile(file);
 		const ast = parse(code, { ecmaVersion: latestEcmaVersion, loc: true });
 
+		const seenIds = new Set();
+
 		traverse(ast, {
 			enter: function (node) {
+				if (
+					node.type === Syntax.BinaryExpression &&
+					node.left.type === Syntax.BinaryExpression &&
+					node.left.right.type === Syntax.Literal &&
+					/\.(css|js)\?contenthash=/.test(node.left.right.value) &&
+					node.right.type === Syntax.MemberExpression &&
+					node.right.object.type === Syntax.ObjectExpression
+				) {
+					for (const property of node.right.object.properties) {
+						if (property.value.type === Syntax.Literal) {
+							seenIds.add(property.key.raw);
+						}
+					}
+				}
+
+				// Useful: https://astexplorer.net/
 				if (
 					node.type === Syntax.BinaryExpression &&
 					node.left.type === Syntax.Literal &&
@@ -67,12 +85,15 @@ for (const { file, cdn } of files) {
 						return;
 					}
 
-					const obj = node.right.type === Syntax.LogicalExpression ? node.right.left.object : node.right.object;
+					const partialFileNames = node.right.type === Syntax.LogicalExpression;
+					const obj = partialFileNames ? node.right.left.object : node.right.object;
 
 					for (const property of obj.properties) {
 						if (property.value.type === Syntax.Literal) {
 							const name = property.value.value;
 
+							seenIds.delete(property.key.raw);
+
 							if (name.endsWith("-json") && !name.endsWith("_english-json")) {
 								continue;
 							}
@@ -85,6 +106,13 @@ for (const { file, cdn } of files) {
 						}
 					}
 
+					for (const id of seenIds) {
+						const fullUrl = cdn + folder + id + suffix;
+
+						urls.add(fullUrl);
+					}
+
+					seenIds.clear();
 					this.skip();
 				}
 			},

package.json

@@ -7,7 +7,7 @@
     "prettier": "prettier --write"
   },
   "dependencies": {
-    "espree": "^9.3.1",
+    "espree": "^10.0.1",
     "estraverse": "^5.3.0",
     "prettier": "^3.0.0"
   },

update.php

@@ -5,7 +5,7 @@
 
 if( \function_exists( 'sys_getloadavg' ) )
 {
-	if( \sys_getloadavg()[ 1 ] > 5.0 )
+	if( ( \sys_getloadavg()[ 1 ] ?? 0.0 ) > 5.0 )
 	{
 		echo 'Not running due to high cpu load';
 		exit;
@@ -530,6 +530,7 @@ private function Fetch( array $URLs, int $WindowSize ) : void
 
 				while( $Done = curl_multi_info_read( $Master ) )
 				{
+					/** @var CurlHandle */
 					$Handle = $Done[ 'handle' ];
 					$URL   = curl_getinfo( $Handle, CURLINFO_EFFECTIVE_URL );
 					$Code  = curl_getinfo( $Handle, CURLINFO_HTTP_CODE );
@@ -554,7 +555,7 @@ private function Fetch( array $URLs, int $WindowSize ) : void
 					{
 						$this->Log( '{yellow}Error ' . $Code . '{normal}    - ' . $URL );
 
-						if( $Code < 400 && $Code > 499 )
+						if( $Code < 100 || $Code >= 500 )
 						{
 							$this->URLsToFetch[ ] =
 							[
@@ -786,11 +787,13 @@ private function ProcessManifests( array $KnownUrls ) : array
 			// Find and delete old chunk~ files
 			$Folders = [];
 
+			$IsChunkFile = static fn( string $Filename ) : bool => preg_match( '/^(chunk~|libraries~|[0-9]+\.)/', $Filename, $Matches ) === 1;
+
 			foreach( $URLsToFetch as $Url )
 			{
 				$Filename = basename( $Url[ 'File' ] );
 
-				if( str_starts_with( $Filename, 'chunk~' ) || str_starts_with( $Filename, 'libraries~' ) )
+				if( $IsChunkFile( $Filename ) )
 				{
 					$Folder = __DIR__ . '/' . dirname( $Url[ 'File' ] ) . '/';
 
@@ -805,18 +808,32 @@ private function ProcessManifests( array $KnownUrls ) : array
 
 			foreach( $Folders as $Folder => $NewChunks )
 			{
+				foreach( new DirectoryIterator( $Folder ) as $FileInfo )
+				{
+					if( $FileInfo->isDot() )
+					{
+						continue;
+					}
+
+					$Filename = $FileInfo->getFilename();
+
+					if( $IsChunkFile( $Filename ) && !isset( $NewChunks[ $Filename ] ) )
+					{
+						$FilepathOnDisk = $FileInfo->getRealPath();
+
+						$this->Log( 'Chunk ' . $FilepathOnDisk . ' no longer exists in manifest' );
+
+						unlink( $FilepathOnDisk );
+					}
+				}
+
 				foreach( [ 'chunk~*', 'libraries~*' ] as $Glob )
 				{
 					foreach( glob( $Folder . $Glob ) as $FilepathOnDisk )
 					{
 						$Filename = basename( $FilepathOnDisk );
 
-						if( !isset( $NewChunks[ $Filename ] ) )
-						{
-							$this->Log( 'Chunk ' . $FilepathOnDisk . ' no longer exists in manifest' );
-
-							unlink( $FilepathOnDisk );
-						}
+						
 					}
 				}
 			}