Add endpoint for S3 ListObjectsV2 API

See r-universe-org/help#574

Author: jeroen

routes/cache.js

@@ -34,7 +34,7 @@ export default function(req, res, next){
     res.set('Cache-Control', `public, max-age=60, stale-while-revalidate=${cdn_cache}`);
 
     if(doc){
-      const revision = 15; // bump to invalidate all caches
+      const revision = 16; // bump to invalidate all caches
       const etag = `W/"${doc._id}${revision}"`;
       const date = new Date(doc._published.getTime() + revision * 1000).toUTCString();
       res.set('ETag', etag);

routes/universe.js

@@ -1,6 +1,7 @@
 import express from 'express';
 import url from 'node:url';
-import {get_universe_packages, get_universe_vignettes, get_package_info,
+import createError from 'http-errors';
+import {get_universe_packages, get_universe_files, get_universe_vignettes, get_package_info,
         get_universe_contributors, get_universe_contributions, get_all_universes} from '../src/db.js';
 const router = express.Router();
 
@@ -122,12 +123,47 @@ function get_contrib_data(user, max = 20){
   });
 }
 
-/* Langing page (TODO) */
+//See https://github.com/r-universe-org/help/issues/574
+function send_s3_list(req, res){
+  var universe = res.locals.universe;
+  var delimiter = req.query['delimiter'];
+  var start_after = req.query['start-after'] || req.query['continuation-token'];
+  var max_keys = parseInt(req.query['max-keys'] || 1000);
+  var prefix = req.query['prefix'] || "";
+  return get_universe_files(universe, prefix, start_after).then(function(files){
+    if(delimiter){
+      var subpaths = files.map(x => x.Key.substring(prefix.length));
+      var dirnames = subpaths.filter(x => x.includes('/')).map(x => prefix + x.split('/')[0]);
+      var commonprefixes = [...new Set(dirnames)];
+      files = files.filter(x => x.Key.substring(prefix.length).includes('/') == false);
+    } else {
+       var commonprefixes = [];
+    }
+    var IsTruncated = files.length > max_keys;
+    files = files.slice(0, max_keys);
+    return res.type('application/xml').render('S3List', {
+      Prefix: prefix,
+      MaxKeys: max_keys,
+      IsTruncated: IsTruncated,
+      NextContinuationToken: IsTruncated ? files[files.length -1].Key : undefined,
+      commonprefixes: commonprefixes,
+      files: files
+    });
+  });
+}
+
 router.get('/', function(req, res, next) {
   //res.render('index');
+  if(req.query['x-id'] == 'ListBuckets'){
+    throw createError(400, "Please use virtual-hosted-style bucket on r-universe.dev TLD");
+  }
+  if(req.query['list-type']){
+    return send_s3_list(req, res);
+  }
   res.set('Cache-control', 'private, max-age=604800'); // Vary does not work in cloudflare currently
   const accept = req.headers['accept'];
   if(accept && accept.includes('html')){
+    /* Langing page (TODO) */
     res.redirect(`/builds`);
   } else {
     res.send(`Welcome to the ${res.locals.universe} universe!`);

src/db.js

@@ -1,6 +1,7 @@
 import {MongoClient, GridFSBucket} from 'mongodb';
 import {Readable} from  "node:stream";
-import {pkgfields} from './tools.js';
+import path from 'node:path';
+import {pkgfields, doc_to_paths} from './tools.js';
 import createError from 'http-errors';
 
 const HOST = process.env.CRANLIKE_MONGODB_SERVER || '127.0.0.1';
@@ -227,6 +228,46 @@ function mongo_universe_binaries(user, type){
   return cursor.toArray();
 }
 
+function mongo_universe_files(user, prefix, start_after){
+  var query = {_user: user, _registered: true, _type: {'$ne': 'failure'}};
+  var proj = {Package:1, Version:1, Built:1, _distro:1, _type:1, _id:1, _published:1, _filesize:1};
+  return mongo_find(query).sort({_id: 1}).project(proj).toArray().then(function(docs){
+    if(!docs.length) //should not happen because we checked earlier
+      throw createError(404, `No packages found in ${universe}`);
+    var files = [];
+    var indexes = {};
+    docs.forEach(function(doc){
+      doc_to_paths(doc).forEach(function(fpath){
+        if(!prefix || fpath.startsWith(prefix)) {
+          files.push({
+            Key: fpath,
+            ETag: doc._id,
+            LastModified: doc._published.toISOString(),
+            Size: doc._filesize
+          });
+          var repodir = path.dirname(fpath);
+          if(!(indexes[repodir] > doc._published)){
+            indexes[repodir] = doc._published;
+          }
+        }
+      });
+    });
+
+    for (const [path, date] of Object.entries(indexes)) {
+      files.push({ Key: path + '/PACKAGES', LastModified: date.toISOString()});
+      files.push({ Key: path + '/PACKAGES.gz', LastModified: date.toISOString()});
+    }
+
+    if(start_after){
+      var index = files.findIndex(x => x.Key == start_after);
+      if(index > -1){
+        files = files.slice(index + 1);
+      }
+    }
+    return files;
+  });
+}
+
 /* NB Contributions are grouped by upstream url instead of package namme to avoid duplicate counting
  * of contributions in repos with many packages, e.g. https://github.com/r-forge/ctm/tree/master/pkg */
 function mongo_universe_contributors(user, limit = 20){
@@ -536,6 +577,14 @@ export function get_universe_contributions(universe, limit){
   }
 }
 
+export function get_universe_files(universe, prefix, start_after){
+  if(production){
+    return mongo_universe_files(universe, prefix, start_after);
+  } else {
+    throw "Not implemented for devel";
+  }
+}
+
 export function get_repositories(){
   if(production){
     return mongo_all_universes()

src/tools.js

@@ -174,3 +174,31 @@ export function match_macos_arch(platform){
   }
   throw createError(404, `Unsupported MacOS version: ${platform}`);
 }
+
+export function doc_to_paths(doc){
+  var type = doc._type;
+  if(type == 'src'){
+    return [`src/contrib/${doc.Package}_${doc.Version}.tar.gz`];
+  }
+  var built = doc.Built && doc.Built.R && doc.Built.R.substring(0,3);
+  if(type == 'win'){
+    return [`bin/windows/contrib/${built}/${doc.Package}_${doc.Version}.zip`];
+  }
+  if(type == 'mac'){
+    var intel = `bin/macosx/big-sur-x86_64/contrib/${built}/${doc.Package}_${doc.Version}.tgz`;
+    var arm = `bin/macosx/big-sur-arm64/contrib/${built}/${doc.Package}_${doc.Version}.tgz`;
+    if(doc.Built.Platform){
+      return [doc.Built.Platform.match("x86_64") ? intel : arm];
+    } else {
+      return [intel, arm];
+    }
+  }
+  if(type == 'linux'){
+    var distro = doc._distro || doc.Distro || 'linux';
+    return [`bin/linux/${distro}/${built}/src/contrib/${doc.Package}_${doc.Version}.tar.gz`];
+  }
+  if(type == 'wasm'){
+    return [`bin/emscripten/contrib/${built}/${doc.Package}_${doc.Version}.tgz`];
+  }
+  throw `Unsupported type: ${type}`;
+}

views/S3List.pug

@@ -0,0 +1,21 @@
+doctype xml
+ListBucketResult(xmlns="http://s3.amazonaws.com/doc/2006-03-01/")
+  Name #{universe}.r-universe.dev
+  Prefix #{Prefix}
+  NextContinuationToken #{NextContinuationToken}
+  KeyCount #{files.length + commonprefixes.length}
+  MaxKeys #{MaxKeys}
+  IsTruncated #{IsTruncated}
+  if StartAfter
+    StartAfter #{StartAfter}
+  each x in commonprefixes
+    CommonPrefixes
+      Prefix #{x}
+  each x in files
+    Contents
+      Key #{x.Key}
+      LastModified #{x.LastModified}
+      if x.ETag
+        ETag "#{x.ETag}"
+      if x.Size
+        Size #{x.Size}