Start with packages api

Author: jeroen

routes/api.js

@@ -1,6 +1,6 @@
 import express from 'express';
 import url from 'node:url';
-import {ls_packages} from '../src/db.js';
+import {ls_packages, get_package_info, get_universe_packages} from '../src/db.js';
 
 const router = express.Router();
 
@@ -9,8 +9,21 @@ router.get('/api', function(req, res, next) {
 });
 
 router.get('/api/ls', function(req, res, next) {
-  ls_packages(res.locals.universe).then(x => res.send(x));
+  return ls_packages(res.locals.universe).then(x => res.send(x));
 });
 
+router.get('/api/packages', function(req, res, next) {
+  var fields = req.query.fields && req.query.fields.split(",");
+  return get_universe_packages(res.locals.universe, fields).then(function(x){
+    res.send(x);
+  });
+});
+
+router.get('/api/packages/:package', function(req, res, next) {
+  var fields = req.query.fields && req.query.fields.split(",");
+  return get_package_info(req.params.package, res.locals.universe, fields).then(function(x){
+    res.send(x)
+  });
+});
 
 export default router;

routes/repos.js

@@ -1,27 +1,10 @@
 import express from 'express';
-import zlib from 'node:zlib';
 import createError from 'http-errors';
-import {doc_to_dcf, match_macos_arch} from '../src/tools.js';
+import {doc_to_dcf, match_macos_arch, doc_to_ndjson, cursor_stream} from '../src/tools.js';
 import {get_universe_binaries, get_packages_index, get_package_hash} from '../src/db.js';
 
 const router = express.Router();
 
-function doc_to_ndjson(x){
-  return JSON.stringify(x) + '\n';
-}
-
-// Somehow node:stream/promises does not catch input on-error callbacks properly
-// so we promisify ourselves. See https://github.com/r-universe-org/help/issues/540
-function cursor_stream(cursor, output, transform, gzip){
-  return new Promise(function(resolve, reject) {
-    var input = cursor.stream({transform: transform}).on('error', reject);
-    if(gzip){
-      input = input.pipe(zlib.createGzip()).on('error', reject);
-    }
-    input.pipe(output).on('finish', resolve).on('error', reject);
-  });
-}
-
 function packages_index(query, req, res, mixed = false){
   query._user = res.locals.universe;
   var format = req.params.format || "PACKAGES.json";

src/db.js

@@ -127,9 +127,18 @@ function array_first(key){
 }
 
 function build_projection(fields){
+  if(!fields || !fields.length) return {_id:0};
   var projection = {Package:1, _type:1, _user:1, _indexed: 1, _id:0};
   fields.forEach(function (f) {
-    projection[f] = 1;
+    if(f == '_binaries'){
+      projection['Built'] = 1;
+      projection['_status'] = 1;
+      projection['_check'] = 1;
+      if(!fields.includes("_commit"))
+        projection['_commit.id'] = 1;
+    } else {
+      projection[f] = 1;
+    }
   });
   return projection;
 }
@@ -138,8 +147,9 @@ function mongo_package_files(pkg, universe){
   return mongo_find({_user: universe, Package: pkg, _registered: true}).toArray();
 }
 
-function mongo_package_info(pkg, universe){
-  return mongo_find({_user: universe, Package: pkg, _registered: true}).toArray().then(function(docs){
+function mongo_package_info(pkg, universe, fields){
+  var proj = build_projection(fields);
+  return mongo_find({_user: universe, Package: pkg, _registered: true}).project(proj).toArray().then(function(docs){
     if(!docs.length) //should never happen because we checked earlier
       throw createError(404, `Package ${pkg} not found in ${universe}`);
     var pkgdata = group_package_data(docs);
@@ -416,9 +426,9 @@ export function get_bucket_stream(hash){
   }
 }
 
-export function get_package_info(pkg, universe){
+export function get_package_info(pkg, universe, fields){
   if(production){
-    return mongo_package_info(pkg, universe);
+    return mongo_package_info(pkg, universe, fields);
   } else {
     console.warn(`Fetching ${pkg} info from API...`)
     if(universe){
@@ -463,7 +473,7 @@ export function get_universe_vignettes(universe){
 
 export function get_universe_packages(universe, fields, limit = 5000){
   if(production){
-    return mongo_universe_packages(universe, fields, limit)
+    return mongo_universe_packages(universe, fields, limit);
   } else {
     console.warn(`Fetching ${universe} packages from API...`)
     var apiurl = `https://${universe}.r-universe.dev/api/packages?stream=1&all=true&limit=${limit}&fields=${fields.join()}`;
@@ -554,7 +564,6 @@ export function get_packages_index(query, fields = [], mixed = false){
   if(production){
     let projection = {...pkgfields};
     fields.forEach(function (f) {
-      console.log("adding", f)
       projection[f] = 1;
     });
     var cursor = mixed ? mongo_aggregate([

src/tools.js

@@ -1,5 +1,6 @@
 import {Buffer} from "node:buffer";
 import tar from 'tar-stream';
+import zlib from 'node:zlib';
 import gunzip from 'gunzip-maybe';
 import {load as cheerio_load} from 'cheerio';
 import hljs from 'highlight.js';
@@ -165,6 +166,22 @@ export function doc_to_dcf(doc){
   }).join("\n") + "\n\n";
 }
 
+export function doc_to_ndjson(x){
+  return JSON.stringify(x) + '\n';
+}
+
+// Somehow node:stream/promises does not catch input on-error callbacks properly
+// so we promisify ourselves. See https://github.com/r-universe-org/help/issues/540
+export function cursor_stream(cursor, output, transform, gzip){
+  return new Promise(function(resolve, reject) {
+    var input = cursor.stream({transform: transform}).on('error', reject);
+    if(gzip){
+      input = input.pipe(zlib.createGzip()).on('error', reject);
+    }
+    input.pipe(output).on('finish', resolve).on('error', reject);
+  });
+}
+
 export function match_macos_arch(platform){
   if(platform.match("arm64|aarch64")){
     return {$not : /x86_64/};