Port contributor chart to frontend

Author: jeroen

routes/cache.js

@@ -34,7 +34,7 @@ export default function(req, res, next){
     res.set('Cache-Control', `public, max-age=60, stale-while-revalidate=${cdn_cache}`);
 
     if(doc){
-      const revision = 7; // bump to invalidate all caches
+      const revision = 12; // bump to invalidate all caches
       const etag = `W/"${doc._id}${revision}"`;
       const date = new Date(doc._published.getTime() + revision * 1000).toUTCString();
       res.set('ETag', etag);

routes/universe.js

@@ -1,6 +1,6 @@
 import express from 'express';
 import url from 'node:url';
-import {get_universe_packages, get_universe_vignettes, get_package_info} from '../src/db.js';
+import {get_universe_packages, get_universe_vignettes, get_package_info, get_universe_contributors, get_universe_contributions} from '../src/db.js';
 const router = express.Router();
 
 function sort_by_package(x,y){
@@ -91,6 +91,33 @@ function retry_url(x){
   return `https://${x._user}.r-universe.dev/api/packages/${x.Package}/${retryversion}/${retrytype}`;
 }
 
+function get_contrib_data(user, max = 20){
+  const p1 = get_universe_contributors(user, 1000);
+  const p2 = get_universe_contributions(user, 1000);
+  return Promise.all([p1, p2]).then(function([contributors, contributions]){
+    if(contributions.length == 0){
+      return contributors.slice(0,max); //org users dont make contributions themselves
+    }
+    var data = contributors.map(function(x){
+      x.contributions = 0;
+      x.packages = [];
+      return x;
+    });
+    contributions.forEach(function(x, i){
+      x.maintainers.forEach(function(maintainer){
+        var rec = data.find(y => y.login == maintainer);
+        if(!rec){
+          rec = {login: maintainer, total: 0, contributions: 0, repos: [], packages: []};
+          data.push(rec);
+        }
+        rec.contributions = rec.contributions + x.contributions;
+        rec.packages = rec.packages.concat(x.packages);
+      });
+    });
+    return data.sort(function(x,y){return (x.total + x.contributions > y.total + y.contributions) ? -1 : 1}).filter(x => x.login != user).slice(0,max);
+  });
+}
+
 /* Langing page (TODO) */
 router.get('/', function(req, res, next) {
   //res.render('index');
@@ -179,7 +206,9 @@ router.get("/datasets", function(req, res, next){
 });
 
 router.get("/contributors", function(req, res, next){
-  res.render('contributors', {});
+  return get_contrib_data(res.locals.universe).then(function(contributors){
+    res.render('contributors', {contributors: contributors});
+  });
 });
 
 router.get("/articles", function(req, res, next){

src/db.js

@@ -227,6 +227,45 @@ function mongo_universe_binaries(user, type){
   return cursor.toArray();
 }
 
+/* NB Contributions are grouped by upstream url instead of package namme to avoid duplicate counting
+ * of contributions in repos with many packages, e.g. https://github.com/r-forge/ctm/tree/master/pkg */
+function mongo_universe_contributors(user, limit = 20){
+  var query = {_universes: user, _type: 'src', '_registered' : true};
+  var cursor = mongo_aggregate([
+    {$match: query},
+    {$project: {
+      _id: 0,
+      contributors: '$_contributors',
+      upstream: '$_upstream'
+    }},
+    {$unwind: "$contributors"},
+    {$group: {_id: "$contributors.user", repos: {$addToSet: {upstream: '$upstream', count: '$contributors.count'}}}},
+    {$project: {_id:0, login: '$_id', total: {$sum: '$repos.count'}, repos: 1}},
+    {$sort:{ total: -1}},
+    {$limit: limit}
+  ]);
+  return cursor.toArray();
+}
+
+function mongo_universe_contributions(user, limit = 20){
+  var query = {_type: 'src', '_contributors.user': user, '_indexed' : true, '_maintainer.login': {$ne: user}};
+  var cursor = mongo_aggregate([
+    {$match: query},
+    {$addFields: {contrib: {$arrayElemAt:['$_contributors', { $indexOfArray: [ "$_contributors.user", user ]}]}}},
+    {$group: {
+      _id: "$_upstream",
+      owner: {$first: '$_user'}, //equals upstream org
+      packages: {$addToSet: '$Package'},
+      maintainers: {$addToSet: '$_maintainer.login'}, //upstreams can have multiple pkgs and maintainers
+      contributions: {$max: '$contrib.count'}
+    }},
+    {$project: {_id:0, contributions:'$contributions', upstream: '$_id', owner: '$owner', packages: '$packages', maintainers: '$maintainers'}},
+    {$sort:{ contributions: -1}},
+    {$limit: limit}
+  ]);
+  return cursor.toArray();
+}
+
 function mongo_all_universes(organizations_only){
   var query = {_type: 'src', _registered: true};
   if(organizations_only){
@@ -479,6 +518,24 @@ export function get_universe_binaries(universe, type){
   }
 }
 
+export function get_universe_contributors(universe, limit){
+  if(production){
+    return mongo_universe_contributors(universe, limit);
+  } else {
+    console.warn(`Fetching contributors data from API...`);
+    return get_ndjson(`https://${universe}.r-universe.dev/stats/contributors?all=1&limit=${limit}`);
+  }
+}
+
+export function get_universe_contributions(universe, limit){
+  if(production){
+    return mongo_universe_contributions(universe, limit);
+  } else {
+    console.warn(`Fetching contributions data from API...`);
+    return get_ndjson(`https://${universe}.r-universe.dev/stats/contributions?limit=${limit}`);
+  }
+}
+
 export function get_repositories(){
   if(production){
     return mongo_all_universes()
@@ -609,6 +666,8 @@ export function get_all_universes(){
   if(production){
     return universes;
   } else {
-    throw "Not implemented for devel";
+    return get_json('https://r-universe.dev/api/universes').then(function(data){
+      return data.map(x => x.universe);
+    });
   }
 }