Skip to content

Commit 315cf87

Browse files
RubenVerborghRubenVerborgh
committed
Pass datasource metadata as a property.
With the previous event-based implementation, views might miss the `metadata` event if they attach late.
1 parent 3a8a738 commit 315cf87

20 files changed

+101
-109
lines changed

‎bin/generate-summary

+1-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ function fromDataSource(uri, datasource, callback, end, chunksize) {
8585
var count = 0;
8686

8787
var stream = datasource.select({ limit: limit, offset: offset }, console.error)
88-
.on('metadata', function (metadata) {
88+
.getProperty('metadata', function (metadata) {
8989
var progress = Math.round((offset / metadata.totalCount) * 100);
9090
console.log(progress);
9191

‎lib/controllers/SummaryController.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ SummaryController.prototype._handleRequest = function (request, response, next)
4242

4343
// Render the summary
4444
var view = this._negotiateView('Summary', request, response);
45-
view.render({ prefixes: this._prefixes, resultStream: streamParser }, request, response);
45+
view.render({ prefixes: this._prefixes, results: streamParser }, request, response);
4646
}
4747
else
4848
next();

‎lib/controllers/TriplePatternFragmentsController.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ TriplePatternFragmentsController.prototype._handleRequest = function (request, r
3636
// Generate the query result
3737
var view = this._negotiateView('TriplePatternFragments', request, response),
3838
settings = this._createFragmentMetadata(request, query, datasourceSettings);
39-
settings.resultStream = datasourceSettings.datasource.select(query,
40-
function (error) { error && next(error); });
39+
settings.results = datasourceSettings.datasource.select(query,
40+
function (error) { error && next(error); });
4141

4242
// Execute the extensions and render the query result
4343
var extensions = this._extensions, extensionId = 0;

‎lib/datasources/CompositeDatasource.js

+37-33
Original file line numberDiff line numberDiff line change
@@ -89,60 +89,64 @@ CompositeDatasource.prototype._getDatasourceInfo = function (query, absoluteOffs
8989
// We checked all datasources, return our accumulated information
9090
callback(chosenDatasource, chosenOffset, totalCount, hasExactCount);
9191
else {
92-
var ignoreTriples = { _push: noop, close: noop };
9392
var datasource = self._getDatasourceById(datasourceIndex);
94-
datasource._executeQuery(emptyQuery, ignoreTriples, function (metadata) {
95-
var count = metadata.totalCount;
96-
var exact = metadata.hasExactCount;
97-
// If we are still looking for an appropriate datasource, we need exact counts!
98-
if (offset > 0 && !exact) {
99-
self._getExactCount(datasource, query, function (exactCount) {
100-
count = exactCount;
101-
exact = true;
102-
continueRecursion();
103-
});
104-
}
105-
else
106-
continueRecursion();
107-
108-
function continueRecursion() {
109-
if (chosenDatasource < 0 && offset < count) {
110-
// We can start querying from this datasource
111-
setImmediate(function () {
112-
findRecursive(datasourceIndex + 1, offset - count, datasourceIndex, offset,
113-
totalCount + count, hasExactCount && exact);
93+
var metadataReader = {
94+
_push: noop,
95+
close: noop,
96+
setProperty: function (name, metadata) {
97+
if (name !== 'metadata') return;
98+
// If we are still looking for an appropriate datasource, we need exact counts
99+
var count = metadata.totalCount, exact = metadata.hasExactCount;
100+
if (offset > 0 && !exact) {
101+
self._getExactCount(datasource, query, function (exactCount) {
102+
count = exactCount;
103+
exact = true;
104+
continueRecursion();
114105
});
115106
}
116-
else {
117-
// We forward our accumulated information and go check the next datasource
118-
setImmediate(function () {
119-
findRecursive(datasourceIndex + 1, offset - count, chosenDatasource, chosenOffset,
120-
totalCount + count, hasExactCount && exact);
121-
});
107+
else
108+
continueRecursion();
109+
110+
function continueRecursion() {
111+
if (chosenDatasource < 0 && offset < count) {
112+
// We can start querying from this datasource
113+
setImmediate(function () {
114+
findRecursive(datasourceIndex + 1, offset - count, datasourceIndex, offset,
115+
totalCount + count, hasExactCount && exact);
116+
});
117+
}
118+
else {
119+
// We forward our accumulated information and go check the next datasource
120+
setImmediate(function () {
121+
findRecursive(datasourceIndex + 1, offset - count, chosenDatasource, chosenOffset,
122+
totalCount + count, hasExactCount && exact);
123+
});
124+
}
122125
}
123-
}
124-
});
126+
},
127+
};
128+
datasource._executeQuery(emptyQuery, metadataReader);
125129
}
126130
}
127131
};
128132

129133
// Writes the results of the query to the given triple stream
130-
CompositeDatasource.prototype._executeQuery = function (query, destination, metadataCallback) {
134+
CompositeDatasource.prototype._executeQuery = function (query, destination) {
131135
var offset = query.offset || 0, limit = query.limit || Infinity;
132136
var self = this;
133137
this._getDatasourceInfo(query, offset, function (datasourceIndex, relativeOffset, totalCount, hasExactCount) {
134138
if (datasourceIndex < 0) {
135139
// No valid datasource has been found
136-
metadataCallback({ totalCount: totalCount, hasExactCount: hasExactCount });
140+
destination.setProperty('metadata', { totalCount: totalCount, hasExactCount: hasExactCount });
137141
destination.close();
138142
}
139143
else {
140144
// Send query to first applicable datasource and optionally emit triples from consecutive datasources
141-
metadataCallback({ totalCount: totalCount, hasExactCount: hasExactCount });
142-
var emitted = 0;
145+
destination.setProperty('metadata', { totalCount: totalCount, hasExactCount: hasExactCount });
143146

144147
// Modify our triple stream so that if all results from one datasource have arrived,
145148
// check if we haven't reached the limit and if so, trigger a new query for the next datasource.
149+
var emitted = 0;
146150
countItems(destination, function (localEmittedCount) {
147151
// This is called after the last element has been pushed
148152

‎lib/datasources/Datasource.js

+3-6
Original file line numberDiff line numberDiff line change
@@ -112,20 +112,17 @@ Datasource.prototype.select = function (query, onError) {
112112
if (triple.object[0] === '_') triple.object = blankNodePrefix + triple.object.substr(2);
113113
return triple;
114114
});
115+
outputTriples.copyProperties(destination, ['metadata']);
115116
onError && outputTriples.on('error', onError);
116117

117118
// Execute the query
118-
try {
119-
this._executeQuery(query, destination, function (metadata) {
120-
setImmediate(function () { outputTriples.emit('metadata', metadata); });
121-
});
122-
}
119+
try { this._executeQuery(query, destination); }
123120
catch (error) { outputTriples.emit('error', error); }
124121
return outputTriples;
125122
};
126123

127124
// Writes the results of the query to the given destination
128-
Datasource.prototype._executeQuery = function (query, destination, metadataCallback) {
125+
Datasource.prototype._executeQuery = function (query, destination) {
129126
throw new Error('_executeQuery has not been implemented');
130127
};
131128

‎lib/datasources/ExternalHdtDatasource.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ ExternalHdtDatasource.prototype._initialize = function (done) {
3333
};
3434

3535
// Writes the results of the query to the given triple stream
36-
ExternalHdtDatasource.prototype._executeQuery = function (query, destination, metadataCallback) {
36+
ExternalHdtDatasource.prototype._executeQuery = function (query, destination) {
3737
// Execute the external HDT utility
3838
var hdtFile = this._hdtFile, offset = query.offset || 0, limit = query.limit || Infinity,
3939
hdt = spawn(hdtUtility, ['--query', (query.subject || '?s') + ' ' +
@@ -52,7 +52,7 @@ ExternalHdtDatasource.prototype._executeQuery = function (query, destination, me
5252
// Ensure the estimated total count is as least as large as the number of triples
5353
if (tripleCount && estimatedTotalCount < offset + tripleCount)
5454
estimatedTotalCount = offset + (tripleCount < query.limit ? tripleCount : 2 * tripleCount);
55-
metadataCallback({ totalCount: estimatedTotalCount, hasExactCount: hasExactCount });
55+
destination.setProperty('metadata', { totalCount: estimatedTotalCount, hasExactCount: hasExactCount });
5656
destination.close();
5757
}
5858
});

‎lib/datasources/HdtDatasource.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ HdtDatasource.prototype._initialize = function (done) {
2828
};
2929

3030
// Writes the results of the query to the given triple stream
31-
HdtDatasource.prototype._executeQuery = function (query, destination, metadataCallback) {
31+
HdtDatasource.prototype._executeQuery = function (query, destination) {
3232
this._hdtDocument.search(query.subject, query.predicate, query.object,
3333
{ limit: query.limit, offset: query.offset },
3434
function (error, triples, estimatedTotalCount, hasExactCount) {
@@ -37,7 +37,7 @@ HdtDatasource.prototype._executeQuery = function (query, destination, metadataCa
3737
var tripleCount = triples.length, offset = query.offset || 0;
3838
if (tripleCount && estimatedTotalCount < offset + tripleCount)
3939
estimatedTotalCount = offset + (tripleCount < query.limit ? tripleCount : 2 * tripleCount);
40-
metadataCallback({ totalCount: estimatedTotalCount, hasExactCount: hasExactCount });
40+
destination.setProperty('metadata', { totalCount: estimatedTotalCount, hasExactCount: hasExactCount });
4141
// Add the triples to the output
4242
for (var i = 0; i < tripleCount; i++)
4343
destination._push(triples[i]);

‎lib/datasources/MemoryDatasource.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,11 @@ MemoryDatasource.prototype._getAllTriples = function (addTriple, done) {
2424
};
2525

2626
// Writes the results of the query to the given triple stream
27-
MemoryDatasource.prototype._executeQuery = function (query, destination, metadataCallback) {
28-
var offset = query.offset || 0, limit = query.limit || Infinity,
27+
MemoryDatasource.prototype._executeQuery = function (query, destination) {
28+
var offset = query.offset || 0, limit = query.limit || Infinity,
2929
triples = this._tripleStore.findByIRI(query.subject, query.predicate, query.object);
3030
// Send the metadata
31-
metadataCallback({ totalCount: triples.length, hasExactCount: true });
31+
destination.setProperty('metadata', { totalCount: triples.length, hasExactCount: true });
3232
// Send the requested subset of triples
3333
for (var i = offset, l = Math.min(offset + limit, triples.length); i < l; i++)
3434
destination._push(triples[i]);

‎lib/datasources/SparqlDatasource.js

+5-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ function SparqlDatasource(options) {
2626
Datasource.extend(SparqlDatasource, ['triplePattern', 'limit', 'offset', 'totalCount']);
2727

2828
// Writes the results of the query to the given triple stream
29-
SparqlDatasource.prototype._executeQuery = function (query, destination, metadataCallback) {
29+
SparqlDatasource.prototype._executeQuery = function (query, destination) {
3030
// Create the HTTP request
3131
var sparqlPattern = this._createTriplePattern(query), self = this,
3232
constructQuery = this._createConstructQuery(sparqlPattern, query.offset, query.limit),
@@ -60,8 +60,10 @@ SparqlDatasource.prototype._executeQuery = function (query, destination, metadat
6060

6161
// Determine the total number of matching triples
6262
this._getPatternCount(sparqlPattern, function (error, totalCount) {
63-
if (error) emitError(error);
64-
else if (typeof totalCount === 'number') metadataCallback({ totalCount: totalCount, hasExactCount: true });
63+
if (error)
64+
emitError(error);
65+
else if (typeof totalCount === 'number')
66+
destination.setProperty('metadata', { totalCount: totalCount, hasExactCount: true });
6567
});
6668

6769
// Emits an error on the triple stream

‎lib/views/summary/SummaryRdfView.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ RdfView.extend(SummaryRdfView);
1414
// Generates triples and quads by sending them to the data and/or metadata callbacks
1515
SummaryRdfView.prototype._generateRdf = function (settings, data, metadata, done) {
1616
// Add summary triples
17-
settings.resultStream.on('data', data);
18-
settings.resultStream.on('end', done);
17+
settings.results.on('data', data);
18+
settings.results.on('end', done);
1919
};
2020

2121
module.exports = SummaryRdfView;

‎lib/views/triplepatternfragments/TriplePatternFragmentsHtmlView.js

+7-4
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,13 @@ HtmlView.extend(TriplePatternFragmentsHtmlView);
1414
// Renders the view with the given settings to the response
1515
TriplePatternFragmentsHtmlView.prototype._render = function (settings, request, response, done) {
1616
// Read the data and metadata
17-
var self = this, triples = [], resultStream = settings.resultStream;
18-
resultStream.on('data', function (triple) { triples.push(triple); });
19-
resultStream.on('end', function () { settings.triples = triples; settings.metadata && renderHtml(); });
20-
resultStream.on('metadata', function (m) { settings.metadata = m; settings.triples && renderHtml(); });
17+
var self = this, triples = settings.triples = [], results = settings.results;
18+
results.on('data', function (triple) { triples.push(triple); });
19+
results.on('end', function () { settings.metadata && renderHtml(); });
20+
results.getProperty('metadata', function (metadata) {
21+
settings.metadata = metadata;
22+
results.ended && renderHtml();
23+
});
2124

2225
// Generates the HTML after the data and metadata have been retrieved
2326
function renderHtml() {

‎lib/views/triplepatternfragments/TriplePatternFragmentsRdfView.js

+8-12
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ RdfView.extend(TriplePatternFragmentsRdfView);
1919

2020
// Generates triples and quads by sending them to the data and/or metadata callbacks
2121
TriplePatternFragmentsRdfView.prototype._generateRdf = function (settings, data, metadata, done) {
22-
var metadataDone = false, triplesDone = false,
23-
datasource = settings.datasource, fragment = settings.fragment, query = settings.query;
22+
var datasource = settings.datasource, fragment = settings.fragment, query = settings.query,
23+
results = settings.results, metadataDone = false;
2424

2525
// Add data source metadata
2626
metadata(datasource.index, hydra + 'member', datasource.url);
@@ -46,7 +46,7 @@ TriplePatternFragmentsRdfView.prototype._generateRdf = function (settings, data,
4646
metadata('_:object', hydra + 'property', rdf + 'object');
4747

4848
// Add fragment metadata
49-
settings.resultStream.on('metadata', function (meta) {
49+
results.getProperty('metadata', function (meta) {
5050
// General fragment metadata
5151
metadata(fragment.url, voID + 'subset', fragment.pageUrl);
5252
metadata(fragment.pageUrl, rdf + 'type', hydra + 'PartialCollectionView');
@@ -70,18 +70,14 @@ TriplePatternFragmentsRdfView.prototype._generateRdf = function (settings, data,
7070
if (totalCount >= query.limit + (query.offset || 0))
7171
metadata(fragment.pageUrl, hydra + 'next', fragment.nextPageUrl);
7272

73-
// End if both the metadata and the data have been written
73+
// End if the data was also written
7474
metadataDone = true;
75-
triplesDone && done();
75+
results.ended && done();
7676
});
7777

78-
// Add data triples
79-
settings.resultStream.on('data', data);
80-
settings.resultStream.on('end', function () {
81-
// End if both the metadata and the data have been written
82-
triplesDone = true;
83-
metadataDone && done();
84-
});
78+
// Add fragment data
79+
results.on('data', data);
80+
results.on('end', function () { metadataDone && done(); });
8581
};
8682

8783
module.exports = TriplePatternFragmentsRdfView;

‎test/controllers/TriplePatternFragmentsController-test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ describe('TriplePatternFragmentsController', function () {
140140
nextPageUrl: 'https://example.org/my-datasource?a=b&c=d&page=2',
141141
previousPageUrl: null,
142142
},
143-
resultStream: {
143+
results: {
144144
stream: 'items',
145145
},
146146
prefixes: prefixes,

‎test/datasources/CompositeDatasource-test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ function itShouldExecute(getDatasource, name, query,
119119
var resultsCount = 0, totalCount, triples = [];
120120
before(function (done) {
121121
var result = getDatasource().select(query);
122-
result.on('metadata', function (metadata) { totalCount = metadata.totalCount; });
122+
result.getProperty('metadata', function (metadata) { totalCount = metadata.totalCount; });
123123
result.on('data', function (triple) { resultsCount++; expectedTriples && triples.push(triple); });
124124
result.on('end', done);
125125
});

‎test/datasources/HdtDatasource-test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ function itShouldExecute(getDatasource, name, query,
193193
var resultsCount = 0, totalCount, triples = [];
194194
before(function (done) {
195195
var result = getDatasource().select(query);
196-
result.on('metadata', function (metadata) { totalCount = metadata.totalCount; });
196+
result.getProperty('metadata', function (metadata) { totalCount = metadata.totalCount; });
197197
result.on('data', function (triple) { resultsCount++; expectedTriples && triples.push(triple); });
198198
result.on('end', done);
199199
});

‎test/datasources/JsonLdDatasource-test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ function itShouldExecute(datasource, name, query, expectedResultsCount, expected
8787
var resultsCount = 0, totalCount;
8888
before(function (done) {
8989
var result = datasource.select(query);
90-
result.on('metadata', function (metadata) { totalCount = metadata.totalCount; });
90+
result.getProperty('metadata', function (metadata) { totalCount = metadata.totalCount; });
9191
result.on('data', function (triple) { resultsCount++; });
9292
result.on('end', done);
9393
});

‎test/datasources/SparqlDatasource-test.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ describe('SparqlDatasource', function () {
152152
request.onThirdCall().returns(test.createHttpResponse(turtleResult, 'text/turtle'));
153153

154154
result = datasource.select({ subject: 'abc', features: { triplePattern: true } });
155-
result.on('metadata', function (metadata) { totalCount = metadata.totalCount; });
155+
result.getProperty('metadata', function (metadata) { totalCount = metadata.totalCount; });
156156
firstArgsCopy = JSON.parse(JSON.stringify(request.firstCall.args[0]));
157157
});
158158

@@ -271,7 +271,7 @@ function itShouldExecute(datasource, request, name, query, constructQuery, count
271271
request.onSecondCall().returns(test.createHttpResponse(countResult, 'text/csv'));
272272

273273
result = datasource.select(query);
274-
result.on('metadata', function (metadata) { totalCount = metadata.totalCount; });
274+
result.getProperty('metadata', function (metadata) { totalCount = metadata.totalCount; });
275275
});
276276

277277
it('should request a matching CONSTRUCT query', function () {

‎test/datasources/TurtleDatasource-test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ function itShouldExecute(datasource, name, query, expectedResultsCount, expected
8787
var resultsCount = 0, totalCount;
8888
before(function (done) {
8989
var result = datasource.select(query);
90-
result.on('metadata', function (metadata) { totalCount = metadata.totalCount; });
90+
result.getProperty('metadata', function (metadata) { totalCount = metadata.totalCount; });
9191
result.on('data', function (triple) { resultsCount++; });
9292
result.on('end', done);
9393
});

‎test/test-setup.js

-10
Original file line numberDiff line numberDiff line change
@@ -46,16 +46,6 @@ test.createStreamCapture = function () {
4646
return stream;
4747
};
4848

49-
// Creates a readable stream from an array
50-
test.streamFromArray = function (items) {
51-
var stream = new Readable({ objectMode: true });
52-
stream._read = function () {
53-
var item = items.shift();
54-
this.push(item || null);
55-
};
56-
return stream;
57-
};
58-
5949
chai.use(function (chai, utils) {
6050
// Checks whether the stream contains the given number of elements
6151
chai.Assertion.addMethod('streamWithLength', function (expectedLength, callback) {

0 commit comments

Comments
 (0)