Skip to content

Commit a6718af

Browse files
bidoubiwabidoubiwa
committed
Add fieldsToIndex as option in backfill script
1 parent c21b466 commit a6718af

File tree

11 files changed

+116
-58
lines changed

11 files changed

+116
-58
lines changed

‎functions/lib/adapter.js

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,17 @@ const logs = require("./logs");
88
* Adapts documents from the Firestore database to Meilisearch compatible documents.
99
* @param {string} documentId Document id.
1010
* @param {DocumentSnapshot} snapshot Snapshot of the data contained in the document read from your Firestore database.
11+
* @param {string} fieldsToIndex list of fields added in the document send to Meilisearch.
1112
* @return {Record<string, any>} A properly formatted document to be added or updated in Meilisearch.
1213
*/
13-
function adaptDocument(documentId, snapshot) {
14-
const fields = (0, util_1.getFieldsToIndex)();
14+
function adaptDocument(documentId, snapshot, fieldsToIndex) {
15+
const fields = (0, util_1.parseFieldsToIndex)(fieldsToIndex);
1516
const data = snapshot.data() || {};
1617
if ('_firestore_id' in data) {
1718
delete data.id;
1819
}
19-
if (fields.length === 0) {
20-
return { _firestore_id: documentId, ...data };
21-
}
22-
const document = Object.keys(data).reduce((acc, key) => {
23-
if (fields.includes(key)) {
24-
const [field, value] = adaptValues(key, data[key]);
25-
return { ...acc, [field]: value };
26-
}
27-
return acc;
28-
}, { _firestore_id: documentId });
29-
return document;
20+
const document = (0, util_1.sanitizeDocuments)(fields, data);
21+
return { _firestore_id: documentId, ...document };
3022
}
3123
exports.adaptDocument = adaptDocument;
3224
/**

‎functions/lib/import/config.js

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ program
1919
.option('-i, --index <index>', "The Uid of the index in Meilisearch to import to. (An index will be created if it doesn't already exist.)")
2020
.option('-b, --batch-size [batch-size]', 'Number of documents to stream into Meilisearch at once.', value => parseInt(value, 10), 1000)
2121
.option('-H, --host <host>', 'The Host of your Meilisearch database. Example: http://localhost:7700.')
22-
.option('-a, --api-key <api-key>', 'The Meilisearch API key with permission to perform actions on indexes. Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.');
22+
.option('-a, --api-key <api-key>', 'The Meilisearch API key with permission to perform actions on indexes. Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.')
23+
.option('-f, --fields-to-index <fields-to-index>', 'test');
2324
const validateInput = (value, name, regex, sizeLimit) => {
2425
if (!value || typeof value !== 'string' || value.trim() === '') {
2526
return `Please supply a ${name}`;
@@ -82,6 +83,12 @@ const questions = [
8283
name: 'apiKey',
8384
type: 'input',
8485
},
86+
{
87+
message: ' What fields do you want to index in Meilisearch? Create a comma-separated list of the field names, or leave it blank to include all fields. The id field is always indexed even when omitted from the list.',
88+
name: 'fieldsToIndex',
89+
default: '*',
90+
type: 'input',
91+
},
8592
];
8693
/**
8794
* Parse the argument from the interactive or non-interactive command line.
@@ -108,10 +115,11 @@ async function parseConfig() {
108115
indexUid: options.index,
109116
host: options.host,
110117
apiKey: options.apiKey,
118+
fieldsToIndex: options.fieldsToIndex,
111119
},
112120
};
113121
}
114-
const { project, sourceCollectionPath, queryCollectionGroup, index, batchSize, host, apiKey, } = await inquirer.prompt(questions);
122+
const { project, sourceCollectionPath, queryCollectionGroup, index, batchSize, host, apiKey, fieldsToIndex, } = await inquirer.prompt(questions);
115123
return {
116124
projectId: project,
117125
sourceCollectionPath: sourceCollectionPath,
@@ -121,6 +129,7 @@ async function parseConfig() {
121129
indexUid: index,
122130
host: host,
123131
apiKey: apiKey,
132+
fieldsToIndex: fieldsToIndex,
124133
},
125134
};
126135
}

‎functions/lib/import/index.js

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ const run = async () => {
4141
* @param {Index} index
4242
*/
4343
async function retrieveCollectionFromFirestore(database, config, index) {
44+
var _a;
4445
const batch = parseInt(config.batchSize);
4546
let query;
4647
let total = 0;
@@ -62,7 +63,7 @@ async function retrieveCollectionFromFirestore(database, config, index) {
6263
const docs = snapshot.docs;
6364
if (docs.length === 0)
6465
break;
65-
total += await sendDocumentsToMeilisearch(docs, index);
66+
total += await sendDocumentsToMeilisearch(docs, index, ((_a = config.meilisearch) === null || _a === void 0 ? void 0 : _a.fieldsToIndex) || '');
6667
if (docs.length) {
6768
lastDocument = docs[docs.length - 1];
6869
}
@@ -75,11 +76,11 @@ async function retrieveCollectionFromFirestore(database, config, index) {
7576
* Adapts documents and indexes them in Meilisearch.
7677
* @param {any} docs
7778
* @param {Index} index
78-
* @param {Change<DocumentSnapshot>} change
79+
* @param {string} fieldsToIndex list of fields added in the document send to Meilisearch.
7980
*/
80-
async function sendDocumentsToMeilisearch(docs, index) {
81+
async function sendDocumentsToMeilisearch(docs, index, fieldsToIndex) {
8182
const document = docs.map(snapshot => {
82-
return (0, adapter_1.adaptDocument)(snapshot.id, snapshot);
83+
return (0, adapter_1.adaptDocument)(snapshot.id, snapshot, fieldsToIndex);
8384
});
8485
try {
8586
await index.addDocuments(document, { primaryKey: '_firestore_id' });

‎functions/lib/index.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ async function handleAddDocument(documentId, snapshot) {
5656
try {
5757
logs.addDocument(documentId);
5858
if ((0, validate_1.validateDocumentId)(documentId)) {
59-
const document = (0, adapter_1.adaptDocument)(documentId, snapshot);
59+
const document = (0, adapter_1.adaptDocument)(documentId, snapshot, config_1.config.meilisearch.fieldsToIndex);
6060
const { taskUid } = await index.addDocuments([document], {
6161
primaryKey: '_firestore_id',
6262
});
@@ -98,7 +98,7 @@ async function handleUpdateDocument(documentId, after) {
9898
try {
9999
logs.updateDocument(documentId);
100100
if ((0, validate_1.validateDocumentId)(documentId)) {
101-
const document = (0, adapter_1.adaptDocument)(documentId, after);
101+
const document = (0, adapter_1.adaptDocument)(documentId, after, config_1.config.meilisearch.fieldsToIndex);
102102
const { taskUid } = await index.addDocuments([document]);
103103
firebase_functions_1.logger.info(`Document update request for document with ID ${documentId} added to task list (task ID ${taskUid}).`);
104104
}

‎functions/lib/util.js

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
'use strict';
22
Object.defineProperty(exports, "__esModule", { value: true });
3-
exports.getFieldsToIndex = exports.getChangedDocumentId = exports.getChangeType = exports.ChangeType = void 0;
4-
const config_1 = require("./config");
3+
exports.sanitizeDocuments = exports.parseFieldsToIndex = exports.getChangedDocumentId = exports.getChangeType = exports.ChangeType = void 0;
54
var ChangeType;
65
(function (ChangeType) {
76
ChangeType[ChangeType["CREATE"] = 0] = "CREATE";
@@ -36,12 +35,35 @@ function getChangedDocumentId(change) {
3635
}
3736
exports.getChangedDocumentId = getChangedDocumentId;
3837
/**
39-
* Returns the MEILISEARCH_FIELDS_TO_INDEX value from the config file and formats it.
38+
* Parse the fieldsToIndex string into an array.
39+
*
40+
* @param {string} fieldsToIndex
4041
* @return {string[]} An array of fields.
4142
*/
42-
function getFieldsToIndex() {
43-
return config_1.config.meilisearch.fieldsToIndex
44-
? config_1.config.meilisearch.fieldsToIndex.split(/[ ,]+/)
45-
: [];
43+
function parseFieldsToIndex(fieldsToIndex) {
44+
return fieldsToIndex ? fieldsToIndex.split(/[ ,]+/) : [];
4645
}
47-
exports.getFieldsToIndex = getFieldsToIndex;
46+
exports.parseFieldsToIndex = parseFieldsToIndex;
47+
/**
48+
* Remove unwanted fields from the document before it is send to Meilisearch.
49+
*
50+
* @param {string[]} fieldsToIndex
51+
* @param {Record<string, any>} document
52+
* @return {Record<string, any>} sanitized document
53+
*
54+
*/
55+
function sanitizeDocuments(fieldsToIndex, document) {
56+
if (fieldsToIndex.length === 0) {
57+
return document;
58+
}
59+
if (fieldsToIndex.includes('*')) {
60+
return document;
61+
}
62+
for (const key in document) {
63+
if (!fieldsToIndex.includes(key)) {
64+
delete document[key];
65+
}
66+
}
67+
return document;
68+
}
69+
exports.sanitizeDocuments = sanitizeDocuments;

‎functions/src/adapter.ts

Lines changed: 10 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
import { DocumentSnapshot } from 'firebase-functions/lib/providers/firestore'
1919
import { firestore } from 'firebase-admin/lib/firestore'
20-
import { getFieldsToIndex } from './util'
20+
import { parseFieldsToIndex, sanitizeDocuments } from './util'
2121
import * as logs from './logs'
2222

2323
type MeilisearchGeoPoint = {
@@ -41,31 +41,24 @@ type FirestoreRow =
4141
* Adapts documents from the Firestore database to Meilisearch compatible documents.
4242
* @param {string} documentId Document id.
4343
* @param {DocumentSnapshot} snapshot Snapshot of the data contained in the document read from your Firestore database.
44+
* @param {string} fieldsToIndex list of fields added in the document send to Meilisearch.
4445
* @return {Record<string, any>} A properly formatted document to be added or updated in Meilisearch.
4546
*/
4647
export function adaptDocument(
4748
documentId: string,
48-
snapshot: DocumentSnapshot
49+
snapshot: DocumentSnapshot,
50+
fieldsToIndex
4951
): Record<string, any> {
50-
const fields = getFieldsToIndex()
52+
const fields = parseFieldsToIndex(fieldsToIndex)
53+
5154
const data = snapshot.data() || {}
5255
if ('_firestore_id' in data) {
5356
delete data.id
5457
}
55-
if (fields.length === 0) {
56-
return { _firestore_id: documentId, ...data }
57-
}
58-
const document = Object.keys(data).reduce(
59-
(acc, key) => {
60-
if (fields.includes(key)) {
61-
const [field, value] = adaptValues(key, data[key])
62-
return { ...acc, [field]: value }
63-
}
64-
return acc
65-
},
66-
{ _firestore_id: documentId }
67-
)
68-
return document
58+
59+
const document = sanitizeDocuments(fields, data)
60+
61+
return { _firestore_id: documentId, ...document }
6962
}
7063

7164
/**

‎functions/src/import/config.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ program
4848
'-a, --api-key <api-key>',
4949
'The Meilisearch API key with permission to perform actions on indexes. Both the private key and the master key are valid choices but we strongly recommend using the private key for security purposes.'
5050
)
51+
.option('-f, --fields-to-index <fields-to-index>', 'test')
5152

5253
const validateInput = (
5354
value: string,

‎functions/src/import/index.ts

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,11 @@ async function retrieveCollectionFromFirestore(
7878
const docs = snapshot.docs
7979

8080
if (docs.length === 0) break
81-
total += await sendDocumentsToMeilisearch(docs, index)
81+
total += await sendDocumentsToMeilisearch(
82+
docs,
83+
index,
84+
config.meilisearch?.fieldsToIndex || ''
85+
)
8286

8387
if (docs.length) {
8488
lastDocument = docs[docs.length - 1]
@@ -94,14 +98,15 @@ async function retrieveCollectionFromFirestore(
9498
* Adapts documents and indexes them in Meilisearch.
9599
* @param {any} docs
96100
* @param {Index} index
97-
* @param {Change<DocumentSnapshot>} change
101+
* @param {string} fieldsToIndex list of fields added in the document send to Meilisearch.
98102
*/
99103
async function sendDocumentsToMeilisearch(
100104
docs: DocumentSnapshot[],
101-
index: Index
105+
index: Index,
106+
fieldsToIndex: string
102107
): Promise<number> {
103108
const document = docs.map(snapshot => {
104-
return adaptDocument(snapshot.id, snapshot)
109+
return adaptDocument(snapshot.id, snapshot, fieldsToIndex)
105110
})
106111
try {
107112
await index.addDocuments(document, { primaryKey: '_firestore_id' })

‎functions/src/index.ts

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,11 @@ async function handleAddDocument(
6666
try {
6767
logs.addDocument(documentId)
6868
if (validateDocumentId(documentId)) {
69-
const document = adaptDocument(documentId, snapshot)
69+
const document = adaptDocument(
70+
documentId,
71+
snapshot,
72+
config.meilisearch.fieldsToIndex
73+
)
7074
const { taskUid } = await index.addDocuments([document], {
7175
primaryKey: '_firestore_id',
7276
})
@@ -119,7 +123,11 @@ async function handleUpdateDocument(
119123
try {
120124
logs.updateDocument(documentId)
121125
if (validateDocumentId(documentId)) {
122-
const document = adaptDocument(documentId, after)
126+
const document = adaptDocument(
127+
documentId,
128+
after,
129+
config.meilisearch.fieldsToIndex
130+
)
123131
const { taskUid } = await index.addDocuments([document])
124132

125133
logger.info(

‎functions/src/util.ts

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
import { DocumentSnapshot } from 'firebase-functions/lib/providers/firestore'
1919
import { Change } from 'firebase-functions'
20-
import { config } from './config'
2120

2221
export enum ChangeType {
2322
CREATE,
@@ -53,11 +52,38 @@ export function getChangedDocumentId(change: Change<DocumentSnapshot>): string {
5352
}
5453

5554
/**
56-
* Returns the MEILISEARCH_FIELDS_TO_INDEX value from the config file and formats it.
55+
* Parse the fieldsToIndex string into an array.
56+
*
57+
* @param {string} fieldsToIndex
5758
* @return {string[]} An array of fields.
5859
*/
59-
export function getFieldsToIndex(): string[] {
60-
return config.meilisearch.fieldsToIndex
61-
? config.meilisearch.fieldsToIndex.split(/[ ,]+/)
62-
: []
60+
export function parseFieldsToIndex(fieldsToIndex: string): string[] {
61+
return fieldsToIndex ? fieldsToIndex.split(/[ ,]+/) : []
62+
}
63+
64+
/**
65+
* Remove unwanted fields from the document before it is send to Meilisearch.
66+
*
67+
* @param {string[]} fieldsToIndex
68+
* @param {Record<string, any>} document
69+
* @return {Record<string, any>} sanitized document
70+
*
71+
*/
72+
export function sanitizeDocuments(
73+
fieldsToIndex: string[],
74+
document: Record<string, any>
75+
): Record<string, any> {
76+
if (fieldsToIndex.length === 0) {
77+
return document
78+
}
79+
if (fieldsToIndex.includes('*')) {
80+
return document
81+
}
82+
83+
for (const key in document) {
84+
if (!fieldsToIndex.includes(key)) {
85+
delete document[key]
86+
}
87+
}
88+
return document
6389
}

‎test-params-example.env

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ MEILISEARCH_FIELDS_TO_INDEX=''
77
MEILISEARCH_INDEX_NAME=movies
88
MEILISEARCH_HOST='http://127.0.0.1:7700'
99
MEILISEARCH_API_KEY='masterKey'
10+
MEILISEARCH_FIELDS_TO_INDEX=

0 commit comments

Comments
 (0)