diff --git a/app/scripts/translatte/commands/exportMigration.ts b/app/scripts/translatte/commands/exportMigration.ts index 6572868a1..fa059c678 100644 --- a/app/scripts/translatte/commands/exportMigration.ts +++ b/app/scripts/translatte/commands/exportMigration.ts @@ -53,7 +53,7 @@ async function exportMigration( }); const fileName = isNotDefined(exportFileName) - ? `go-strings-${yyyy}-${mm}-${dd}` + ? `go-migration-strings-${yyyy}-${mm}-${dd}` : exportFileName; await workbook.xlsx.writeFile(`${fileName}.xlsx`); diff --git a/app/scripts/translatte/commands/exportServerStringsToExcel.ts b/app/scripts/translatte/commands/exportServerStringsToExcel.ts new file mode 100644 index 000000000..94f6f5387 --- /dev/null +++ b/app/scripts/translatte/commands/exportServerStringsToExcel.ts @@ -0,0 +1,71 @@ +import xlsx from 'exceljs'; + +import { fetchServerState } from "../utils"; +import { isFalsyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; + +async function exportServerStringsToExcel( + apiUrl: string, + authToken?: string, + exportFileName?: string, +) { + const serverStrings = await fetchServerState(apiUrl, authToken); + + const workbook = new xlsx.Workbook(); + const now = new Date(); + workbook.created = now; + + const yyyy = now.getFullYear(); + const mm = (now.getMonth() + 1).toString().padStart(2, '0'); + const dd = now.getDate().toString().padStart(2, '0'); + const HH = now.getHours().toString().padStart(2, '0'); + const MM = now.getMinutes().toString().padStart(2, '0'); + + const worksheet = workbook.addWorksheet( + `${yyyy}-${mm}-${dd} ${HH}-${MM}` + ); + + worksheet.columns = [ + { header: 'Namespace', key: 'namespace' }, + { header: 'Key', key: 'key' }, + { header: 'EN', key: 'en' }, + { header: 'FR', key: 'fr' }, + { header: 'ES', key: 'es' }, + { header: 'AR', key: 'ar' }, + ] + + const keyGroupedStrings = mapToList( + listToGroupList( + serverStrings, + ({ page_name, key }) => `${page_name}:${key}`, + ), + (list) => { + const value = listToMap( + list, + ({ language }) => language, + ({ value }) => value + ); + const { key, page_name } = list[0]; + + return { + namespace: page_name, + key: key, + en: value.en, + fr: value.fr, + es: value.es, + ar: value.ar, + }; + }, + ); + + Object.values(keyGroupedStrings).forEach((keyGroupedString) => { + worksheet.addRow(keyGroupedString); + }); + + const fileName = isFalsyString(exportFileName) + ? `go-server-strings-${yyyy}-${mm}-${dd}` + : exportFileName; + + await workbook.xlsx.writeFile(`${fileName}.xlsx`); +} + +export default exportServerStringsToExcel; diff --git a/app/scripts/translatte/commands/pushMigration.ts b/app/scripts/translatte/commands/pushMigration.ts new file mode 100644 index 000000000..5fbad4367 --- /dev/null +++ b/app/scripts/translatte/commands/pushMigration.ts @@ -0,0 +1,208 @@ +import { isDefined, isNotDefined, listToGroupList, listToMap, mapToMap } from "@togglecorp/fujs"; +import { Language, MigrationActionItem } from "../types"; +import { fetchServerState, getCombinedKey, languages, postLanguageStrings, readMigrations } from "../utils"; +import { Md5 } from "ts-md5"; + +async function pushMigration(migrationFilePath: string, apiUrl: string, authToken: string) { + const serverStrings = await fetchServerState(apiUrl, authToken); + + const serverStringMapByCombinedKey = mapToMap( + listToGroupList( + serverStrings, + ({ key, page_name }) => getCombinedKey(key, page_name), + ), + (key) => key, + (list) => listToMap( + list, + ({ language }) => language, + ) + ); + + const migrations = await readMigrations( + [migrationFilePath] + ); + + const actions = migrations[0].content.actions; + + + function getItemsForNamespaceUpdate(actionItem: MigrationActionItem, language: Language) { + if (actionItem.action !== 'update') { + return undefined; + } + + if (isNotDefined(actionItem.newNamespace)) { + return undefined; + } + + const oldCombinedKey = getCombinedKey( + actionItem.key, + actionItem.namespace, + ); + + const oldStringItem = serverStringMapByCombinedKey[oldCombinedKey]?.[language]; + + if (isNotDefined(oldStringItem)) { + return undefined; + } + + return [ + { + action: 'delete' as const, + key: actionItem.key, + page_name: actionItem.namespace, + }, + { + action: 'set' as const, + key: actionItem.key, + page_name: actionItem.newNamespace, + value: oldStringItem.value, + hash: oldStringItem.hash, + }, + ]; + } + + function getItemsForKeyUpdate(actionItem: MigrationActionItem, language: Language) { + if (actionItem.action !== 'update') { + return undefined; + } + + if (isNotDefined(actionItem.newKey)) { + return undefined; + } + + const oldCombinedKey = getCombinedKey( + actionItem.key, + actionItem.namespace, + ); + + const oldStringItem = serverStringMapByCombinedKey[oldCombinedKey]?.[language]; + + if (isNotDefined(oldStringItem)) { + return undefined; + } + + return [ + { + action: 'delete' as const, + key: actionItem.key, + page_name: actionItem.namespace, + }, + { + action: 'set' as const, + key: actionItem.newKey, + page_name: actionItem.namespace, + value: oldStringItem.value, + hash: oldStringItem.hash, + }, + ]; + } + + const serverActions = listToMap( + languages.map((language) => { + const serverActionsForCurrentLanguage = actions.flatMap((actionItem) => { + if (language === 'en') { + if (actionItem.action === 'add') { + return { + action: 'set' as const, + key: actionItem.key, + page_name: actionItem.namespace, + value: actionItem.value, + hash: Md5.hashStr(actionItem.value), + } + } + + if (actionItem.action === 'remove') { + return { + action: 'delete' as const, + key: actionItem.key, + page_name: actionItem.namespace, + } + } + + if (isDefined(actionItem.newNamespace)) { + return getItemsForNamespaceUpdate(actionItem, language); + } + + if (isDefined(actionItem.newKey)) { + return getItemsForKeyUpdate(actionItem, language); + } + + if (isDefined(actionItem.newValue)) { + return { + action: 'set' as const, + key: actionItem.key, + page_name: actionItem.namespace, + value: actionItem.newValue, + hash: Md5.hashStr(actionItem.newValue), + } + } + } else { + if (actionItem.action === 'remove') { + return { + action: 'delete' as const, + key: actionItem.key, + page_name: actionItem.namespace, + } + } + + if (actionItem.action === 'update') { + if (isDefined(actionItem.newNamespace)) { + return getItemsForNamespaceUpdate(actionItem, language); + } + + if (isDefined(actionItem.newKey)) { + return getItemsForKeyUpdate(actionItem, language); + } + } + } + + return undefined; + }).filter(isDefined); + + return { + language, + actions: serverActionsForCurrentLanguage, + } + }), + ({ language }) => language, + ); + + console.log('Pusing actions for en...') + const enResponse = await postLanguageStrings( + serverActions.en.language, + serverActions.en.actions, + apiUrl, + authToken, + ); + console.log(await enResponse.json()); + + + console.log('Pusing actions for fr...') + const frResponse = await postLanguageStrings( + serverActions.fr.language, + serverActions.fr.actions, + apiUrl, + authToken, + ); + console.log(await frResponse.json()); + + console.log('Pusing actions for es...') + const esResponse = await postLanguageStrings( + serverActions.es.language, + serverActions.es.actions, + apiUrl, + authToken, + ); + console.log(await esResponse.json()); + + console.log('Pusing actions for ar...') + const arResponse = await postLanguageStrings( + serverActions.ar.language, + serverActions.ar.actions, + apiUrl, + authToken, + ); + console.log(await arResponse.json()); +} + +export default pushMigration; diff --git a/app/scripts/translatte/commands/pushStringsFromExcel.ts b/app/scripts/translatte/commands/pushStringsFromExcel.ts new file mode 100644 index 000000000..b9da7b38b --- /dev/null +++ b/app/scripts/translatte/commands/pushStringsFromExcel.ts @@ -0,0 +1,142 @@ +import xlsx from 'exceljs'; +import { Md5 } from 'ts-md5'; +import { isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; + +import { Language, ServerActionItem } from '../types'; +import { postLanguageStrings } from '../utils'; + +async function pushStringsFromExcel(importFilePath: string, apiUrl: string, accessToken: string) { + const workbook = new xlsx.Workbook(); + + await workbook.xlsx.readFile(importFilePath); + + const firstSheet = workbook.worksheets[0]; + const columns = firstSheet.columns.map( + (column) => { + const key = column.values?.[1]?.toString(); + if (isNotDefined(key)) { + return undefined; + } + return { key, column: column.number } + } + ).filter(isDefined); + + const columnMap = listToMap( + columns, + ({ key }) => key, + ({ column }) => column, + ); + + const strings: { + key: string; + namespace: string; + language: Language; + value: string; + hash: string; + }[] = []; + + firstSheet.eachRow( + (row) => { + const keyColumn = columnMap['key']; + const key = isDefined(keyColumn) ? row.getCell(keyColumn).value?.toString() : undefined; + + const namespaceColumn = columnMap['namespace']; + const namespace = isDefined(namespaceColumn) ? row.getCell(namespaceColumn).value?.toString() : undefined; + + if (isNotDefined(key) || isNotDefined(namespace)) { + return; + } + + const enColumn = columnMap['en']; + const en = isDefined(enColumn) ? row.getCell(enColumn).value?.toString() : undefined; + + const arColumn = columnMap['ar']; + const ar = isDefined(arColumn) ? row.getCell(arColumn).value?.toString() : undefined; + + const frColumn = columnMap['fr']; + const fr = isDefined(frColumn) ? row.getCell(frColumn).value?.toString() : undefined; + + const esColumn = columnMap['es']; + const es = isDefined(esColumn) ? row.getCell(esColumn).value?.toString() : undefined; + + if (isNotDefined(en)) { + return; + } + + const hash = Md5.hashStr(en); + + strings.push({ + key, + namespace, + language: 'en', + value: en, + hash, + }); + + if (isDefined(ar)) { + strings.push({ + key, + namespace, + language: 'ar', + value: ar, + hash, + }); + } + + if (isDefined(fr)) { + strings.push({ + key, + namespace, + language: 'fr', + value: fr, + hash, + }); + } + + if (isDefined(es)) { + strings.push({ + key, + namespace, + language: 'es', + value: es, + hash, + }); + } + } + ); + + const languageGroupedActions = mapToList( + listToGroupList( + strings, + ({ language }) => language, + (languageString) => { + const serverAction: ServerActionItem = { + action: 'set', + key: languageString.key, + page_name: languageString.namespace, + value: languageString.value, + hash: languageString.hash, + } + + return serverAction; + }, + ), + (actions, language) => ({ + language: language as Language, + actions, + }) + ); + + const postPromises = languageGroupedActions.map( + (languageStrings) => postLanguageStrings( + languageStrings.language, + languageStrings.actions, + apiUrl, + accessToken, + ) + ) + + await Promise.all(postPromises); +} + +export default pushStringsFromExcel; diff --git a/app/scripts/translatte/main.ts b/app/scripts/translatte/main.ts index 079f9c747..17a1ddbd3 100644 --- a/app/scripts/translatte/main.ts +++ b/app/scripts/translatte/main.ts @@ -1,6 +1,7 @@ import yargs from 'yargs/yargs'; import { hideBin } from 'yargs/helpers'; import { cwd } from 'process'; +import { join, basename } from 'path'; import lint from './commands/lint'; import listMigrations from './commands/listMigrations'; @@ -9,7 +10,9 @@ import mergeMigrations from './commands/mergeMigrations'; import applyMigrations from './commands/applyMigrations'; import generateMigration from './commands/generateMigration'; import exportMigration from './commands/exportMigration'; -import { join, basename } from 'path'; +import pushMigration from './commands/pushMigration'; +import pushStringsFromExcel from './commands/pushStringsFromExcel'; +import exportServerStringsToExcel from './commands/exportServerStringsToExcel'; const currentDir = cwd(); @@ -160,7 +163,7 @@ yargs(hideBin(process.argv)) }, ) .command( - 'export-migration ', + 'export-migration-to-excel ', 'Export migration file to excel format which can be used to translate the new and updated strings', (yargs) => { yargs.positional('MIGRATION_FILE_PATH', { @@ -188,6 +191,97 @@ yargs(hideBin(process.argv)) ); }, ) + .command( + 'push-migration ', + 'Push migration file to the server', + (yargs) => { + yargs.positional('MIGRATION_FILE_PATH', { + type: 'string', + describe: 'Find the migration file on MIGRATION_FILE_PATH', + }); + yargs.options({ + 'api-url': { + type: 'string', + describe: 'URL for the API server', + require: true, + }, + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + }); + }, + async (argv) => { + const migrationFilePath = (argv.MIGRATION_FILE_PATH as string); + + await pushMigration( + migrationFilePath, + argv.apiUrl as string, + argv.authToken as string, + ); + }, + ) + .command( + 'push-strings-from-excel ', + 'Import migration from excel file and push it to server', + (yargs) => { + yargs.positional('IMPORT_FILE_PATH', { + type: 'string', + describe: 'Find the import file on IMPORT_FILE_PATH', + }); + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + 'api-url': { + type: 'string', + describe: 'URL for the API server', + require: true, + } + }); + }, + async (argv) => { + const importFilePath = (argv.IMPORT_FILE_PATH as string); + + await pushStringsFromExcel( + importFilePath, + argv.apiUrl as string, + argv.authToken as string, + ); + }, + ) + .command( + 'export-server-strings ', + 'Export server strings to excel file', + (yargs) => { + yargs.positional('API_URL', { + type: 'string', + describe: 'Fetch server strings from API_URL, language is auto appended (e.g. API_URL/en)', + }); + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: false, + }, + 'output-file-name': { + type: 'string', + describe: 'Output excel file name', + require: false, + }, + }); + }, + async (argv) => { + await exportServerStringsToExcel( + argv.API_URL as string, + argv.authToken as string | undefined, + argv.outputFileName as string | undefined + ); + }, + ) .strictCommands() .showHelpOnFail(false) .parse() diff --git a/app/scripts/translatte/types.ts b/app/scripts/translatte/types.ts index 37e0b9063..02ac4403b 100644 --- a/app/scripts/translatte/types.ts +++ b/app/scripts/translatte/types.ts @@ -48,3 +48,17 @@ export interface SourceFileContent { last_migration?: string; strings: SourceStringItem[]; } + +export type ServerActionItem = { + action: 'set', + key: string, + page_name: string, + value: string, + hash: string, +} | { + action: 'delete' + key: string; + page_name: string; +} + +export type Language = 'en' | 'fr' | 'es' | 'ar' diff --git a/app/scripts/translatte/utils.ts b/app/scripts/translatte/utils.ts index 4b221c380..0bf5ef9c6 100644 --- a/app/scripts/translatte/utils.ts +++ b/app/scripts/translatte/utils.ts @@ -15,6 +15,9 @@ import { TranslationFileContent, MigrationFileContent, SourceFileContent, + Language, + ServerActionItem, + SourceStringItem, } from './types'; const readFilePromisify = promisify(readFile); @@ -23,6 +26,57 @@ const unlinkPromisify = promisify(unlink); // Utilities +export function getCombinedKey(key: string, namespace: string) { + return `${namespace}:${key}`; +} + +export function resolveUrl(from: string, to: string) { + const resolvedUrl = new URL(to, new URL(from, 'resolve://')); + if (resolvedUrl.protocol === 'resolve:') { + const { pathname, search, hash } = resolvedUrl; + return pathname + search + hash; + } + return resolvedUrl.toString(); +} + +export async function fetchLanguageStrings(language: Language, apiUrl: string, authToken?: string) { + const endpoint = resolveUrl(apiUrl, language); + const headers: RequestInit['headers'] = { + 'Accept': 'application/json' + } + + if (isDefined(authToken)) { + headers['Authorization'] = `Token ${authToken}`; + } + + const promise = fetch( + endpoint, + { + method: 'GET', + headers, + } + ); + + return promise; +} + +export async function postLanguageStrings(language: Language, actions: ServerActionItem[], apiUrl: string, authToken: string) { + const endpoint = resolveUrl(apiUrl, language); + const promise = fetch( + endpoint, + { + method: 'POST', + headers: { + 'Authorization': `Token ${authToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ actions }), + } + ); + + return promise; +} + export function oneOneMapping( prevState: T[], currentState: T[], @@ -329,3 +383,37 @@ export async function removeFiles(files: string[]) { )); await Promise.all(removePromises); } + +export const languages: Language[] = ['en', 'fr', 'es', 'ar']; + +export async function fetchServerState(apiUrl: string, authToken?: string) { + const responsePromises = languages.map( + (language) => fetchLanguageStrings(language, apiUrl, authToken) + ); + + const responses = await Promise.all(responsePromises); + + const languageJsonPromises = responses.map( + (response) => response.json() + ); + + const languageStrings = await Promise.all(languageJsonPromises); + + const serverStrings = languageStrings.flatMap( + (languageString) => { + const language: Language = languageString.code; + + const strings: SourceStringItem[] = languageString.strings.map( + (string: Omit) => ({ + ...string, + language, + }) + ) + + return strings; + } + ); + + return serverStrings; +} +