reindex-lgcc/reindex.mjs

518 lines
20 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env node
/**
* reindex.mjs
*
* Reindexa activities_translations o conferences_translations
* desde Directus a Typesense vía bulk upsert.
*
* USO:
* node reindex.mjs --collection activities # (default)
* node reindex.mjs --collection conferences
* node reindex.mjs --collection activities --limit 100 --offset 200
* node reindex.mjs --collection conferences --dry-run --limit 50
*
* Config: copiar .env.example a .env y llenar variables.
*/
const __start = Date.now();
// ── Config desde .env ────────────────────────────────────────────
import { readFileSync, existsSync } from 'node:fs';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const __dir = dirname(fileURLToPath(import.meta.url));
const dotEnv = resolve(__dir, '.env');
function loadEnv(path) {
if (!existsSync(path)) return {};
const env = {};
for (const line of readFileSync(path, 'utf-8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#') || !trimmed.includes('=')) continue;
const eq = trimmed.indexOf('=');
const key = trimmed.slice(0, eq).trim();
const val = trimmed.slice(eq + 1).trim().replace(/^["']|["']$/g, '');
env[key] = val;
}
return env;
}
const dotenv = loadEnv(dotEnv);
// ── CLI args ─────────────────────────────────────────────────────
const ARGS = {};
for (const arg of process.argv.slice(2)) {
if (arg === '--dry-run') ARGS.dryRun = true;
else if (arg.startsWith('--collection=')) ARGS.collection = arg.split('=')[1];
else if (arg === '--collection') ARGS.collection = process.argv[process.argv.indexOf(arg) + 1];
else if (arg.startsWith('--limit=')) ARGS.limit = parseInt(arg.split('=')[1], 10);
else if (arg.startsWith('--limit')) ARGS.limit = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10);
else if (arg.startsWith('--offset=')) ARGS.offset = parseInt(arg.split('=')[1], 10);
else if (arg.startsWith('--offset')) ARGS.offset = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10);
}
// ── CFG ──────────────────────────────────────────────────────────
const CFG = {
directusUrl: dotenv.DIRECTUS_URL || process.env.DIRECTUS_URL || '',
directusToken: dotenv.DIRECTUS_TOKEN || process.env.DIRECTUS_TOKEN || '',
typesenseUrl: dotenv.TYPESENSE_URL || process.env.TYPESENSE_URL || '',
typesenseKey: dotenv.TYPESENSE_ADMIN_KEY || process.env.TYPESENSE_ADMIN_KEY || '',
batchSize: parseInt(dotenv.BATCH_SIZE || process.env.BATCH_SIZE || '500'),
pageLimit: parseInt(dotenv.PAGE_LIMIT || process.env.PAGE_LIMIT || '200'),
limit: ARGS.limit ?? parseInt(dotenv.LIMIT || process.env.LIMIT || '0'),
offset: ARGS.offset ?? parseInt(dotenv.OFFSET || process.env.OFFSET || '0'),
collection: ARGS.collection || 'activities',
dryRun: !!ARGS.dryRun,
};
// ── Definición de colecciones ────────────────────────────────────
const COLLECTIONS = {
activities: {
label: 'activities',
directusCollection: 'activities_translations',
relationField: 'activities_id',
extraFields: ['interventions.text'],
mainCollection: 'activities',
paragraphsCollection: 'activities_paragraphs',
groupField: 'activities_id',
type: 'activities',
buildId: (data, rel, locale) => `${locale}-${data.id}`,
getParagraphHtml: (data) => data.interventions?.[0]?.text ?? '',
draftDefault: true,
isPrivate: (rel) => rel.private ?? false,
},
conferences: {
label: 'conferences',
directusCollection: 'conferences_translations',
relationField: 'conferences_id',
extraFields: [],
mainCollection: 'conferences',
paragraphsCollection: 'conferences_paragraphs',
groupField: 'conferences_id',
type: 'conferences',
buildId: (data, rel, locale) => `${locale}-${rel.wpid ?? data.id ?? ''}`,
getParagraphHtml: (data) => data.html ?? '',
draftDefault: false,
isPrivate: (rel) => !rel.public,
},
};
const COL = COLLECTIONS[CFG.collection];
if (!COL) {
console.error(` ❌ Colección desconocida: "${CFG.collection}". Usa --collection activities | conferences`);
process.exit(1);
}
// ── Helpers ──────────────────────────────────────────────────────
const COLORS = {
red: s => `\x1b[31m${s}\x1b[0m`,
green: s => `\x1b[32m${s}\x1b[0m`,
yellow: s => `\x1b[33m${s}\x1b[0m`,
blue: s => `\x1b[34m${s}\x1b[0m`,
magenta: s => `\x1b[35m${s}\x1b[0m`,
cyan: s => `\x1b[36m${s}\x1b[0m`,
gray: s => `\x1b[90m${s}\x1b[0m`,
bold: s => `\x1b[1m${s}\x1b[0m`,
};
function log(...args) { console.log(...args); }
function elapsed() {
const sec = Math.floor((Date.now() - __start) / 1000);
const m = Math.floor(sec / 60);
const s = sec % 60;
return m > 0 ? `${m}m ${s}s` : `${s}s`;
}
function fmtNum(n) {
return n.toLocaleString('en-US');
}
// ── API calls ────────────────────────────────────────────────────
async function directusFetch(path) {
const url = `${CFG.directusUrl}/items/${path}`;
const sep = path.includes('?') ? '&' : '?';
const fullUrl = `${url}${sep}access_token=${CFG.directusToken}`;
const res = await fetch(fullUrl, {
headers: { 'User-Agent': 'reindex/1.0' },
});
if (!res.ok) {
const text = await res.text().catch(() => '');
throw new Error(`Directus ${res.status} ${res.statusText}: ${text.slice(0, 200)}`);
}
return res.json();
}
async function typesenseImport(collection, docs) {
if (docs.length === 0) return { success: 0, error: 0, errors: [] };
const body = docs.map(d => JSON.stringify(d)).join('\n');
const url = `${CFG.typesenseUrl}/collections/${collection}/documents/import?action=upsert`;
const res = await fetch(url, {
method: 'POST',
headers: {
'X-TYPESENSE-API-KEY': CFG.typesenseKey,
'Content-Type': 'text/plain',
},
body,
});
const text = await res.text();
let success = 0, error = 0;
const errors = [];
if (!text.startsWith('{')) {
throw new Error(`Typesense import devolvió formato inesperado: ${text.slice(0, 200)}`);
}
for (const line of text.trim().split('\n')) {
if (!line.trim()) continue;
try {
const result = JSON.parse(line);
if (result.success) success++;
else { error++; errors.push(result.error || 'desconocido'); }
} catch (e) {
error++;
errors.push(`error parseando línea: ${line.slice(0, 100)}`);
}
}
return { success, error, errors };
}
// ── Transformación ───────────────────────────────────────────────
function getFilename(obj) {
return obj?.filename_disk ?? null;
}
function transformItem(data, col) {
const rel = data[col.relationField] || {};
const dateStr = rel.date;
const dateObj = dateStr ? new Date(dateStr) : null;
const locale = data.languages_code?.code ?? 'es';
const yy = dateObj?.getFullYear?.();
const mm = dateObj ? String(dateObj.getMonth() + 1).padStart(2, '0') : '';
const dd = dateObj ? String(dateObj.getDate()).padStart(2, '0') : '';
const dateFormatted = dateObj ? `${yy}${mm}${dd}` : '';
const code = dateFormatted ? `${dateFormatted}-${rel.activity ?? ''}` : '';
const timestamp = dateObj ? Math.floor(dateObj.getTime() / 1000) : 0;
const dateIso = dateObj ? `${yy}-${mm}-${dd}` : null;
const docId = col.buildId(data, rel, locale);
const files = {};
if (data.youtube?.startsWith('http')) files.youtube = data.youtube;
if (getFilename(data.mp3)) files.audio = getFilename(data.mp3);
if (getFilename(data.video)) files.video = getFilename(data.video);
if (getFilename(data.pdf)) files.simple = getFilename(data.pdf);
if (getFilename(data.pdf_booklet)) files.booklet = getFilename(data.pdf_booklet);
const document = {
id: docId,
code,
locale,
type: col.type,
title: data.title?.replace(/"/g, '\\"') ?? '',
timestamp,
date: dateIso,
activity: rel.activity ?? null,
duration: rel.duration ?? 0,
bible_study: rel.bible_study ?? null,
place: rel.place ?? null,
city: rel.city ?? '',
state: rel.state ?? '',
country: rel.country ?? '',
draft: rel.draft ?? col.draftDefault,
private: col.isPrivate(rel),
files,
thumbnail: getFilename(rel.thumbnail),
directus: '',
wp: String(rel.wpid ?? ''),
slug: data.slug ?? null,
year: yy ? String(yy) : '',
month: mm,
};
for (const key of Object.keys(document)) {
if (document[key] === undefined) delete document[key];
}
if (document.files && Object.keys(document.files).length === 0) {
document.files = {};
}
// ── Párrafos ───────────────────────────────────────────
const bodyHtml = col.getParagraphHtml(data);
const paragraphs = [];
if (bodyHtml) {
const hasPTags = /<p[^>]*>/i.test(bodyHtml);
const rawParagraphs = hasPTags
? bodyHtml.split(/<\/?p[^>]*>/).map(p => p.trim()).filter(Boolean)
: bodyHtml.split('\n\n').map(p => p.trim()).filter(Boolean);
for (const para of rawParagraphs) {
if (!para || para.trim() === '' || para === '\u00a0') continue;
const fixedHtml = para.trim().startsWith('<p')
? para.trim()
: `<p>${para.trim()}</p>`;
const pNum = paragraphs.length + 1;
const pId = `${docId}-${pNum}`;
paragraphs.push({
id: pId,
[col.groupField]: docId,
code: document.code,
html: fixedHtml,
text: fixedHtml.replace(/<[^>]+>/g, ''),
number: pNum,
locale,
type: data.type ?? col.type,
});
}
}
return { document, paragraphs };
}
// ── Fetch desde Directus ─────────────────────────────────────────
async function fetchAllTranslations(col) {
const allItems = [];
let page = 1;
let hasMore = true;
const fields = [
'*',
'languages_code.code',
`${col.relationField}.*`,
`${col.relationField}.thumbnail.filename_disk`,
'mp3.filename_disk',
'video.filename_disk',
'pdf.filename_disk',
'pdf_booklet.filename_disk',
...col.extraFields,
].join(',');
const limit = CFG.limit > 0 ? Math.min(CFG.limit, CFG.pageLimit) : CFG.pageLimit;
const offset = CFG.offset || 0;
let remaining = CFG.limit > 0 ? CFG.limit : Infinity;
while (hasMore && remaining > 0) {
const currentLimit = Math.min(limit, remaining);
const pageOffset = offset + ((page - 1) * currentLimit);
const qs = `${col.directusCollection}?limit=${currentLimit}&page=${page}&offset=${pageOffset}&fields=${encodeURIComponent(fields)}`;
log(COLORS.blue(` → Directus: page ${page} (limit ${currentLimit}, offset ${pageOffset})...`));
const json = await directusFetch(qs);
const items = json.data || [];
if (items.length === 0) {
hasMore = false;
break;
}
for (const item of items) {
allItems.push(item);
remaining--;
}
page++;
if (items.length < currentLimit) hasMore = false;
}
return allItems;
}
// ── Import a Typesense ───────────────────────────────────────────
async function importBatch(collection, docs, label) {
const result = await typesenseImport(collection, docs);
const color = result.error > 0 ? COLORS.yellow : COLORS.green;
log(color(` ${label}: ${fmtNum(result.success)} ok, ${fmtNum(result.error)} fail`));
return result;
}
// ── Main ─────────────────────────────────────────────────────────
async function main() {
log('');
log(COLORS.bold(COLORS.cyan('════════════════════════════════════════')));
log(COLORS.bold(COLORS.cyan(` REINDEX ${COL.label.toUpperCase()} → TYPESENSE`)));
log(COLORS.bold(COLORS.cyan('════════════════════════════════════════')));
log('');
// ── Validación ────────────────────────────────────────
const missing = [];
if (!CFG.directusUrl) missing.push('DIRECTUS_URL');
if (!CFG.directusToken) missing.push('DIRECTUS_TOKEN');
if (!CFG.typesenseUrl) missing.push('TYPESENSE_URL');
if (!CFG.typesenseKey) missing.push('TYPESENSE_ADMIN_KEY');
if (missing.length > 0) {
log(COLORS.red(` ❌ Faltan variables: ${missing.join(', ')}`));
log(COLORS.gray(` Copia .env.example a .env y completa los valores`));
process.exit(1);
}
log(` Colección: ${COLORS.cyan(COL.label)}`);
log(` DirectUS: ${COLORS.blue(CFG.directusUrl)}`);
log(` Typesense: ${COLORS.blue(CFG.typesenseUrl)}`);
if (CFG.limit > 0) log(` Límite: ${COLORS.yellow(fmtNum(CFG.limit))} registros`);
if (CFG.offset > 0) log(` Offset: ${COLORS.yellow(fmtNum(CFG.offset))} (saltando los primeros)`);
if (CFG.dryRun) log(` ${COLORS.magenta('Modo: DRY RUN — no se escribirá nada en Typesense')}`);
log('');
// ── 1. Fetch desde Directus ───────────────────────────
log(COLORS.bold('Fase 1: Leyendo datos desde Directus...'));
let translations;
try {
translations = await fetchAllTranslations(COL);
} catch (err) {
log(COLORS.red(` ❌ Error leyendo Directus: ${err.message}`));
process.exit(1);
}
log(COLORS.green(`${fmtNum(translations.length)} ${COL.directusCollection} leídas (${elapsed()})`));
log('');
// ── 2. Transformar ────────────────────────────────────
log(COLORS.bold('Fase 2: Transformando datos...'));
const allDocuments = [];
const allParagraphs = [];
let docsWithContent = 0;
for (const item of translations) {
const { document, paragraphs } = transformItem(item, COL);
allDocuments.push(document);
allParagraphs.push(...paragraphs);
if (paragraphs.length > 0) docsWithContent++;
}
const uniqueDates = new Set(allDocuments.map(d => d.code?.split('-')[0]).filter(Boolean)).size;
log(COLORS.green(`${fmtNum(allDocuments.length)} documentos para ${COL.mainCollection}`));
log(COLORS.green(`${fmtNum(allParagraphs.length)} párrafos para ${COL.paragraphsCollection}`));
log(COLORS.gray(` ${fmtNum(docsWithContent)} con contenido, ${fmtNum(uniqueDates)} fechas únicas`));
log('');
// ── 3. Dry-run: salir ─────────────────────────────────
if (CFG.dryRun) {
log(COLORS.magenta('╔══════════════════════════════════════╗'));
log(COLORS.magenta('║ DRY RUN — nada se importó ║'));
log(COLORS.magenta('╚══════════════════════════════════════╝'));
log('');
if (allDocuments.length > 0) {
log(COLORS.gray(' Primer documento (sample):'));
log(COLORS.gray(` ${JSON.stringify(allDocuments[0], null, 2).slice(0, 600)}`));
}
log('');
return;
}
// ── 3. Importar a Typesense ───────────────────────────
log(COLORS.bold(`Fase 3: Importando a Typesense (batches de ${CFG.batchSize})...`));
log(COLORS.gray(' (upsert: si el id existe se actualiza, si no se crea)'));
let totalDocOk = 0, totalDocFail = 0;
let totalParOk = 0, totalParFail = 0;
const allErrors = [];
log(COLORS.cyan(` ── ${COL.mainCollection} ──`));
for (let i = 0; i < allDocuments.length; i += CFG.batchSize) {
const batch = allDocuments.slice(i, i + CFG.batchSize);
const label = `[${fmtNum(i + 1)}${fmtNum(Math.min(i + CFG.batchSize, allDocuments.length))}/${fmtNum(allDocuments.length)}]`;
try {
const result = await importBatch(COL.mainCollection, batch, label);
totalDocOk += result.success;
totalDocFail += result.error;
allErrors.push(...result.errors);
} catch (err) {
log(COLORS.red(` ${label} ERROR: ${err.message}`));
totalDocFail += batch.length;
}
}
log(COLORS.cyan(` ── ${COL.paragraphsCollection} ──`));
for (let i = 0; i < allParagraphs.length; i += CFG.batchSize) {
const batch = allParagraphs.slice(i, i + CFG.batchSize);
const label = `[${fmtNum(i + 1)}${fmtNum(Math.min(i + CFG.batchSize, allParagraphs.length))}/${fmtNum(allParagraphs.length)}]`;
try {
const result = await importBatch(COL.paragraphsCollection, batch, label);
totalParOk += result.success;
totalParFail += result.error;
allErrors.push(...result.errors);
} catch (err) {
log(COLORS.red(` ${label} ERROR: ${err.message}`));
totalParFail += batch.length;
}
}
log('');
// ── Reporte final ─────────────────────────────────────
const hasErrors = totalDocFail > 0 || totalParFail > 0;
const titleColor = hasErrors ? COLORS.yellow : COLORS.green;
log(titleColor(COLORS.bold('════════════════════════════════════════')));
log(titleColor(COLORS.bold(' REPORTE FINAL')));
log(titleColor(COLORS.bold('════════════════════════════════════════')));
log('');
log(` Directus:`);
log(` ${COL.directusCollection}: ${COLORS.cyan(fmtNum(translations.length))}`);
log(` fechas únicas: ${COLORS.cyan(fmtNum(uniqueDates))}`);
log(` con párrafos: ${COLORS.cyan(fmtNum(docsWithContent))}`);
log('');
log(` Typesense:`);
const docOk = COLORS.green(fmtNum(totalDocOk));
const docFail = totalDocFail > 0 ? COLORS.red(fmtNum(totalDocFail)) : COLORS.green(fmtNum(totalDocFail));
log(` ${COL.mainCollection}: ${docOk} importados, ${docFail} errores`);
const parOk = COLORS.green(fmtNum(totalParOk));
const parFail = totalParFail > 0 ? COLORS.red(fmtNum(totalParFail)) : COLORS.green(fmtNum(totalParFail));
log(` ${COL.paragraphsCollection}: ${parOk} importados, ${parFail} errores`);
log('');
log(` ${COLORS.bold('Duración:')} ${elapsed()}`);
log('');
if (allErrors.length > 0) {
log(COLORS.yellow(COLORS.bold(' ⚠️ Errores de Typesense (primeros 10):')));
for (const err of allErrors.slice(0, 10)) {
log(COLORS.red(` - ${err}`));
}
if (allErrors.length > 10) {
log(COLORS.gray(` ... y ${allErrors.length - 10} más`));
}
log('');
}
if (allDocuments.length > 0) {
log(COLORS.gray(' Muestra — primer documento importado:'));
log(COLORS.gray(` id: ${allDocuments[0].id}`));
log(COLORS.gray(` title: ${(allDocuments[0].title ?? '').slice(0, 60)}`));
log(COLORS.gray(` files: ${JSON.stringify(allDocuments[0].files)}`));
log(COLORS.gray(` thumbnail: ${allDocuments[0].thumbnail}`));
log(COLORS.gray(` párrafos: ${allParagraphs.filter(p => p[COL.groupField] === allDocuments[0].id).length}`));
log('');
}
log(COLORS.bold(hasErrors ? COLORS.yellow(' ⚠️ Proceso completado con errores') : COLORS.green(' ✅ Proceso completado exitosamente')));
log('');
}
main().catch(err => {
log(COLORS.red(`\n ❌ Error fatal: ${err.message}`));
log(err.stack);
process.exit(1);
});