407 lines
16 KiB
JavaScript
407 lines
16 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* reindex-entrelineas.mjs
|
||
*
|
||
* Reindexa entrelineas_translations desde Directus a Typesense.
|
||
* Los items se transforman según la lógica del flow de Directus
|
||
* (decodeEntities → stripHtml → cleanHtml).
|
||
*
|
||
* USO:
|
||
* node reindex-entrelineas.mjs
|
||
* node reindex-entrelineas.mjs --limit 100 --offset 200
|
||
* node reindex-entrelineas.mjs --dry-run --limit 50
|
||
*
|
||
* Config: copiar .env.example a .env y llenar variables.
|
||
*/
|
||
|
||
const __start = Date.now();
|
||
|
||
// ── Config desde .env ────────────────────────────────────────────
|
||
import { readFileSync, existsSync } from 'node:fs';
|
||
import { resolve, dirname } from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
|
||
const __dir = dirname(fileURLToPath(import.meta.url));
|
||
const dotEnv = resolve(__dir, '.env');
|
||
|
||
function loadEnv(path) {
|
||
if (!existsSync(path)) return {};
|
||
const env = {};
|
||
for (const line of readFileSync(path, 'utf-8').split('\n')) {
|
||
const trimmed = line.trim();
|
||
if (!trimmed || trimmed.startsWith('#') || !trimmed.includes('=')) continue;
|
||
const eq = trimmed.indexOf('=');
|
||
const key = trimmed.slice(0, eq).trim();
|
||
const val = trimmed.slice(eq + 1).trim().replace(/^["']|["']$/g, '');
|
||
env[key] = val;
|
||
}
|
||
return env;
|
||
}
|
||
|
||
const dotenv = loadEnv(dotEnv);
|
||
|
||
// ── CLI args ─────────────────────────────────────────────────────
|
||
const ARGS = {};
|
||
for (const arg of process.argv.slice(2)) {
|
||
if (arg === '--dry-run') ARGS.dryRun = true;
|
||
else if (arg.startsWith('--limit=')) ARGS.limit = parseInt(arg.split('=')[1], 10);
|
||
else if (arg === '--limit') ARGS.limit = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10);
|
||
else if (arg.startsWith('--offset=')) ARGS.offset = parseInt(arg.split('=')[1], 10);
|
||
else if (arg === '--offset') ARGS.offset = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10);
|
||
}
|
||
|
||
// ── CFG ──────────────────────────────────────────────────────────
|
||
const CFG = {
|
||
directusUrl: dotenv.ENTRELINEAS_DIRECTUS_URL || dotenv.DIRECTUS_URL || process.env.ENTRELINEAS_DIRECTUS_URL || process.env.DIRECTUS_URL || '',
|
||
directusToken: dotenv.ENTRELINEAS_DIRECTUS_TOKEN || dotenv.DIRECTUS_TOKEN || process.env.ENTRELINEAS_DIRECTUS_TOKEN || process.env.DIRECTUS_TOKEN || '',
|
||
typesenseUrl: dotenv.ENTRELINEAS_TYPESENSE_URL || dotenv.TYPESENSE_URL || process.env.ENTRELINEAS_TYPESENSE_URL || process.env.TYPESENSE_URL || '',
|
||
typesenseKey: dotenv.ENTRELINEAS_TYPESENSE_ADMIN_KEY || dotenv.TYPESENSE_ADMIN_KEY || process.env.ENTRELINEAS_TYPESENSE_ADMIN_KEY || process.env.TYPESENSE_ADMIN_KEY || '',
|
||
typesenseCollection: dotenv.TYPESENSE_COLLECTION || process.env.TYPESENSE_COLLECTION || 'entrelineas',
|
||
batchSize: parseInt(dotenv.BATCH_SIZE || process.env.BATCH_SIZE || '500'),
|
||
pageLimit: parseInt(dotenv.PAGE_LIMIT || process.env.PAGE_LIMIT || '200'),
|
||
limit: ARGS.limit ?? parseInt(dotenv.LIMIT || process.env.LIMIT || '0'),
|
||
offset: ARGS.offset ?? parseInt(dotenv.OFFSET || process.env.OFFSET || '0'),
|
||
dryRun: !!ARGS.dryRun,
|
||
};
|
||
|
||
// ── Helpers ──────────────────────────────────────────────────────
|
||
|
||
const COLORS = {
|
||
red: s => `\x1b[31m${s}\x1b[0m`,
|
||
green: s => `\x1b[32m${s}\x1b[0m`,
|
||
yellow: s => `\x1b[33m${s}\x1b[0m`,
|
||
blue: s => `\x1b[34m${s}\x1b[0m`,
|
||
magenta: s => `\x1b[35m${s}\x1b[0m`,
|
||
cyan: s => `\x1b[36m${s}\x1b[0m`,
|
||
gray: s => `\x1b[90m${s}\x1b[0m`,
|
||
bold: s => `\x1b[1m${s}\x1b[0m`,
|
||
};
|
||
|
||
function log(...args) { console.log(...args); }
|
||
|
||
function elapsed() {
|
||
const sec = Math.floor((Date.now() - __start) / 1000);
|
||
const m = Math.floor(sec / 60);
|
||
const s = sec % 60;
|
||
return m > 0 ? `${m}m ${s}s` : `${s}s`;
|
||
}
|
||
|
||
function fmtNum(n) {
|
||
return n.toLocaleString('en-US');
|
||
}
|
||
|
||
// ── API calls ────────────────────────────────────────────────────
|
||
|
||
async function directusFetch(path) {
|
||
const url = `${CFG.directusUrl}/items/${path}`;
|
||
const sep = path.includes('?') ? '&' : '?';
|
||
const fullUrl = `${url}${sep}access_token=${CFG.directusToken}`;
|
||
|
||
const res = await fetch(fullUrl, {
|
||
headers: { 'User-Agent': 'reindex/1.0' },
|
||
});
|
||
if (!res.ok) {
|
||
const text = await res.text().catch(() => '');
|
||
throw new Error(`Directus ${res.status} ${res.statusText}: ${text.slice(0, 200)}`);
|
||
}
|
||
return res.json();
|
||
}
|
||
|
||
async function typesenseImport(docs) {
|
||
if (docs.length === 0) return { success: 0, error: 0, errors: [] };
|
||
|
||
const body = docs.map(d => JSON.stringify(d)).join('\n');
|
||
const url = `${CFG.typesenseUrl}/collections/${CFG.typesenseCollection}/documents/import?action=upsert`;
|
||
const res = await fetch(url, {
|
||
method: 'POST',
|
||
headers: {
|
||
'X-TYPESENSE-API-KEY': CFG.typesenseKey,
|
||
'Content-Type': 'text/plain',
|
||
},
|
||
body,
|
||
});
|
||
|
||
const text = await res.text();
|
||
|
||
let success = 0, error = 0;
|
||
const errors = [];
|
||
|
||
if (!text.startsWith('{')) {
|
||
throw new Error(`Typesense import devolvió formato inesperado: ${text.slice(0, 200)}`);
|
||
}
|
||
|
||
for (const line of text.trim().split('\n')) {
|
||
if (!line.trim()) continue;
|
||
try {
|
||
const result = JSON.parse(line);
|
||
if (result.success) success++;
|
||
else { error++; errors.push(result.error || 'desconocido'); }
|
||
} catch (e) {
|
||
error++;
|
||
errors.push(`error parseando línea: ${line.slice(0, 100)}`);
|
||
}
|
||
}
|
||
|
||
return { success, error, errors };
|
||
}
|
||
|
||
// ── HTML utilities (desde el Directus flow) ──────────────────────
|
||
|
||
const NAMED = {
|
||
amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ',
|
||
aacute: 'á', eacute: 'é', iacute: 'í', oacute: 'ó', uacute: 'ú',
|
||
Aacute: 'Á', Eacute: 'É', Iacute: 'Í', Oacute: 'Ó', Uacute: 'Ú',
|
||
ntilde: 'ñ', Ntilde: 'Ñ',
|
||
uuml: 'ü', Uuml: 'Ü',
|
||
iquest: '¿', iexcl: '¡',
|
||
ldquo: '\u201C', rdquo: '\u201D', lsquo: '\u2018', rsquo: '\u2019',
|
||
laquo: '«', raquo: '»',
|
||
mdash: '\u2014', ndash: '\u2013',
|
||
hellip: '\u2026', middot: '·',
|
||
};
|
||
|
||
function decodeEntities(str) {
|
||
if (!str || typeof str !== 'string') return str;
|
||
return str.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (full, body) => {
|
||
if (body[0] === '#') {
|
||
const isHex = body[1] === 'x' || body[1] === 'X';
|
||
const code = parseInt(isHex ? body.slice(2) : body.slice(1), isHex ? 16 : 10);
|
||
if (Number.isFinite(code)) {
|
||
try { return String.fromCodePoint(code); } catch (_) { return full; }
|
||
}
|
||
return full;
|
||
}
|
||
return NAMED[body] != null ? NAMED[body] : full;
|
||
});
|
||
}
|
||
|
||
function stripHtml(str) {
|
||
if (!str || typeof str !== 'string') return str;
|
||
return str
|
||
.replace(/<(br|p|div|li|tr|td|th|h[1-6]|blockquote|pre)(\s[^>]*)?\/?>/gi, ' ')
|
||
.replace(/<[^>]+>/g, '')
|
||
.replace(/\s+/g, ' ')
|
||
.replace(/«\s+/g, '«')
|
||
.replace(/\s+»/g, '»')
|
||
.trim();
|
||
}
|
||
|
||
function cleanHtml(str) {
|
||
if (!str || typeof str !== 'string') return str;
|
||
return str
|
||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||
.replace(/\s+class=(["'])[^"']*\1/gi, '')
|
||
.replace(/\s+data-[\w-]+=(["'])[^"']*\1/gi, '')
|
||
.replace(/\bwidth\s*:\s*[\d.]+\s*(?:cm|mm|pt|px|em|rem|in|pc)\s*;?/gi, '')
|
||
.replace(/[ \t]*[\r\n]+[ \t]*/g, ' ')
|
||
.trim();
|
||
}
|
||
|
||
// ── Transformación ───────────────────────────────────────────────
|
||
|
||
function transformItem(data) {
|
||
const rel = data.entrelineas_id || {};
|
||
|
||
const rawLocale = data.languages_code;
|
||
const locale = typeof rawLocale === 'string' ? rawLocale : (rawLocale?.code ?? '');
|
||
|
||
return {
|
||
id: `${rel.id}`,
|
||
type: rel.type,
|
||
filter: rel.filter,
|
||
locale,
|
||
origin: data.description,
|
||
image: rel.image?.filename_disk ?? null,
|
||
text: stripHtml(decodeEntities(data.text)),
|
||
html: cleanHtml(data.text),
|
||
draft: rel.draft ?? false,
|
||
};
|
||
}
|
||
|
||
// ── Fetch desde Directus ─────────────────────────────────────────
|
||
|
||
async function fetchAllTranslations() {
|
||
const allItems = [];
|
||
let page = 1;
|
||
let hasMore = true;
|
||
|
||
const fields = [
|
||
'*',
|
||
'entrelineas_id.*',
|
||
'entrelineas_id.image.*',
|
||
].join(',');
|
||
|
||
const limit = CFG.limit > 0 ? Math.min(CFG.limit, CFG.pageLimit) : CFG.pageLimit;
|
||
const offset = CFG.offset || 0;
|
||
let remaining = CFG.limit > 0 ? CFG.limit : Infinity;
|
||
|
||
while (hasMore && remaining > 0) {
|
||
const currentLimit = Math.min(limit, remaining);
|
||
const pageOffset = offset + ((page - 1) * currentLimit);
|
||
const qs = `entrelineas_translations?limit=${currentLimit}&page=${page}&offset=${pageOffset}&fields=${encodeURIComponent(fields)}`;
|
||
|
||
log(COLORS.blue(` → Directus: page ${page} (limit ${currentLimit}, offset ${pageOffset})...`));
|
||
|
||
const json = await directusFetch(qs);
|
||
const items = json.data || [];
|
||
|
||
if (items.length === 0) {
|
||
hasMore = false;
|
||
break;
|
||
}
|
||
|
||
for (const item of items) {
|
||
allItems.push(item);
|
||
remaining--;
|
||
}
|
||
|
||
page++;
|
||
if (items.length < currentLimit) hasMore = false;
|
||
}
|
||
|
||
return allItems;
|
||
}
|
||
|
||
// ── Main ─────────────────────────────────────────────────────────
|
||
|
||
async function main() {
|
||
log('');
|
||
log(COLORS.bold(COLORS.cyan('════════════════════════════════════════')));
|
||
log(COLORS.bold(COLORS.cyan(' REINDEX ENTRELINEAS → TYPESENSE')));
|
||
log(COLORS.bold(COLORS.cyan('════════════════════════════════════════')));
|
||
log('');
|
||
|
||
// ── Validación ────────────────────────────────────────
|
||
const missing = [];
|
||
if (!CFG.directusUrl) missing.push('DIRECTUS_URL');
|
||
if (!CFG.directusToken) missing.push('DIRECTUS_TOKEN');
|
||
if (!CFG.typesenseUrl) missing.push('TYPESENSE_URL');
|
||
if (!CFG.typesenseKey) missing.push('TYPESENSE_ADMIN_KEY');
|
||
|
||
if (missing.length > 0) {
|
||
log(COLORS.red(` ❌ Faltan variables: ${missing.join(', ')}`));
|
||
log(COLORS.gray(` Copia .env.example a .env y completa los valores`));
|
||
process.exit(1);
|
||
}
|
||
|
||
log(` Tiposense: ${COLORS.blue(CFG.typesenseUrl)}/collections/${COLORS.cyan(CFG.typesenseCollection)}`);
|
||
log(` Directus: ${COLORS.blue(CFG.directusUrl)}`);
|
||
if (CFG.limit > 0) log(` Límite: ${COLORS.yellow(fmtNum(CFG.limit))} registros`);
|
||
if (CFG.offset > 0) log(` Offset: ${COLORS.yellow(fmtNum(CFG.offset))} (saltando los primeros)`);
|
||
if (CFG.dryRun) log(` ${COLORS.magenta('Modo: DRY RUN — no se escribirá nada en Typesense')}`);
|
||
log('');
|
||
|
||
// ── 1. Fetch desde Directus ───────────────────────────
|
||
log(COLORS.bold('Fase 1: Leyendo datos desde Directus...'));
|
||
let translations;
|
||
try {
|
||
translations = await fetchAllTranslations();
|
||
} catch (err) {
|
||
log(COLORS.red(` ❌ Error leyendo Directus: ${err.message}`));
|
||
process.exit(1);
|
||
}
|
||
|
||
log(COLORS.green(` ✅ ${fmtNum(translations.length)} entrelineas_translations leídas (${elapsed()})`));
|
||
log('');
|
||
|
||
// ── 2. Transformar ────────────────────────────────────
|
||
log(COLORS.bold('Fase 2: Transformando datos...'));
|
||
const documents = [];
|
||
|
||
for (const item of translations) {
|
||
documents.push(transformItem(item));
|
||
}
|
||
|
||
log(COLORS.green(` ✅ ${fmtNum(documents.length)} documentos para ${CFG.typesenseCollection}`));
|
||
log('');
|
||
|
||
// ── Dry-run: salir ────────────────────────────────────
|
||
if (CFG.dryRun) {
|
||
log(COLORS.magenta('╔══════════════════════════════════════╗'));
|
||
log(COLORS.magenta('║ DRY RUN — nada se importó ║'));
|
||
log(COLORS.magenta('╚══════════════════════════════════════╝'));
|
||
log('');
|
||
if (documents.length > 0) {
|
||
log(COLORS.gray(' Primer documento (sample):'));
|
||
log(COLORS.gray(` ${JSON.stringify(documents[0], null, 2).slice(0, 600)}`));
|
||
}
|
||
log('');
|
||
return;
|
||
}
|
||
|
||
// ── 3. Importar a Typesense ───────────────────────────
|
||
log(COLORS.bold(`Fase 3: Importando a Typesense (batches de ${CFG.batchSize})...`));
|
||
log(COLORS.gray(' (upsert: si el id existe se actualiza, si no se crea)'));
|
||
|
||
let totalOk = 0, totalFail = 0;
|
||
const allErrors = [];
|
||
|
||
for (let i = 0; i < documents.length; i += CFG.batchSize) {
|
||
const batch = documents.slice(i, i + CFG.batchSize);
|
||
const label = `[${fmtNum(i + 1)}–${fmtNum(Math.min(i + CFG.batchSize, documents.length))}/${fmtNum(documents.length)}]`;
|
||
try {
|
||
const result = await typesenseImport(batch);
|
||
totalOk += result.success;
|
||
totalFail += result.error;
|
||
allErrors.push(...result.errors);
|
||
const color = result.error > 0 ? COLORS.yellow : COLORS.green;
|
||
log(color(` ${label}: ${fmtNum(result.success)} ok, ${fmtNum(result.error)} fail`));
|
||
} catch (err) {
|
||
log(COLORS.red(` ${label} ERROR: ${err.message}`));
|
||
totalFail += batch.length;
|
||
}
|
||
}
|
||
|
||
log('');
|
||
|
||
// ── Reporte final ─────────────────────────────────────
|
||
const hasErrors = totalFail > 0;
|
||
const titleColor = hasErrors ? COLORS.yellow : COLORS.green;
|
||
|
||
log(titleColor(COLORS.bold('════════════════════════════════════════')));
|
||
log(titleColor(COLORS.bold(' REPORTE FINAL')));
|
||
log(titleColor(COLORS.bold('════════════════════════════════════════')));
|
||
log('');
|
||
log(` Directus:`);
|
||
log(` entrelineas_translations: ${COLORS.cyan(fmtNum(translations.length))}`);
|
||
log('');
|
||
log(` Typesense (${CFG.typesenseCollection}):`);
|
||
const ok = COLORS.green(fmtNum(totalOk));
|
||
const fail = totalFail > 0 ? COLORS.red(fmtNum(totalFail)) : COLORS.green(fmtNum(totalFail));
|
||
log(` ${ok} importados, ${fail} errores`);
|
||
log('');
|
||
log(` ${COLORS.bold('Duración:')} ${elapsed()}`);
|
||
log('');
|
||
|
||
if (allErrors.length > 0) {
|
||
log(COLORS.yellow(COLORS.bold(' ⚠️ Errores de Typesense (primeros 10):')));
|
||
for (const err of allErrors.slice(0, 10)) {
|
||
log(COLORS.red(` - ${err}`));
|
||
}
|
||
if (allErrors.length > 10) {
|
||
log(COLORS.gray(` ... y ${allErrors.length - 10} más`));
|
||
}
|
||
log('');
|
||
}
|
||
|
||
if (documents.length > 0) {
|
||
log(COLORS.gray(' Muestra — primer documento importado:'));
|
||
log(COLORS.gray(` id: ${documents[0].id}`));
|
||
log(COLORS.gray(` type: ${documents[0].type}`));
|
||
log(COLORS.gray(` filter: ${documents[0].filter}`));
|
||
log(COLORS.gray(` locale: ${documents[0].locale}`));
|
||
log(COLORS.gray(` image: ${documents[0].image ?? '(sin imagen)'}`));
|
||
log(COLORS.gray(` draft: ${documents[0].draft}`));
|
||
log('');
|
||
}
|
||
|
||
log(COLORS.bold(hasErrors ? COLORS.yellow(' ⚠️ Proceso completado con errores') : COLORS.green(' ✅ Proceso completado exitosamente')));
|
||
log('');
|
||
}
|
||
|
||
main().catch(err => {
|
||
log(COLORS.red(`\n ❌ Error fatal: ${err.message}`));
|
||
log(err.stack);
|
||
process.exit(1);
|
||
});
|