#!/usr/bin/env node /** * reindex-entrelineas.mjs * * Reindexa entrelineas_translations desde Directus a Typesense. * Los items se transforman según la lógica del flow de Directus * (decodeEntities → stripHtml → cleanHtml). * * USO: * node reindex-entrelineas.mjs * node reindex-entrelineas.mjs --limit 100 --offset 200 * node reindex-entrelineas.mjs --dry-run --limit 50 * * Config: copiar .env.example a .env y llenar variables. */ const __start = Date.now(); // ── Config desde .env ──────────────────────────────────────────── import { readFileSync, existsSync } from 'node:fs'; import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; const __dir = dirname(fileURLToPath(import.meta.url)); const dotEnv = resolve(__dir, '.env'); function loadEnv(path) { if (!existsSync(path)) return {}; const env = {}; for (const line of readFileSync(path, 'utf-8').split('\n')) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith('#') || !trimmed.includes('=')) continue; const eq = trimmed.indexOf('='); const key = trimmed.slice(0, eq).trim(); const val = trimmed.slice(eq + 1).trim().replace(/^["']|["']$/g, ''); env[key] = val; } return env; } const dotenv = loadEnv(dotEnv); // ── CLI args ───────────────────────────────────────────────────── const ARGS = {}; for (const arg of process.argv.slice(2)) { if (arg === '--dry-run') ARGS.dryRun = true; else if (arg.startsWith('--limit=')) ARGS.limit = parseInt(arg.split('=')[1], 10); else if (arg === '--limit') ARGS.limit = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10); else if (arg.startsWith('--offset=')) ARGS.offset = parseInt(arg.split('=')[1], 10); else if (arg === '--offset') ARGS.offset = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10); } // ── CFG ────────────────────────────────────────────────────────── const CFG = { directusUrl: dotenv.ENTRELINEAS_DIRECTUS_URL || dotenv.DIRECTUS_URL || process.env.ENTRELINEAS_DIRECTUS_URL || process.env.DIRECTUS_URL || '', directusToken: dotenv.ENTRELINEAS_DIRECTUS_TOKEN || dotenv.DIRECTUS_TOKEN || process.env.ENTRELINEAS_DIRECTUS_TOKEN || process.env.DIRECTUS_TOKEN || '', typesenseUrl: dotenv.ENTRELINEAS_TYPESENSE_URL || dotenv.TYPESENSE_URL || process.env.ENTRELINEAS_TYPESENSE_URL || process.env.TYPESENSE_URL || '', typesenseKey: dotenv.ENTRELINEAS_TYPESENSE_ADMIN_KEY || dotenv.TYPESENSE_ADMIN_KEY || process.env.ENTRELINEAS_TYPESENSE_ADMIN_KEY || process.env.TYPESENSE_ADMIN_KEY || '', typesenseCollection: dotenv.TYPESENSE_COLLECTION || process.env.TYPESENSE_COLLECTION || 'entrelineas', batchSize: parseInt(dotenv.BATCH_SIZE || process.env.BATCH_SIZE || '500'), pageLimit: parseInt(dotenv.PAGE_LIMIT || process.env.PAGE_LIMIT || '200'), limit: ARGS.limit ?? parseInt(dotenv.LIMIT || process.env.LIMIT || '0'), offset: ARGS.offset ?? parseInt(dotenv.OFFSET || process.env.OFFSET || '0'), dryRun: !!ARGS.dryRun, }; // ── Helpers ────────────────────────────────────────────────────── const COLORS = { red: s => `\x1b[31m${s}\x1b[0m`, green: s => `\x1b[32m${s}\x1b[0m`, yellow: s => `\x1b[33m${s}\x1b[0m`, blue: s => `\x1b[34m${s}\x1b[0m`, magenta: s => `\x1b[35m${s}\x1b[0m`, cyan: s => `\x1b[36m${s}\x1b[0m`, gray: s => `\x1b[90m${s}\x1b[0m`, bold: s => `\x1b[1m${s}\x1b[0m`, }; function log(...args) { console.log(...args); } function elapsed() { const sec = Math.floor((Date.now() - __start) / 1000); const m = Math.floor(sec / 60); const s = sec % 60; return m > 0 ? `${m}m ${s}s` : `${s}s`; } function fmtNum(n) { return n.toLocaleString('en-US'); } // ── API calls ──────────────────────────────────────────────────── async function directusFetch(path) { const url = `${CFG.directusUrl}/items/${path}`; const sep = path.includes('?') ? '&' : '?'; const fullUrl = `${url}${sep}access_token=${CFG.directusToken}`; const res = await fetch(fullUrl, { headers: { 'User-Agent': 'reindex/1.0' }, }); if (!res.ok) { const text = await res.text().catch(() => ''); throw new Error(`Directus ${res.status} ${res.statusText}: ${text.slice(0, 200)}`); } return res.json(); } async function typesenseImport(docs) { if (docs.length === 0) return { success: 0, error: 0, errors: [] }; const body = docs.map(d => JSON.stringify(d)).join('\n'); const url = `${CFG.typesenseUrl}/collections/${CFG.typesenseCollection}/documents/import?action=upsert`; const res = await fetch(url, { method: 'POST', headers: { 'X-TYPESENSE-API-KEY': CFG.typesenseKey, 'Content-Type': 'text/plain', }, body, }); const text = await res.text(); let success = 0, error = 0; const errors = []; if (!text.startsWith('{')) { throw new Error(`Typesense import devolvió formato inesperado: ${text.slice(0, 200)}`); } for (const line of text.trim().split('\n')) { if (!line.trim()) continue; try { const result = JSON.parse(line); if (result.success) success++; else { error++; errors.push(result.error || 'desconocido'); } } catch (e) { error++; errors.push(`error parseando línea: ${line.slice(0, 100)}`); } } return { success, error, errors }; } // ── HTML utilities (desde el Directus flow) ────────────────────── const NAMED = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', aacute: 'á', eacute: 'é', iacute: 'í', oacute: 'ó', uacute: 'ú', Aacute: 'Á', Eacute: 'É', Iacute: 'Í', Oacute: 'Ó', Uacute: 'Ú', ntilde: 'ñ', Ntilde: 'Ñ', uuml: 'ü', Uuml: 'Ü', iquest: '¿', iexcl: '¡', ldquo: '\u201C', rdquo: '\u201D', lsquo: '\u2018', rsquo: '\u2019', laquo: '«', raquo: '»', mdash: '\u2014', ndash: '\u2013', hellip: '\u2026', middot: '·', }; function decodeEntities(str) { if (!str || typeof str !== 'string') return str; return str.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (full, body) => { if (body[0] === '#') { const isHex = body[1] === 'x' || body[1] === 'X'; const code = parseInt(isHex ? body.slice(2) : body.slice(1), isHex ? 16 : 10); if (Number.isFinite(code)) { try { return String.fromCodePoint(code); } catch (_) { return full; } } return full; } return NAMED[body] != null ? NAMED[body] : full; }); } function stripHtml(str) { if (!str || typeof str !== 'string') return str; return str .replace(/<(br|p|div|li|tr|td|th|h[1-6]|blockquote|pre)(\s[^>]*)?\/?>/gi, ' ') .replace(/<[^>]+>/g, '') .replace(/\s+/g, ' ') .replace(/«\s+/g, '«') .replace(/\s+»/g, '»') .trim(); } function cleanHtml(str) { if (!str || typeof str !== 'string') return str; return str .replace(/]*>[\s\S]*?<\/style>/gi, '') .replace(/\s+class=(["'])[^"']*\1/gi, '') .replace(/\s+data-[\w-]+=(["'])[^"']*\1/gi, '') .replace(/\bwidth\s*:\s*[\d.]+\s*(?:cm|mm|pt|px|em|rem|in|pc)\s*;?/gi, '') .replace(/[ \t]*[\r\n]+[ \t]*/g, ' ') .trim(); } // ── Transformación ─────────────────────────────────────────────── function transformItem(data) { const rel = data.entrelineas_id || {}; const rawLocale = data.languages_code; const locale = typeof rawLocale === 'string' ? rawLocale : (rawLocale?.code ?? ''); return { id: `${rel.id}`, type: rel.type, filter: rel.filter, locale, origin: data.description, image: rel.image?.filename_disk ?? null, text: stripHtml(decodeEntities(data.text)), html: cleanHtml(data.text), draft: rel.draft ?? false, }; } // ── Fetch desde Directus ───────────────────────────────────────── async function fetchAllTranslations() { const allItems = []; let page = 1; let hasMore = true; const fields = [ '*', 'entrelineas_id.*', 'entrelineas_id.image.*', ].join(','); const limit = CFG.limit > 0 ? Math.min(CFG.limit, CFG.pageLimit) : CFG.pageLimit; const offset = CFG.offset || 0; let remaining = CFG.limit > 0 ? CFG.limit : Infinity; while (hasMore && remaining > 0) { const currentLimit = Math.min(limit, remaining); const pageOffset = offset + ((page - 1) * currentLimit); const qs = `entrelineas_translations?limit=${currentLimit}&page=${page}&offset=${pageOffset}&fields=${encodeURIComponent(fields)}`; log(COLORS.blue(` → Directus: page ${page} (limit ${currentLimit}, offset ${pageOffset})...`)); const json = await directusFetch(qs); const items = json.data || []; if (items.length === 0) { hasMore = false; break; } for (const item of items) { allItems.push(item); remaining--; } page++; if (items.length < currentLimit) hasMore = false; } return allItems; } // ── Main ───────────────────────────────────────────────────────── async function main() { log(''); log(COLORS.bold(COLORS.cyan('════════════════════════════════════════'))); log(COLORS.bold(COLORS.cyan(' REINDEX ENTRELINEAS → TYPESENSE'))); log(COLORS.bold(COLORS.cyan('════════════════════════════════════════'))); log(''); // ── Validación ──────────────────────────────────────── const missing = []; if (!CFG.directusUrl) missing.push('DIRECTUS_URL'); if (!CFG.directusToken) missing.push('DIRECTUS_TOKEN'); if (!CFG.typesenseUrl) missing.push('TYPESENSE_URL'); if (!CFG.typesenseKey) missing.push('TYPESENSE_ADMIN_KEY'); if (missing.length > 0) { log(COLORS.red(` ❌ Faltan variables: ${missing.join(', ')}`)); log(COLORS.gray(` Copia .env.example a .env y completa los valores`)); process.exit(1); } log(` Tiposense: ${COLORS.blue(CFG.typesenseUrl)}/collections/${COLORS.cyan(CFG.typesenseCollection)}`); log(` Directus: ${COLORS.blue(CFG.directusUrl)}`); if (CFG.limit > 0) log(` Límite: ${COLORS.yellow(fmtNum(CFG.limit))} registros`); if (CFG.offset > 0) log(` Offset: ${COLORS.yellow(fmtNum(CFG.offset))} (saltando los primeros)`); if (CFG.dryRun) log(` ${COLORS.magenta('Modo: DRY RUN — no se escribirá nada en Typesense')}`); log(''); // ── 1. Fetch desde Directus ─────────────────────────── log(COLORS.bold('Fase 1: Leyendo datos desde Directus...')); let translations; try { translations = await fetchAllTranslations(); } catch (err) { log(COLORS.red(` ❌ Error leyendo Directus: ${err.message}`)); process.exit(1); } log(COLORS.green(` ✅ ${fmtNum(translations.length)} entrelineas_translations leídas (${elapsed()})`)); log(''); // ── 2. Transformar ──────────────────────────────────── log(COLORS.bold('Fase 2: Transformando datos...')); const documents = []; for (const item of translations) { documents.push(transformItem(item)); } log(COLORS.green(` ✅ ${fmtNum(documents.length)} documentos para ${CFG.typesenseCollection}`)); log(''); // ── Dry-run: salir ──────────────────────────────────── if (CFG.dryRun) { log(COLORS.magenta('╔══════════════════════════════════════╗')); log(COLORS.magenta('║ DRY RUN — nada se importó ║')); log(COLORS.magenta('╚══════════════════════════════════════╝')); log(''); if (documents.length > 0) { log(COLORS.gray(' Primer documento (sample):')); log(COLORS.gray(` ${JSON.stringify(documents[0], null, 2).slice(0, 600)}`)); } log(''); return; } // ── 3. Importar a Typesense ─────────────────────────── log(COLORS.bold(`Fase 3: Importando a Typesense (batches de ${CFG.batchSize})...`)); log(COLORS.gray(' (upsert: si el id existe se actualiza, si no se crea)')); let totalOk = 0, totalFail = 0; const allErrors = []; for (let i = 0; i < documents.length; i += CFG.batchSize) { const batch = documents.slice(i, i + CFG.batchSize); const label = `[${fmtNum(i + 1)}–${fmtNum(Math.min(i + CFG.batchSize, documents.length))}/${fmtNum(documents.length)}]`; try { const result = await typesenseImport(batch); totalOk += result.success; totalFail += result.error; allErrors.push(...result.errors); const color = result.error > 0 ? COLORS.yellow : COLORS.green; log(color(` ${label}: ${fmtNum(result.success)} ok, ${fmtNum(result.error)} fail`)); } catch (err) { log(COLORS.red(` ${label} ERROR: ${err.message}`)); totalFail += batch.length; } } log(''); // ── Reporte final ───────────────────────────────────── const hasErrors = totalFail > 0; const titleColor = hasErrors ? COLORS.yellow : COLORS.green; log(titleColor(COLORS.bold('════════════════════════════════════════'))); log(titleColor(COLORS.bold(' REPORTE FINAL'))); log(titleColor(COLORS.bold('════════════════════════════════════════'))); log(''); log(` Directus:`); log(` entrelineas_translations: ${COLORS.cyan(fmtNum(translations.length))}`); log(''); log(` Typesense (${CFG.typesenseCollection}):`); const ok = COLORS.green(fmtNum(totalOk)); const fail = totalFail > 0 ? COLORS.red(fmtNum(totalFail)) : COLORS.green(fmtNum(totalFail)); log(` ${ok} importados, ${fail} errores`); log(''); log(` ${COLORS.bold('Duración:')} ${elapsed()}`); log(''); if (allErrors.length > 0) { log(COLORS.yellow(COLORS.bold(' ⚠️ Errores de Typesense (primeros 10):'))); for (const err of allErrors.slice(0, 10)) { log(COLORS.red(` - ${err}`)); } if (allErrors.length > 10) { log(COLORS.gray(` ... y ${allErrors.length - 10} más`)); } log(''); } if (documents.length > 0) { log(COLORS.gray(' Muestra — primer documento importado:')); log(COLORS.gray(` id: ${documents[0].id}`)); log(COLORS.gray(` type: ${documents[0].type}`)); log(COLORS.gray(` filter: ${documents[0].filter}`)); log(COLORS.gray(` locale: ${documents[0].locale}`)); log(COLORS.gray(` image: ${documents[0].image ?? '(sin imagen)'}`)); log(COLORS.gray(` draft: ${documents[0].draft}`)); log(''); } log(COLORS.bold(hasErrors ? COLORS.yellow(' ⚠️ Proceso completado con errores') : COLORS.green(' ✅ Proceso completado exitosamente'))); log(''); } main().catch(err => { log(COLORS.red(`\n ❌ Error fatal: ${err.message}`)); log(err.stack); process.exit(1); });