From 7c8ceb50865fe1ad5a05c05baf3c65ea2016d6c1 Mon Sep 17 00:00:00 2001 From: Esteban Date: Sun, 5 Jul 2026 12:29:02 -0500 Subject: [PATCH] first commit --- .env | 32 ++ .env.example | 34 ++ .opencode/skills/reindex-activities/SKILL.md | 37 ++ README.md | 54 ++ reindex-entrelineas.mjs | 406 +++++++++++++++ reindex-print-materials.mjs | 414 +++++++++++++++ reindex.mjs | 517 +++++++++++++++++++ 7 files changed, 1494 insertions(+) create mode 100644 .env create mode 100644 .env.example create mode 100644 .opencode/skills/reindex-activities/SKILL.md create mode 100644 README.md create mode 100644 reindex-entrelineas.mjs create mode 100644 reindex-print-materials.mjs create mode 100644 reindex.mjs diff --git a/.env b/.env new file mode 100644 index 0000000..5f3910c --- /dev/null +++ b/.env @@ -0,0 +1,32 @@ +# ── Directus ───────────────────────────────────────────── +DIRECTUS_URL=https://directus.carpa.com +DIRECTUS_TOKEN=dUILDpE5gV224XqOB5xUTzE69sk8VSOL + +# ── Typesense ──────────────────────────────────────────── +TYPESENSE_URL=https://searchts.carpa.com +TYPESENSE_ADMIN_KEY=3KrmYlcirARCxG4AZPV5bnJgQD0qtoW0 + +# ── Opciones ───────────────────────────────────────────── +BATCH_SIZE=500 # Documentos por lote de import a Typesense +PAGE_LIMIT=200 # Registros por página desde Directus +# LIMIT=0 # 0 = todos, o el número que quieras procesar +# OFFSET=0 # Para continuar desde donde quedaste +# +# # ── Directus (reindex-entrelineas.mjs) ───────────────────── +# Si no se definen, el script usa DIRECTUS_URL / DIRECTUS_TOKEN +ENTRELINEAS_DIRECTUS_URL=https://entrelineas.carpa.com +ENTRELINEAS_DIRECTUS_TOKEN=zv-CV9POA3B1_Olzb74CF-MrDRWfN98Y + +ENTRELINEAS_TYPESENSE_URL=https://searchts.carpa.com +ENTRELINEAS_TYPESENSE_ADMIN_KEY=3KrmYlcirARCxG4AZPV5bnJgQD0qtoW0 +ENTRELINEAS_TYPESENSE_COLLECTION=entrelineas + +# ── Directus (reindex-print-materials.mjs) ──────────────── +# Si no se definen, el script usa DIRECTUS_URL / DIRECTUS_TOKEN +PRINT_DIRECTUS_URL=https://directus.carpa.com +PRINT_DIRECTUS_TOKEN=dUILDpE5gV224XqOB5xUTzE69sk8VSOL + +# ── Typesense (reindex-print-materials.mjs) ─────────────── +PRINT_TYPESENSE_URL=https://searchts.carpa.com +PRINT_TYPESENSE_ADMIN_KEY=3KrmYlcirARCxG4AZPV5bnJgQD0qtoW0 +PRINT_TYPESENSE_COLLECTION=print-materials diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ea309ac --- /dev/null +++ b/.env.example @@ -0,0 +1,34 @@ +# ── Directus (reindex.mjs) ───────────────────────────────── +DIRECTUS_URL=https://directus.carpa.com +DIRECTUS_TOKEN=tu_token_de_acceso + +# ── Directus (reindex-entrelineas.mjs) ───────────────────── +# Si no se definen, el script usa DIRECTUS_URL / DIRECTUS_TOKEN +# ENTRELINEAS_DIRECTUS_URL=https://entrelineas.carpa.com +# ENTRELINEAS_DIRECTUS_TOKEN=tu_token_entrelineas + +# ── Typesense ──────────────────────────────────────────── +TYPESENSE_URL=https://searchts.carpa.com +TYPESENSE_ADMIN_KEY=admin_key_de_typesense + +# ── Typesense (reindex-entrelineas.mjs) ─────────────────── +# Si no se definen, el script usa TYPESENSE_URL / TYPESENSE_ADMIN_KEY +# ENTRELINEAS_TYPESENSE_URL=https://searchts.carpa.com +# ENTRELINEAS_TYPESENSE_ADMIN_KEY=admin_key +# ENTRELINEAS_TYPESENSE_COLLECTION=entrelineas + +# ── Directus (reindex-print-materials.mjs) ──────────────── +# Si no se definen, el script usa DIRECTUS_URL / DIRECTUS_TOKEN +# PRINT_DIRECTUS_URL=https://directus.carpa.com +# PRINT_DIRECTUS_TOKEN=tu_token + +# ── Typesense (reindex-print-materials.mjs) ─────────────── +# PRINT_TYPESENSE_URL=https://searchts.carpa.com +# PRINT_TYPESENSE_ADMIN_KEY=admin_key +# PRINT_TYPESENSE_COLLECTION=print-materials + +# ── Opciones comunes ────────────────────────────────────── +BATCH_SIZE=500 # Documentos por lote de import a Typesense +PAGE_LIMIT=200 # Registros por página desde Directus +# LIMIT=0 # 0 = todos, o el número que quieras procesar +# OFFSET=0 # Para continuar desde donde quedaste diff --git a/.opencode/skills/reindex-activities/SKILL.md b/.opencode/skills/reindex-activities/SKILL.md new file mode 100644 index 0000000..8eac867 --- /dev/null +++ b/.opencode/skills/reindex-activities/SKILL.md @@ -0,0 +1,37 @@ +--- +name: reindex-activities +description: Use when reindexing Carpa data from Directus to Typesense. Covers activities, conferences, entrelineas, and print-materials collections. Run from the reindex-activities/ directory. +--- + +# Reindex Carpa → Typesense + +Scripts para reindexar datos desde Directus a Typesense usando upsert, ubicados en `C:\Users\Esteban\Documents\Carpa Code\reindex-activities\`. + +## Scripts disponibles + +| Colección | Comando | +|-----------|---------| +| activities + paragraphs | `node reindex.mjs --collection activities` | +| conferences + paragraphs | `node reindex.mjs --collection conferences` | +| entrelineas | `node reindex-entrelineas.mjs` | +| print-materials | `node reindex-print-materials.mjs` | + +## Flags comunes + +- `--dry-run` — valida transformación sin escribir en Typesense +- `--limit N` — procesar solo N registros +- `--offset N` — saltar los primeros N registros + +## Configuración + +Copiar `.env.example` a `.env`. Cada script acepta vars específicas: + +- `reindex.mjs`: `DIRECTUS_URL`, `DIRECTUS_TOKEN`, `TYPESENSE_URL`, `TYPESENSE_ADMIN_KEY` +- `reindex-entrelineas.mjs`: fallback a `ENTRELINEAS_DIRECTUS_*` → `DIRECTUS_*` +- `reindex-print-materials.mjs`: fallback a `PRINT_DIRECTUS_*` → `DIRECTUS_*` + +## Workflow típico + +1. Validar con `--dry-run --limit 5` +2. Si la transformación se ve bien, ejecutar sin flags para index completo +3. Si hay errores, revisar los primeros 10 mostrados en el reporte diff --git a/README.md b/README.md new file mode 100644 index 0000000..d33f84d --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# Reindex Carpa → Typesense + +Scripts para reindexar colecciones desde Directus a Typesense (upsert). + +## Requisitos + +- Node.js 18+ +- `.env` configurado (ver `.env.example`) + +## Scripts + +| Comando | Directus | Typesense | +|---------|----------|-----------| +| `node reindex.mjs --collection activities` | `activities_translations` | `activities` + `activities_paragraphs` | +| `node reindex.mjs --collection conferences` | `conferences_translations` | `conferences` + `conferences_paragraphs` | +| `node reindex-entrelineas.mjs` | `entrelineas_translations` | `entrelineas` | +| `node reindex-print-materials.mjs` | `print_materials` | `print-materials` | + +## Uso + +```bash +# Validar transformación sin escribir en Typesense +node reindex-entrelineas.mjs --dry-run --limit 5 +node reindex-print-materials.mjs --dry-run --limit 5 + +# Index completo +node reindex-entrelineas.mjs +node reindex-print-materials.mjs + +# Con límite y offset para lotes parciales +node reindex-entrelineas.mjs --limit 100 --offset 200 +``` + +## Opciones comunes + +| Flag | Descripción | +|------|-------------| +| `--limit N` | Procesar solo N registros | +| `--offset N` | Saltar los primeros N registros | +| `--dry-run` | Solo lectura + transformación, no importa a Typesense | + +## Variables de entorno + +Ver `.env.example`. Cada script soporta vars específicas con fallback a las genéricas: + +| Script | Vars específicas | Fallback | +|--------|-----------------|----------| +| `reindex-entrelineas.mjs` | `ENTRELINEAS_DIRECTUS_*`, `ENTRELINEAS_TYPESENSE_*` | `DIRECTUS_*`, `TYPESENSE_*` | +| `reindex-print-materials.mjs` | `PRINT_DIRECTUS_*`, `PRINT_TYPESENSE_*` | `DIRECTUS_*`, `TYPESENSE_*` | + +## Notas + +- Todos los imports usan `?action=upsert` — no elimina documentos existentes. +- Si el Directus de entrelineas/print-materials es distinto al de activities, definir las vars específicas en `.env`. diff --git a/reindex-entrelineas.mjs b/reindex-entrelineas.mjs new file mode 100644 index 0000000..daa428c --- /dev/null +++ b/reindex-entrelineas.mjs @@ -0,0 +1,406 @@ +#!/usr/bin/env node +/** + * reindex-entrelineas.mjs + * + * Reindexa entrelineas_translations desde Directus a Typesense. + * Los items se transforman según la lógica del flow de Directus + * (decodeEntities → stripHtml → cleanHtml). + * + * USO: + * node reindex-entrelineas.mjs + * node reindex-entrelineas.mjs --limit 100 --offset 200 + * node reindex-entrelineas.mjs --dry-run --limit 50 + * + * Config: copiar .env.example a .env y llenar variables. + */ + +const __start = Date.now(); + +// ── Config desde .env ──────────────────────────────────────────── +import { readFileSync, existsSync } from 'node:fs'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dir = dirname(fileURLToPath(import.meta.url)); +const dotEnv = resolve(__dir, '.env'); + +function loadEnv(path) { + if (!existsSync(path)) return {}; + const env = {}; + for (const line of readFileSync(path, 'utf-8').split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#') || !trimmed.includes('=')) continue; + const eq = trimmed.indexOf('='); + const key = trimmed.slice(0, eq).trim(); + const val = trimmed.slice(eq + 1).trim().replace(/^["']|["']$/g, ''); + env[key] = val; + } + return env; +} + +const dotenv = loadEnv(dotEnv); + +// ── CLI args ───────────────────────────────────────────────────── +const ARGS = {}; +for (const arg of process.argv.slice(2)) { + if (arg === '--dry-run') ARGS.dryRun = true; + else if (arg.startsWith('--limit=')) ARGS.limit = parseInt(arg.split('=')[1], 10); + else if (arg === '--limit') ARGS.limit = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10); + else if (arg.startsWith('--offset=')) ARGS.offset = parseInt(arg.split('=')[1], 10); + else if (arg === '--offset') ARGS.offset = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10); +} + +// ── CFG ────────────────────────────────────────────────────────── +const CFG = { + directusUrl: dotenv.ENTRELINEAS_DIRECTUS_URL || dotenv.DIRECTUS_URL || process.env.ENTRELINEAS_DIRECTUS_URL || process.env.DIRECTUS_URL || '', + directusToken: dotenv.ENTRELINEAS_DIRECTUS_TOKEN || dotenv.DIRECTUS_TOKEN || process.env.ENTRELINEAS_DIRECTUS_TOKEN || process.env.DIRECTUS_TOKEN || '', + typesenseUrl: dotenv.ENTRELINEAS_TYPESENSE_URL || dotenv.TYPESENSE_URL || process.env.ENTRELINEAS_TYPESENSE_URL || process.env.TYPESENSE_URL || '', + typesenseKey: dotenv.ENTRELINEAS_TYPESENSE_ADMIN_KEY || dotenv.TYPESENSE_ADMIN_KEY || process.env.ENTRELINEAS_TYPESENSE_ADMIN_KEY || process.env.TYPESENSE_ADMIN_KEY || '', + typesenseCollection: dotenv.TYPESENSE_COLLECTION || process.env.TYPESENSE_COLLECTION || 'entrelineas', + batchSize: parseInt(dotenv.BATCH_SIZE || process.env.BATCH_SIZE || '500'), + pageLimit: parseInt(dotenv.PAGE_LIMIT || process.env.PAGE_LIMIT || '200'), + limit: ARGS.limit ?? parseInt(dotenv.LIMIT || process.env.LIMIT || '0'), + offset: ARGS.offset ?? parseInt(dotenv.OFFSET || process.env.OFFSET || '0'), + dryRun: !!ARGS.dryRun, +}; + +// ── Helpers ────────────────────────────────────────────────────── + +const COLORS = { + red: s => `\x1b[31m${s}\x1b[0m`, + green: s => `\x1b[32m${s}\x1b[0m`, + yellow: s => `\x1b[33m${s}\x1b[0m`, + blue: s => `\x1b[34m${s}\x1b[0m`, + magenta: s => `\x1b[35m${s}\x1b[0m`, + cyan: s => `\x1b[36m${s}\x1b[0m`, + gray: s => `\x1b[90m${s}\x1b[0m`, + bold: s => `\x1b[1m${s}\x1b[0m`, +}; + +function log(...args) { console.log(...args); } + +function elapsed() { + const sec = Math.floor((Date.now() - __start) / 1000); + const m = Math.floor(sec / 60); + const s = sec % 60; + return m > 0 ? `${m}m ${s}s` : `${s}s`; +} + +function fmtNum(n) { + return n.toLocaleString('en-US'); +} + +// ── API calls ──────────────────────────────────────────────────── + +async function directusFetch(path) { + const url = `${CFG.directusUrl}/items/${path}`; + const sep = path.includes('?') ? '&' : '?'; + const fullUrl = `${url}${sep}access_token=${CFG.directusToken}`; + + const res = await fetch(fullUrl, { + headers: { 'User-Agent': 'reindex/1.0' }, + }); + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new Error(`Directus ${res.status} ${res.statusText}: ${text.slice(0, 200)}`); + } + return res.json(); +} + +async function typesenseImport(docs) { + if (docs.length === 0) return { success: 0, error: 0, errors: [] }; + + const body = docs.map(d => JSON.stringify(d)).join('\n'); + const url = `${CFG.typesenseUrl}/collections/${CFG.typesenseCollection}/documents/import?action=upsert`; + const res = await fetch(url, { + method: 'POST', + headers: { + 'X-TYPESENSE-API-KEY': CFG.typesenseKey, + 'Content-Type': 'text/plain', + }, + body, + }); + + const text = await res.text(); + + let success = 0, error = 0; + const errors = []; + + if (!text.startsWith('{')) { + throw new Error(`Typesense import devolvió formato inesperado: ${text.slice(0, 200)}`); + } + + for (const line of text.trim().split('\n')) { + if (!line.trim()) continue; + try { + const result = JSON.parse(line); + if (result.success) success++; + else { error++; errors.push(result.error || 'desconocido'); } + } catch (e) { + error++; + errors.push(`error parseando línea: ${line.slice(0, 100)}`); + } + } + + return { success, error, errors }; +} + +// ── HTML utilities (desde el Directus flow) ────────────────────── + +const NAMED = { + amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', + aacute: 'á', eacute: 'é', iacute: 'í', oacute: 'ó', uacute: 'ú', + Aacute: 'Á', Eacute: 'É', Iacute: 'Í', Oacute: 'Ó', Uacute: 'Ú', + ntilde: 'ñ', Ntilde: 'Ñ', + uuml: 'ü', Uuml: 'Ü', + iquest: '¿', iexcl: '¡', + ldquo: '\u201C', rdquo: '\u201D', lsquo: '\u2018', rsquo: '\u2019', + laquo: '«', raquo: '»', + mdash: '\u2014', ndash: '\u2013', + hellip: '\u2026', middot: '·', +}; + +function decodeEntities(str) { + if (!str || typeof str !== 'string') return str; + return str.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (full, body) => { + if (body[0] === '#') { + const isHex = body[1] === 'x' || body[1] === 'X'; + const code = parseInt(isHex ? body.slice(2) : body.slice(1), isHex ? 16 : 10); + if (Number.isFinite(code)) { + try { return String.fromCodePoint(code); } catch (_) { return full; } + } + return full; + } + return NAMED[body] != null ? NAMED[body] : full; + }); +} + +function stripHtml(str) { + if (!str || typeof str !== 'string') return str; + return str + .replace(/<(br|p|div|li|tr|td|th|h[1-6]|blockquote|pre)(\s[^>]*)?\/?>/gi, ' ') + .replace(/<[^>]+>/g, '') + .replace(/\s+/g, ' ') + .replace(/«\s+/g, '«') + .replace(/\s+»/g, '»') + .trim(); +} + +function cleanHtml(str) { + if (!str || typeof str !== 'string') return str; + return str + .replace(/]*>[\s\S]*?<\/style>/gi, '') + .replace(/\s+class=(["'])[^"']*\1/gi, '') + .replace(/\s+data-[\w-]+=(["'])[^"']*\1/gi, '') + .replace(/\bwidth\s*:\s*[\d.]+\s*(?:cm|mm|pt|px|em|rem|in|pc)\s*;?/gi, '') + .replace(/[ \t]*[\r\n]+[ \t]*/g, ' ') + .trim(); +} + +// ── Transformación ─────────────────────────────────────────────── + +function transformItem(data) { + const rel = data.entrelineas_id || {}; + + const rawLocale = data.languages_code; + const locale = typeof rawLocale === 'string' ? rawLocale : (rawLocale?.code ?? ''); + + return { + id: `${rel.id}`, + type: rel.type, + filter: rel.filter, + locale, + origin: data.description, + image: rel.image?.filename_disk ?? null, + text: stripHtml(decodeEntities(data.text)), + html: cleanHtml(data.text), + draft: rel.draft ?? false, + }; +} + +// ── Fetch desde Directus ───────────────────────────────────────── + +async function fetchAllTranslations() { + const allItems = []; + let page = 1; + let hasMore = true; + + const fields = [ + '*', + 'entrelineas_id.*', + 'entrelineas_id.image.*', + ].join(','); + + const limit = CFG.limit > 0 ? Math.min(CFG.limit, CFG.pageLimit) : CFG.pageLimit; + const offset = CFG.offset || 0; + let remaining = CFG.limit > 0 ? CFG.limit : Infinity; + + while (hasMore && remaining > 0) { + const currentLimit = Math.min(limit, remaining); + const pageOffset = offset + ((page - 1) * currentLimit); + const qs = `entrelineas_translations?limit=${currentLimit}&page=${page}&offset=${pageOffset}&fields=${encodeURIComponent(fields)}`; + + log(COLORS.blue(` → Directus: page ${page} (limit ${currentLimit}, offset ${pageOffset})...`)); + + const json = await directusFetch(qs); + const items = json.data || []; + + if (items.length === 0) { + hasMore = false; + break; + } + + for (const item of items) { + allItems.push(item); + remaining--; + } + + page++; + if (items.length < currentLimit) hasMore = false; + } + + return allItems; +} + +// ── Main ───────────────────────────────────────────────────────── + +async function main() { + log(''); + log(COLORS.bold(COLORS.cyan('════════════════════════════════════════'))); + log(COLORS.bold(COLORS.cyan(' REINDEX ENTRELINEAS → TYPESENSE'))); + log(COLORS.bold(COLORS.cyan('════════════════════════════════════════'))); + log(''); + + // ── Validación ──────────────────────────────────────── + const missing = []; + if (!CFG.directusUrl) missing.push('DIRECTUS_URL'); + if (!CFG.directusToken) missing.push('DIRECTUS_TOKEN'); + if (!CFG.typesenseUrl) missing.push('TYPESENSE_URL'); + if (!CFG.typesenseKey) missing.push('TYPESENSE_ADMIN_KEY'); + + if (missing.length > 0) { + log(COLORS.red(` ❌ Faltan variables: ${missing.join(', ')}`)); + log(COLORS.gray(` Copia .env.example a .env y completa los valores`)); + process.exit(1); + } + + log(` Tiposense: ${COLORS.blue(CFG.typesenseUrl)}/collections/${COLORS.cyan(CFG.typesenseCollection)}`); + log(` Directus: ${COLORS.blue(CFG.directusUrl)}`); + if (CFG.limit > 0) log(` Límite: ${COLORS.yellow(fmtNum(CFG.limit))} registros`); + if (CFG.offset > 0) log(` Offset: ${COLORS.yellow(fmtNum(CFG.offset))} (saltando los primeros)`); + if (CFG.dryRun) log(` ${COLORS.magenta('Modo: DRY RUN — no se escribirá nada en Typesense')}`); + log(''); + + // ── 1. Fetch desde Directus ─────────────────────────── + log(COLORS.bold('Fase 1: Leyendo datos desde Directus...')); + let translations; + try { + translations = await fetchAllTranslations(); + } catch (err) { + log(COLORS.red(` ❌ Error leyendo Directus: ${err.message}`)); + process.exit(1); + } + + log(COLORS.green(` ✅ ${fmtNum(translations.length)} entrelineas_translations leídas (${elapsed()})`)); + log(''); + + // ── 2. Transformar ──────────────────────────────────── + log(COLORS.bold('Fase 2: Transformando datos...')); + const documents = []; + + for (const item of translations) { + documents.push(transformItem(item)); + } + + log(COLORS.green(` ✅ ${fmtNum(documents.length)} documentos para ${CFG.typesenseCollection}`)); + log(''); + + // ── Dry-run: salir ──────────────────────────────────── + if (CFG.dryRun) { + log(COLORS.magenta('╔══════════════════════════════════════╗')); + log(COLORS.magenta('║ DRY RUN — nada se importó ║')); + log(COLORS.magenta('╚══════════════════════════════════════╝')); + log(''); + if (documents.length > 0) { + log(COLORS.gray(' Primer documento (sample):')); + log(COLORS.gray(` ${JSON.stringify(documents[0], null, 2).slice(0, 600)}`)); + } + log(''); + return; + } + + // ── 3. Importar a Typesense ─────────────────────────── + log(COLORS.bold(`Fase 3: Importando a Typesense (batches de ${CFG.batchSize})...`)); + log(COLORS.gray(' (upsert: si el id existe se actualiza, si no se crea)')); + + let totalOk = 0, totalFail = 0; + const allErrors = []; + + for (let i = 0; i < documents.length; i += CFG.batchSize) { + const batch = documents.slice(i, i + CFG.batchSize); + const label = `[${fmtNum(i + 1)}–${fmtNum(Math.min(i + CFG.batchSize, documents.length))}/${fmtNum(documents.length)}]`; + try { + const result = await typesenseImport(batch); + totalOk += result.success; + totalFail += result.error; + allErrors.push(...result.errors); + const color = result.error > 0 ? COLORS.yellow : COLORS.green; + log(color(` ${label}: ${fmtNum(result.success)} ok, ${fmtNum(result.error)} fail`)); + } catch (err) { + log(COLORS.red(` ${label} ERROR: ${err.message}`)); + totalFail += batch.length; + } + } + + log(''); + + // ── Reporte final ───────────────────────────────────── + const hasErrors = totalFail > 0; + const titleColor = hasErrors ? COLORS.yellow : COLORS.green; + + log(titleColor(COLORS.bold('════════════════════════════════════════'))); + log(titleColor(COLORS.bold(' REPORTE FINAL'))); + log(titleColor(COLORS.bold('════════════════════════════════════════'))); + log(''); + log(` Directus:`); + log(` entrelineas_translations: ${COLORS.cyan(fmtNum(translations.length))}`); + log(''); + log(` Typesense (${CFG.typesenseCollection}):`); + const ok = COLORS.green(fmtNum(totalOk)); + const fail = totalFail > 0 ? COLORS.red(fmtNum(totalFail)) : COLORS.green(fmtNum(totalFail)); + log(` ${ok} importados, ${fail} errores`); + log(''); + log(` ${COLORS.bold('Duración:')} ${elapsed()}`); + log(''); + + if (allErrors.length > 0) { + log(COLORS.yellow(COLORS.bold(' ⚠️ Errores de Typesense (primeros 10):'))); + for (const err of allErrors.slice(0, 10)) { + log(COLORS.red(` - ${err}`)); + } + if (allErrors.length > 10) { + log(COLORS.gray(` ... y ${allErrors.length - 10} más`)); + } + log(''); + } + + if (documents.length > 0) { + log(COLORS.gray(' Muestra — primer documento importado:')); + log(COLORS.gray(` id: ${documents[0].id}`)); + log(COLORS.gray(` type: ${documents[0].type}`)); + log(COLORS.gray(` filter: ${documents[0].filter}`)); + log(COLORS.gray(` locale: ${documents[0].locale}`)); + log(COLORS.gray(` image: ${documents[0].image ?? '(sin imagen)'}`)); + log(COLORS.gray(` draft: ${documents[0].draft}`)); + log(''); + } + + log(COLORS.bold(hasErrors ? COLORS.yellow(' ⚠️ Proceso completado con errores') : COLORS.green(' ✅ Proceso completado exitosamente'))); + log(''); +} + +main().catch(err => { + log(COLORS.red(`\n ❌ Error fatal: ${err.message}`)); + log(err.stack); + process.exit(1); +}); diff --git a/reindex-print-materials.mjs b/reindex-print-materials.mjs new file mode 100644 index 0000000..2305c35 --- /dev/null +++ b/reindex-print-materials.mjs @@ -0,0 +1,414 @@ +#!/usr/bin/env node +/** + * reindex-print-materials.mjs + * + * Reindexa print_materials desde Directus a Typesense. + * + * Los items de Directus contienen traducciones anidadas (translations[]), + * cada una con sus propios archivos (files[]) e imágenes (files_images[]). + * + * USO: + * node reindex-print-materials.mjs + * node reindex-print-materials.mjs --limit 10 --offset 0 + * node reindex-print-materials.mjs --dry-run --limit 5 + * + * Config: copiar .env.example a .env y llenar variables. + */ + +const __start = Date.now(); + +// ── Config desde .env ──────────────────────────────────────────── +import { readFileSync, existsSync } from 'node:fs'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dir = dirname(fileURLToPath(import.meta.url)); +const dotEnv = resolve(__dir, '.env'); + +function loadEnv(path) { + if (!existsSync(path)) return {}; + const env = {}; + for (const line of readFileSync(path, 'utf-8').split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#') || !trimmed.includes('=')) continue; + const eq = trimmed.indexOf('='); + const key = trimmed.slice(0, eq).trim(); + const val = trimmed.slice(eq + 1).trim().replace(/^["']|["']$/g, ''); + env[key] = val; + } + return env; +} + +const dotenv = loadEnv(dotEnv); + +// ── CLI args ───────────────────────────────────────────────────── +const ARGS = {}; +for (const arg of process.argv.slice(2)) { + if (arg === '--dry-run') ARGS.dryRun = true; + else if (arg.startsWith('--limit=')) ARGS.limit = parseInt(arg.split('=')[1], 10); + else if (arg === '--limit') ARGS.limit = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10); + else if (arg.startsWith('--offset=')) ARGS.offset = parseInt(arg.split('=')[1], 10); + else if (arg === '--offset') ARGS.offset = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10); +} + +// ── CFG ────────────────────────────────────────────────────────── +const CFG = { + directusUrl: dotenv.PRINT_DIRECTUS_URL || dotenv.DIRECTUS_URL || process.env.PRINT_DIRECTUS_URL || process.env.DIRECTUS_URL || '', + directusToken: dotenv.PRINT_DIRECTUS_TOKEN || dotenv.DIRECTUS_TOKEN || process.env.PRINT_DIRECTUS_TOKEN || process.env.DIRECTUS_TOKEN || '', + typesenseUrl: dotenv.PRINT_TYPESENSE_URL || dotenv.TYPESENSE_URL || process.env.PRINT_TYPESENSE_URL || process.env.TYPESENSE_URL || '', + typesenseKey: dotenv.PRINT_TYPESENSE_ADMIN_KEY || dotenv.TYPESENSE_ADMIN_KEY || process.env.PRINT_TYPESENSE_ADMIN_KEY || process.env.TYPESENSE_ADMIN_KEY || '', + typesenseCollection: dotenv.PRINT_TYPESENSE_COLLECTION || dotenv.TYPESENSE_COLLECTION || process.env.PRINT_TYPESENSE_COLLECTION || process.env.TYPESENSE_COLLECTION || 'print-materials', + batchSize: parseInt(dotenv.BATCH_SIZE || process.env.BATCH_SIZE || '500'), + pageLimit: parseInt(dotenv.PAGE_LIMIT || process.env.PAGE_LIMIT || '200'), + limit: ARGS.limit ?? parseInt(dotenv.LIMIT || process.env.LIMIT || '0'), + offset: ARGS.offset ?? parseInt(dotenv.OFFSET || process.env.OFFSET || '0'), + dryRun: !!ARGS.dryRun, +}; + +// ── Helpers ────────────────────────────────────────────────────── + +const COLORS = { + red: s => `\x1b[31m${s}\x1b[0m`, + green: s => `\x1b[32m${s}\x1b[0m`, + yellow: s => `\x1b[33m${s}\x1b[0m`, + blue: s => `\x1b[34m${s}\x1b[0m`, + magenta: s => `\x1b[35m${s}\x1b[0m`, + cyan: s => `\x1b[36m${s}\x1b[0m`, + gray: s => `\x1b[90m${s}\x1b[0m`, + bold: s => `\x1b[1m${s}\x1b[0m`, +}; + +function log(...args) { console.log(...args); } + +function elapsed() { + const sec = Math.floor((Date.now() - __start) / 1000); + const m = Math.floor(sec / 60); + const s = sec % 60; + return m > 0 ? `${m}m ${s}s` : `${s}s`; +} + +function fmtNum(n) { + return n.toLocaleString('en-US'); +} + +// ── API calls ──────────────────────────────────────────────────── + +async function directusFetch(path) { + const url = `${CFG.directusUrl}/items/${path}`; + const sep = path.includes('?') ? '&' : '?'; + const fullUrl = `${url}${sep}access_token=${CFG.directusToken}`; + + const res = await fetch(fullUrl, { + headers: { 'User-Agent': 'reindex/1.0' }, + }); + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new Error(`Directus ${res.status} ${res.statusText}: ${text.slice(0, 200)}`); + } + return res.json(); +} + +async function typesenseImport(docs) { + if (docs.length === 0) return { success: 0, error: 0, errors: [] }; + + const body = docs.map(d => JSON.stringify(d)).join('\n'); + const url = `${CFG.typesenseUrl}/collections/${CFG.typesenseCollection}/documents/import?action=upsert`; + const res = await fetch(url, { + method: 'POST', + headers: { + 'X-TYPESENSE-API-KEY': CFG.typesenseKey, + 'Content-Type': 'text/plain', + }, + body, + }); + + const text = await res.text(); + + let success = 0, error = 0; + const errors = []; + + if (!text.startsWith('{')) { + throw new Error(`Typesense import devolvió formato inesperado: ${text.slice(0, 200)}`); + } + + for (const line of text.trim().split('\n')) { + if (!line.trim()) continue; + try { + const result = JSON.parse(line); + if (result.success) success++; + else { error++; errors.push(result.error || 'desconocido'); } + } catch (e) { + error++; + errors.push(`error parseando línea: ${line.slice(0, 100)}`); + } + } + + return { success, error, errors }; +} + +// ── Helpers de transformación ──────────────────────────────────── + +function toUnixTimestamp(dateStr) { + if (!dateStr) return undefined; + const ts = Math.floor(new Date(dateStr).getTime() / 1000); + return isNaN(ts) ? undefined : ts; +} + +function getThumbnail(material) { + return material.thumbnail?.filename_disk ?? null; +} + +// ── Transformación ─────────────────────────────────────────────── +// Replica la lógica del n8n (formateo a Typesense) + +function transformMaterial(material) { + const translations = material.translations || []; + const docs = []; + + for (const t of translations) { + if (!t.title) continue; + + const doc = { + id: String(material.id) + '_' + t.languages_code, + material_id: material.id, + title: t.title, + thumbnail: getThumbnail(material), + slug: t.slug || String(material.id), + locale: t.languages_code, + type: material.type, + private: !!t.private, + images: (t.files_images || []).map(item => ({ + file: item.directus_files_id?.filename_disk || '', + downloadname: '', + })).filter(img => img.file), + files: (t.files || []).map(item => ({ + filename: item.file?.filename_disk || '', + downloadname: item.name || '', + type: item.tipo || '', + })).filter(f => f.filename), + related: (material.material_relacionado || []).map(item => { + const val = typeof item === 'object' ? (item.related_print_materials_id ?? item) : item; + return parseInt(val, 10); + }).filter(n => !isNaN(n)), + }; + + const printCategory = (material.print_category || []) + .map(item => String(typeof item === 'object' ? (item.print_categories_id ?? item.id ?? item) : item)) + .join(',') || undefined; + if (printCategory) doc['print-category'] = printCategory; + + if (t.slug_personalizado) doc['custom-slug'] = t.slug_personalizado; + if (material.subtype) doc.subtype = material.subtype; + if (material.number != null) doc.number = material.number; + if (t.notas) doc.notes = t.notas; + + const ts = toUnixTimestamp(material.date); + if (ts !== undefined) doc.date = ts; + + docs.push(doc); + } + + return docs; +} + +// ── Fetch desde Directus ───────────────────────────────────────── + +async function fetchAllMaterials() { + const allItems = []; + let page = 1; + let hasMore = true; + + const fields = [ + '*', + 'thumbnail.filename_disk', + 'translations.*', + 'translations.files.*', + 'translations.files.file.filename_disk', + 'translations.files_images.*', + 'translations.files_images.directus_files_id.filename_disk', + 'material_relacionado.related_print_materials_id', + 'print_category.print_categories_id', + ]; + + const fieldsParam = fields.map(f => `fields[]=${encodeURIComponent(f)}`).join('&'); + + const limit = CFG.limit > 0 ? Math.min(CFG.limit, CFG.pageLimit) : CFG.pageLimit; + const offset = CFG.offset || 0; + let remaining = CFG.limit > 0 ? CFG.limit : Infinity; + + while (hasMore && remaining > 0) { + const currentLimit = Math.min(limit, remaining); + const pageOffset = offset + ((page - 1) * currentLimit); + const qs = `print_materials?limit=${currentLimit}&page=${page}&offset=${pageOffset}&${fieldsParam}`; + + log(COLORS.blue(` → Directus: page ${page} (limit ${currentLimit}, offset ${pageOffset})...`)); + + const json = await directusFetch(qs); + const items = json.data || []; + + if (items.length === 0) { + hasMore = false; + break; + } + + for (const item of items) { + allItems.push(item); + remaining--; + } + + page++; + if (items.length < currentLimit) hasMore = false; + } + + return allItems; +} + +// ── Main ───────────────────────────────────────────────────────── + +async function main() { + log(''); + log(COLORS.bold(COLORS.cyan('════════════════════════════════════════'))); + log(COLORS.bold(COLORS.cyan(' REINDEX PRINT-MATERIALS → TYPESENSE'))); + log(COLORS.bold(COLORS.cyan('════════════════════════════════════════'))); + log(''); + + // ── Validación ──────────────────────────────────────── + const missing = []; + if (!CFG.directusUrl) missing.push('DIRECTUS_URL'); + if (!CFG.directusToken) missing.push('DIRECTUS_TOKEN'); + if (!CFG.typesenseUrl) missing.push('TYPESENSE_URL'); + if (!CFG.typesenseKey) missing.push('TYPESENSE_ADMIN_KEY'); + + if (missing.length > 0) { + log(COLORS.red(` ❌ Faltan variables: ${missing.join(', ')}`)); + log(COLORS.gray(` Copia .env.example a .env y completa los valores`)); + process.exit(1); + } + + log(` Tiposense: ${COLORS.blue(CFG.typesenseUrl)}/collections/${COLORS.cyan(CFG.typesenseCollection)}`); + log(` Directus: ${COLORS.blue(CFG.directusUrl)}`); + if (CFG.limit > 0) log(` Límite: ${COLORS.yellow(fmtNum(CFG.limit))} registros`); + if (CFG.offset > 0) log(` Offset: ${COLORS.yellow(fmtNum(CFG.offset))} (saltando los primeros)`); + if (CFG.dryRun) log(` ${COLORS.magenta('Modo: DRY RUN — no se escribirá nada en Typesense')}`); + log(''); + + // ── 1. Fetch desde Directus ─────────────────────────── + log(COLORS.bold('Fase 1: Leyendo datos desde Directus...')); + let materials; + try { + materials = await fetchAllMaterials(); + } catch (err) { + log(COLORS.red(` ❌ Error leyendo Directus: ${err.message}`)); + process.exit(1); + } + + log(COLORS.green(` ✅ ${fmtNum(materials.length)} print_materials leídos (${elapsed()})`)); + log(''); + + // ── 2. Transformar ──────────────────────────────────── + log(COLORS.bold('Fase 2: Transformando datos...')); + const documents = []; + let materialsWithContent = 0; + + for (const material of materials) { + const docs = transformMaterial(material); + documents.push(...docs); + if (docs.length > 0) materialsWithContent++; + } + + log(COLORS.green(` ✅ ${fmtNum(documents.length)} documentos generados desde ${fmtNum(materialsWithContent)} materiales`)); + log(''); + + // ── Dry-run: salir ──────────────────────────────────── + if (CFG.dryRun) { + log(COLORS.magenta('╔══════════════════════════════════════╗')); + log(COLORS.magenta('║ DRY RUN — nada se importó ║')); + log(COLORS.magenta('╚══════════════════════════════════════╝')); + log(''); + if (documents.length > 0) { + log(COLORS.gray(' Primer documento (sample):')); + log(COLORS.gray(` ${JSON.stringify(documents[0], null, 2).slice(0, 800)}`)); + } + log(''); + return; + } + + // ── 3. Importar a Typesense ─────────────────────────── + log(COLORS.bold(`Fase 3: Importando a Typesense (batches de ${CFG.batchSize})...`)); + log(COLORS.gray(' (upsert: si el id existe se actualiza, si no se crea)')); + + let totalOk = 0, totalFail = 0; + const allErrors = []; + + for (let i = 0; i < documents.length; i += CFG.batchSize) { + const batch = documents.slice(i, i + CFG.batchSize); + const label = `[${fmtNum(i + 1)}–${fmtNum(Math.min(i + CFG.batchSize, documents.length))}/${fmtNum(documents.length)}]`; + try { + const result = await typesenseImport(batch); + totalOk += result.success; + totalFail += result.error; + allErrors.push(...result.errors); + const color = result.error > 0 ? COLORS.yellow : COLORS.green; + log(color(` ${label}: ${fmtNum(result.success)} ok, ${fmtNum(result.error)} fail`)); + } catch (err) { + log(COLORS.red(` ${label} ERROR: ${err.message}`)); + totalFail += batch.length; + } + } + + log(''); + + // ── Reporte final ───────────────────────────────────── + const hasErrors = totalFail > 0; + const titleColor = hasErrors ? COLORS.yellow : COLORS.green; + + log(titleColor(COLORS.bold('════════════════════════════════════════'))); + log(titleColor(COLORS.bold(' REPORTE FINAL'))); + log(titleColor(COLORS.bold('════════════════════════════════════════'))); + log(''); + log(` Directus:`); + log(` print_materials: ${COLORS.cyan(fmtNum(materials.length))}`); + log(` con traducciones: ${COLORS.cyan(fmtNum(materialsWithContent))}`); + log(''); + log(` Typesense (${CFG.typesenseCollection}):`); + const ok = COLORS.green(fmtNum(totalOk)); + const fail = totalFail > 0 ? COLORS.red(fmtNum(totalFail)) : COLORS.green(fmtNum(totalFail)); + log(` ${ok} importados, ${fail} errores`); + log(''); + log(` ${COLORS.bold('Duración:')} ${elapsed()}`); + log(''); + + if (allErrors.length > 0) { + log(COLORS.yellow(COLORS.bold(' ⚠️ Errores de Typesense (primeros 10):'))); + for (const err of allErrors.slice(0, 10)) { + log(COLORS.red(` - ${err}`)); + } + if (allErrors.length > 10) { + log(COLORS.gray(` ... y ${allErrors.length - 10} más`)); + } + log(''); + } + + if (documents.length > 0) { + log(COLORS.gray(' Muestra — primer documento importado:')); + log(COLORS.gray(` id: ${documents[0].id}`)); + log(COLORS.gray(` material_id: ${documents[0].material_id}`)); + log(COLORS.gray(` title: ${(documents[0].title ?? '').slice(0, 60)}`)); + log(COLORS.gray(` locale: ${documents[0].locale}`)); + log(COLORS.gray(` type: ${documents[0].type}`)); + log(COLORS.gray(` private: ${documents[0].private}`)); + log(COLORS.gray(` files: ${(documents[0].files ?? []).length}`)); + log(COLORS.gray(` images: ${(documents[0].images ?? []).length}`)); + log(COLORS.gray(` related: ${JSON.stringify(documents[0].related ?? [])}`)); + log(''); + } + + log(COLORS.bold(hasErrors ? COLORS.yellow(' ⚠️ Proceso completado con errores') : COLORS.green(' ✅ Proceso completado exitosamente'))); + log(''); +} + +main().catch(err => { + log(COLORS.red(`\n ❌ Error fatal: ${err.message}`)); + log(err.stack); + process.exit(1); +}); diff --git a/reindex.mjs b/reindex.mjs new file mode 100644 index 0000000..3c2ee4b --- /dev/null +++ b/reindex.mjs @@ -0,0 +1,517 @@ +#!/usr/bin/env node +/** + * reindex.mjs + * + * Reindexa activities_translations o conferences_translations + * desde Directus a Typesense vía bulk upsert. + * + * USO: + * node reindex.mjs --collection activities # (default) + * node reindex.mjs --collection conferences + * node reindex.mjs --collection activities --limit 100 --offset 200 + * node reindex.mjs --collection conferences --dry-run --limit 50 + * + * Config: copiar .env.example a .env y llenar variables. + */ + +const __start = Date.now(); + +// ── Config desde .env ──────────────────────────────────────────── +import { readFileSync, existsSync } from 'node:fs'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dir = dirname(fileURLToPath(import.meta.url)); +const dotEnv = resolve(__dir, '.env'); + +function loadEnv(path) { + if (!existsSync(path)) return {}; + const env = {}; + for (const line of readFileSync(path, 'utf-8').split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#') || !trimmed.includes('=')) continue; + const eq = trimmed.indexOf('='); + const key = trimmed.slice(0, eq).trim(); + const val = trimmed.slice(eq + 1).trim().replace(/^["']|["']$/g, ''); + env[key] = val; + } + return env; +} + +const dotenv = loadEnv(dotEnv); + +// ── CLI args ───────────────────────────────────────────────────── +const ARGS = {}; +for (const arg of process.argv.slice(2)) { + if (arg === '--dry-run') ARGS.dryRun = true; + else if (arg.startsWith('--collection=')) ARGS.collection = arg.split('=')[1]; + else if (arg === '--collection') ARGS.collection = process.argv[process.argv.indexOf(arg) + 1]; + else if (arg.startsWith('--limit=')) ARGS.limit = parseInt(arg.split('=')[1], 10); + else if (arg.startsWith('--limit')) ARGS.limit = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10); + else if (arg.startsWith('--offset=')) ARGS.offset = parseInt(arg.split('=')[1], 10); + else if (arg.startsWith('--offset')) ARGS.offset = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10); +} + +// ── CFG ────────────────────────────────────────────────────────── +const CFG = { + directusUrl: dotenv.DIRECTUS_URL || process.env.DIRECTUS_URL || '', + directusToken: dotenv.DIRECTUS_TOKEN || process.env.DIRECTUS_TOKEN || '', + typesenseUrl: dotenv.TYPESENSE_URL || process.env.TYPESENSE_URL || '', + typesenseKey: dotenv.TYPESENSE_ADMIN_KEY || process.env.TYPESENSE_ADMIN_KEY || '', + batchSize: parseInt(dotenv.BATCH_SIZE || process.env.BATCH_SIZE || '500'), + pageLimit: parseInt(dotenv.PAGE_LIMIT || process.env.PAGE_LIMIT || '200'), + limit: ARGS.limit ?? parseInt(dotenv.LIMIT || process.env.LIMIT || '0'), + offset: ARGS.offset ?? parseInt(dotenv.OFFSET || process.env.OFFSET || '0'), + collection: ARGS.collection || 'activities', + dryRun: !!ARGS.dryRun, +}; + +// ── Definición de colecciones ──────────────────────────────────── +const COLLECTIONS = { + activities: { + label: 'activities', + directusCollection: 'activities_translations', + relationField: 'activities_id', + extraFields: ['interventions.text'], + mainCollection: 'activities', + paragraphsCollection: 'activities_paragraphs', + groupField: 'activities_id', + type: 'activities', + buildId: (data, rel, locale) => `${locale}-${data.id}`, + getParagraphHtml: (data) => data.interventions?.[0]?.text ?? '', + draftDefault: true, + isPrivate: (rel) => rel.private ?? false, + }, + conferences: { + label: 'conferences', + directusCollection: 'conferences_translations', + relationField: 'conferences_id', + extraFields: [], + mainCollection: 'conferences', + paragraphsCollection: 'conferences_paragraphs', + groupField: 'conferences_id', + type: 'conferences', + buildId: (data, rel, locale) => `${locale}-${rel.wpid ?? data.id ?? ''}`, + getParagraphHtml: (data) => data.html ?? '', + draftDefault: false, + isPrivate: (rel) => !rel.public, + }, +}; + +const COL = COLLECTIONS[CFG.collection]; +if (!COL) { + console.error(` ❌ Colección desconocida: "${CFG.collection}". Usa --collection activities | conferences`); + process.exit(1); +} + +// ── Helpers ────────────────────────────────────────────────────── + +const COLORS = { + red: s => `\x1b[31m${s}\x1b[0m`, + green: s => `\x1b[32m${s}\x1b[0m`, + yellow: s => `\x1b[33m${s}\x1b[0m`, + blue: s => `\x1b[34m${s}\x1b[0m`, + magenta: s => `\x1b[35m${s}\x1b[0m`, + cyan: s => `\x1b[36m${s}\x1b[0m`, + gray: s => `\x1b[90m${s}\x1b[0m`, + bold: s => `\x1b[1m${s}\x1b[0m`, +}; + +function log(...args) { console.log(...args); } + +function elapsed() { + const sec = Math.floor((Date.now() - __start) / 1000); + const m = Math.floor(sec / 60); + const s = sec % 60; + return m > 0 ? `${m}m ${s}s` : `${s}s`; +} + +function fmtNum(n) { + return n.toLocaleString('en-US'); +} + +// ── API calls ──────────────────────────────────────────────────── + +async function directusFetch(path) { + const url = `${CFG.directusUrl}/items/${path}`; + const sep = path.includes('?') ? '&' : '?'; + const fullUrl = `${url}${sep}access_token=${CFG.directusToken}`; + + const res = await fetch(fullUrl, { + headers: { 'User-Agent': 'reindex/1.0' }, + }); + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new Error(`Directus ${res.status} ${res.statusText}: ${text.slice(0, 200)}`); + } + return res.json(); +} + +async function typesenseImport(collection, docs) { + if (docs.length === 0) return { success: 0, error: 0, errors: [] }; + + const body = docs.map(d => JSON.stringify(d)).join('\n'); + const url = `${CFG.typesenseUrl}/collections/${collection}/documents/import?action=upsert`; + const res = await fetch(url, { + method: 'POST', + headers: { + 'X-TYPESENSE-API-KEY': CFG.typesenseKey, + 'Content-Type': 'text/plain', + }, + body, + }); + + const text = await res.text(); + + let success = 0, error = 0; + const errors = []; + + if (!text.startsWith('{')) { + throw new Error(`Typesense import devolvió formato inesperado: ${text.slice(0, 200)}`); + } + + for (const line of text.trim().split('\n')) { + if (!line.trim()) continue; + try { + const result = JSON.parse(line); + if (result.success) success++; + else { error++; errors.push(result.error || 'desconocido'); } + } catch (e) { + error++; + errors.push(`error parseando línea: ${line.slice(0, 100)}`); + } + } + + return { success, error, errors }; +} + +// ── Transformación ─────────────────────────────────────────────── + +function getFilename(obj) { + return obj?.filename_disk ?? null; +} + +function transformItem(data, col) { + const rel = data[col.relationField] || {}; + const dateStr = rel.date; + const dateObj = dateStr ? new Date(dateStr) : null; + + const locale = data.languages_code?.code ?? 'es'; + + const yy = dateObj?.getFullYear?.(); + const mm = dateObj ? String(dateObj.getMonth() + 1).padStart(2, '0') : ''; + const dd = dateObj ? String(dateObj.getDate()).padStart(2, '0') : ''; + const dateFormatted = dateObj ? `${yy}${mm}${dd}` : ''; + const code = dateFormatted ? `${dateFormatted}-${rel.activity ?? ''}` : ''; + + const timestamp = dateObj ? Math.floor(dateObj.getTime() / 1000) : 0; + const dateIso = dateObj ? `${yy}-${mm}-${dd}` : null; + + const docId = col.buildId(data, rel, locale); + + const files = {}; + if (data.youtube?.startsWith('http')) files.youtube = data.youtube; + if (getFilename(data.mp3)) files.audio = getFilename(data.mp3); + if (getFilename(data.video)) files.video = getFilename(data.video); + if (getFilename(data.pdf)) files.simple = getFilename(data.pdf); + if (getFilename(data.pdf_booklet)) files.booklet = getFilename(data.pdf_booklet); + + const document = { + id: docId, + code, + locale, + type: col.type, + title: data.title?.replace(/"/g, '\\"') ?? '', + timestamp, + date: dateIso, + activity: rel.activity ?? null, + duration: rel.duration ?? 0, + bible_study: rel.bible_study ?? null, + place: rel.place ?? null, + city: rel.city ?? '', + state: rel.state ?? '', + country: rel.country ?? '', + draft: rel.draft ?? col.draftDefault, + private: col.isPrivate(rel), + files, + thumbnail: getFilename(rel.thumbnail), + directus: '', + wp: String(rel.wpid ?? ''), + slug: data.slug ?? null, + year: yy ? String(yy) : '', + month: mm, + }; + + for (const key of Object.keys(document)) { + if (document[key] === undefined) delete document[key]; + } + if (document.files && Object.keys(document.files).length === 0) { + document.files = {}; + } + + // ── Párrafos ─────────────────────────────────────────── + const bodyHtml = col.getParagraphHtml(data); + const paragraphs = []; + + if (bodyHtml) { + const hasPTags = /]*>/i.test(bodyHtml); + const rawParagraphs = hasPTags + ? bodyHtml.split(/<\/?p[^>]*>/).map(p => p.trim()).filter(Boolean) + : bodyHtml.split('\n\n').map(p => p.trim()).filter(Boolean); + + for (const para of rawParagraphs) { + if (!para || para.trim() === '' || para === '\u00a0') continue; + + const fixedHtml = para.trim().startsWith('${para.trim()}

`; + + const pNum = paragraphs.length + 1; + const pId = `${docId}-${pNum}`; + + paragraphs.push({ + id: pId, + [col.groupField]: docId, + code: document.code, + html: fixedHtml, + text: fixedHtml.replace(/<[^>]+>/g, ''), + number: pNum, + locale, + type: data.type ?? col.type, + }); + } + } + + return { document, paragraphs }; +} + +// ── Fetch desde Directus ───────────────────────────────────────── + +async function fetchAllTranslations(col) { + const allItems = []; + let page = 1; + let hasMore = true; + + const fields = [ + '*', + 'languages_code.code', + `${col.relationField}.*`, + `${col.relationField}.thumbnail.filename_disk`, + 'mp3.filename_disk', + 'video.filename_disk', + 'pdf.filename_disk', + 'pdf_booklet.filename_disk', + ...col.extraFields, + ].join(','); + + const limit = CFG.limit > 0 ? Math.min(CFG.limit, CFG.pageLimit) : CFG.pageLimit; + const offset = CFG.offset || 0; + let remaining = CFG.limit > 0 ? CFG.limit : Infinity; + + while (hasMore && remaining > 0) { + const currentLimit = Math.min(limit, remaining); + const pageOffset = offset + ((page - 1) * currentLimit); + const qs = `${col.directusCollection}?limit=${currentLimit}&page=${page}&offset=${pageOffset}&fields=${encodeURIComponent(fields)}`; + + log(COLORS.blue(` → Directus: page ${page} (limit ${currentLimit}, offset ${pageOffset})...`)); + + const json = await directusFetch(qs); + const items = json.data || []; + + if (items.length === 0) { + hasMore = false; + break; + } + + for (const item of items) { + allItems.push(item); + remaining--; + } + + page++; + if (items.length < currentLimit) hasMore = false; + } + + return allItems; +} + +// ── Import a Typesense ─────────────────────────────────────────── + +async function importBatch(collection, docs, label) { + const result = await typesenseImport(collection, docs); + const color = result.error > 0 ? COLORS.yellow : COLORS.green; + log(color(` ${label}: ${fmtNum(result.success)} ok, ${fmtNum(result.error)} fail`)); + return result; +} + +// ── Main ───────────────────────────────────────────────────────── + +async function main() { + log(''); + log(COLORS.bold(COLORS.cyan('════════════════════════════════════════'))); + log(COLORS.bold(COLORS.cyan(` REINDEX ${COL.label.toUpperCase()} → TYPESENSE`))); + log(COLORS.bold(COLORS.cyan('════════════════════════════════════════'))); + log(''); + + // ── Validación ──────────────────────────────────────── + const missing = []; + if (!CFG.directusUrl) missing.push('DIRECTUS_URL'); + if (!CFG.directusToken) missing.push('DIRECTUS_TOKEN'); + if (!CFG.typesenseUrl) missing.push('TYPESENSE_URL'); + if (!CFG.typesenseKey) missing.push('TYPESENSE_ADMIN_KEY'); + + if (missing.length > 0) { + log(COLORS.red(` ❌ Faltan variables: ${missing.join(', ')}`)); + log(COLORS.gray(` Copia .env.example a .env y completa los valores`)); + process.exit(1); + } + + log(` Colección: ${COLORS.cyan(COL.label)}`); + log(` DirectUS: ${COLORS.blue(CFG.directusUrl)}`); + log(` Typesense: ${COLORS.blue(CFG.typesenseUrl)}`); + if (CFG.limit > 0) log(` Límite: ${COLORS.yellow(fmtNum(CFG.limit))} registros`); + if (CFG.offset > 0) log(` Offset: ${COLORS.yellow(fmtNum(CFG.offset))} (saltando los primeros)`); + if (CFG.dryRun) log(` ${COLORS.magenta('Modo: DRY RUN — no se escribirá nada en Typesense')}`); + log(''); + + // ── 1. Fetch desde Directus ─────────────────────────── + log(COLORS.bold('Fase 1: Leyendo datos desde Directus...')); + let translations; + try { + translations = await fetchAllTranslations(COL); + } catch (err) { + log(COLORS.red(` ❌ Error leyendo Directus: ${err.message}`)); + process.exit(1); + } + + log(COLORS.green(` ✅ ${fmtNum(translations.length)} ${COL.directusCollection} leídas (${elapsed()})`)); + log(''); + + // ── 2. Transformar ──────────────────────────────────── + log(COLORS.bold('Fase 2: Transformando datos...')); + const allDocuments = []; + const allParagraphs = []; + let docsWithContent = 0; + + for (const item of translations) { + const { document, paragraphs } = transformItem(item, COL); + allDocuments.push(document); + allParagraphs.push(...paragraphs); + if (paragraphs.length > 0) docsWithContent++; + } + + const uniqueDates = new Set(allDocuments.map(d => d.code?.split('-')[0]).filter(Boolean)).size; + + log(COLORS.green(` ✅ ${fmtNum(allDocuments.length)} documentos para ${COL.mainCollection}`)); + log(COLORS.green(` ✅ ${fmtNum(allParagraphs.length)} párrafos para ${COL.paragraphsCollection}`)); + log(COLORS.gray(` ${fmtNum(docsWithContent)} con contenido, ${fmtNum(uniqueDates)} fechas únicas`)); + log(''); + + // ── 3. Dry-run: salir ───────────────────────────────── + if (CFG.dryRun) { + log(COLORS.magenta('╔══════════════════════════════════════╗')); + log(COLORS.magenta('║ DRY RUN — nada se importó ║')); + log(COLORS.magenta('╚══════════════════════════════════════╝')); + log(''); + if (allDocuments.length > 0) { + log(COLORS.gray(' Primer documento (sample):')); + log(COLORS.gray(` ${JSON.stringify(allDocuments[0], null, 2).slice(0, 600)}`)); + } + log(''); + return; + } + + // ── 3. Importar a Typesense ─────────────────────────── + log(COLORS.bold(`Fase 3: Importando a Typesense (batches de ${CFG.batchSize})...`)); + log(COLORS.gray(' (upsert: si el id existe se actualiza, si no se crea)')); + + let totalDocOk = 0, totalDocFail = 0; + let totalParOk = 0, totalParFail = 0; + const allErrors = []; + + log(COLORS.cyan(` ── ${COL.mainCollection} ──`)); + for (let i = 0; i < allDocuments.length; i += CFG.batchSize) { + const batch = allDocuments.slice(i, i + CFG.batchSize); + const label = `[${fmtNum(i + 1)}–${fmtNum(Math.min(i + CFG.batchSize, allDocuments.length))}/${fmtNum(allDocuments.length)}]`; + try { + const result = await importBatch(COL.mainCollection, batch, label); + totalDocOk += result.success; + totalDocFail += result.error; + allErrors.push(...result.errors); + } catch (err) { + log(COLORS.red(` ${label} ERROR: ${err.message}`)); + totalDocFail += batch.length; + } + } + + log(COLORS.cyan(` ── ${COL.paragraphsCollection} ──`)); + for (let i = 0; i < allParagraphs.length; i += CFG.batchSize) { + const batch = allParagraphs.slice(i, i + CFG.batchSize); + const label = `[${fmtNum(i + 1)}–${fmtNum(Math.min(i + CFG.batchSize, allParagraphs.length))}/${fmtNum(allParagraphs.length)}]`; + try { + const result = await importBatch(COL.paragraphsCollection, batch, label); + totalParOk += result.success; + totalParFail += result.error; + allErrors.push(...result.errors); + } catch (err) { + log(COLORS.red(` ${label} ERROR: ${err.message}`)); + totalParFail += batch.length; + } + } + + log(''); + + // ── Reporte final ───────────────────────────────────── + const hasErrors = totalDocFail > 0 || totalParFail > 0; + const titleColor = hasErrors ? COLORS.yellow : COLORS.green; + + log(titleColor(COLORS.bold('════════════════════════════════════════'))); + log(titleColor(COLORS.bold(' REPORTE FINAL'))); + log(titleColor(COLORS.bold('════════════════════════════════════════'))); + log(''); + log(` Directus:`); + log(` ${COL.directusCollection}: ${COLORS.cyan(fmtNum(translations.length))}`); + log(` fechas únicas: ${COLORS.cyan(fmtNum(uniqueDates))}`); + log(` con párrafos: ${COLORS.cyan(fmtNum(docsWithContent))}`); + log(''); + log(` Typesense:`); + const docOk = COLORS.green(fmtNum(totalDocOk)); + const docFail = totalDocFail > 0 ? COLORS.red(fmtNum(totalDocFail)) : COLORS.green(fmtNum(totalDocFail)); + log(` ${COL.mainCollection}: ${docOk} importados, ${docFail} errores`); + const parOk = COLORS.green(fmtNum(totalParOk)); + const parFail = totalParFail > 0 ? COLORS.red(fmtNum(totalParFail)) : COLORS.green(fmtNum(totalParFail)); + log(` ${COL.paragraphsCollection}: ${parOk} importados, ${parFail} errores`); + log(''); + log(` ${COLORS.bold('Duración:')} ${elapsed()}`); + log(''); + + if (allErrors.length > 0) { + log(COLORS.yellow(COLORS.bold(' ⚠️ Errores de Typesense (primeros 10):'))); + for (const err of allErrors.slice(0, 10)) { + log(COLORS.red(` - ${err}`)); + } + if (allErrors.length > 10) { + log(COLORS.gray(` ... y ${allErrors.length - 10} más`)); + } + log(''); + } + + if (allDocuments.length > 0) { + log(COLORS.gray(' Muestra — primer documento importado:')); + log(COLORS.gray(` id: ${allDocuments[0].id}`)); + log(COLORS.gray(` title: ${(allDocuments[0].title ?? '').slice(0, 60)}`)); + log(COLORS.gray(` files: ${JSON.stringify(allDocuments[0].files)}`)); + log(COLORS.gray(` thumbnail: ${allDocuments[0].thumbnail}`)); + log(COLORS.gray(` párrafos: ${allParagraphs.filter(p => p[COL.groupField] === allDocuments[0].id).length}`)); + log(''); + } + + log(COLORS.bold(hasErrors ? COLORS.yellow(' ⚠️ Proceso completado con errores') : COLORS.green(' ✅ Proceso completado exitosamente'))); + log(''); +} + +main().catch(err => { + log(COLORS.red(`\n ❌ Error fatal: ${err.message}`)); + log(err.stack); + process.exit(1); +});