415 lines
17 KiB
JavaScript
415 lines
17 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* reindex-print-materials.mjs
|
||
*
|
||
* Reindexa print_materials desde Directus a Typesense.
|
||
*
|
||
* Los items de Directus contienen traducciones anidadas (translations[]),
|
||
* cada una con sus propios archivos (files[]) e imágenes (files_images[]).
|
||
*
|
||
* USO:
|
||
* node reindex-print-materials.mjs
|
||
* node reindex-print-materials.mjs --limit 10 --offset 0
|
||
* node reindex-print-materials.mjs --dry-run --limit 5
|
||
*
|
||
* Config: copiar .env.example a .env y llenar variables.
|
||
*/
|
||
|
||
const __start = Date.now();
|
||
|
||
// ── Config desde .env ────────────────────────────────────────────
|
||
import { readFileSync, existsSync } from 'node:fs';
|
||
import { resolve, dirname } from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
|
||
const __dir = dirname(fileURLToPath(import.meta.url));
|
||
const dotEnv = resolve(__dir, '.env');
|
||
|
||
function loadEnv(path) {
|
||
if (!existsSync(path)) return {};
|
||
const env = {};
|
||
for (const line of readFileSync(path, 'utf-8').split('\n')) {
|
||
const trimmed = line.trim();
|
||
if (!trimmed || trimmed.startsWith('#') || !trimmed.includes('=')) continue;
|
||
const eq = trimmed.indexOf('=');
|
||
const key = trimmed.slice(0, eq).trim();
|
||
const val = trimmed.slice(eq + 1).trim().replace(/^["']|["']$/g, '');
|
||
env[key] = val;
|
||
}
|
||
return env;
|
||
}
|
||
|
||
const dotenv = loadEnv(dotEnv);
|
||
|
||
// ── CLI args ─────────────────────────────────────────────────────
|
||
const ARGS = {};
|
||
for (const arg of process.argv.slice(2)) {
|
||
if (arg === '--dry-run') ARGS.dryRun = true;
|
||
else if (arg.startsWith('--limit=')) ARGS.limit = parseInt(arg.split('=')[1], 10);
|
||
else if (arg === '--limit') ARGS.limit = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10);
|
||
else if (arg.startsWith('--offset=')) ARGS.offset = parseInt(arg.split('=')[1], 10);
|
||
else if (arg === '--offset') ARGS.offset = parseInt(process.argv[process.argv.indexOf(arg) + 1], 10);
|
||
}
|
||
|
||
// ── CFG ──────────────────────────────────────────────────────────
|
||
const CFG = {
|
||
directusUrl: dotenv.PRINT_DIRECTUS_URL || dotenv.DIRECTUS_URL || process.env.PRINT_DIRECTUS_URL || process.env.DIRECTUS_URL || '',
|
||
directusToken: dotenv.PRINT_DIRECTUS_TOKEN || dotenv.DIRECTUS_TOKEN || process.env.PRINT_DIRECTUS_TOKEN || process.env.DIRECTUS_TOKEN || '',
|
||
typesenseUrl: dotenv.PRINT_TYPESENSE_URL || dotenv.TYPESENSE_URL || process.env.PRINT_TYPESENSE_URL || process.env.TYPESENSE_URL || '',
|
||
typesenseKey: dotenv.PRINT_TYPESENSE_ADMIN_KEY || dotenv.TYPESENSE_ADMIN_KEY || process.env.PRINT_TYPESENSE_ADMIN_KEY || process.env.TYPESENSE_ADMIN_KEY || '',
|
||
typesenseCollection: dotenv.PRINT_TYPESENSE_COLLECTION || dotenv.TYPESENSE_COLLECTION || process.env.PRINT_TYPESENSE_COLLECTION || process.env.TYPESENSE_COLLECTION || 'print-materials',
|
||
batchSize: parseInt(dotenv.BATCH_SIZE || process.env.BATCH_SIZE || '500'),
|
||
pageLimit: parseInt(dotenv.PAGE_LIMIT || process.env.PAGE_LIMIT || '200'),
|
||
limit: ARGS.limit ?? parseInt(dotenv.LIMIT || process.env.LIMIT || '0'),
|
||
offset: ARGS.offset ?? parseInt(dotenv.OFFSET || process.env.OFFSET || '0'),
|
||
dryRun: !!ARGS.dryRun,
|
||
};
|
||
|
||
// ── Helpers ──────────────────────────────────────────────────────
|
||
|
||
const COLORS = {
|
||
red: s => `\x1b[31m${s}\x1b[0m`,
|
||
green: s => `\x1b[32m${s}\x1b[0m`,
|
||
yellow: s => `\x1b[33m${s}\x1b[0m`,
|
||
blue: s => `\x1b[34m${s}\x1b[0m`,
|
||
magenta: s => `\x1b[35m${s}\x1b[0m`,
|
||
cyan: s => `\x1b[36m${s}\x1b[0m`,
|
||
gray: s => `\x1b[90m${s}\x1b[0m`,
|
||
bold: s => `\x1b[1m${s}\x1b[0m`,
|
||
};
|
||
|
||
function log(...args) { console.log(...args); }
|
||
|
||
function elapsed() {
|
||
const sec = Math.floor((Date.now() - __start) / 1000);
|
||
const m = Math.floor(sec / 60);
|
||
const s = sec % 60;
|
||
return m > 0 ? `${m}m ${s}s` : `${s}s`;
|
||
}
|
||
|
||
function fmtNum(n) {
|
||
return n.toLocaleString('en-US');
|
||
}
|
||
|
||
// ── API calls ────────────────────────────────────────────────────
|
||
|
||
async function directusFetch(path) {
|
||
const url = `${CFG.directusUrl}/items/${path}`;
|
||
const sep = path.includes('?') ? '&' : '?';
|
||
const fullUrl = `${url}${sep}access_token=${CFG.directusToken}`;
|
||
|
||
const res = await fetch(fullUrl, {
|
||
headers: { 'User-Agent': 'reindex/1.0' },
|
||
});
|
||
if (!res.ok) {
|
||
const text = await res.text().catch(() => '');
|
||
throw new Error(`Directus ${res.status} ${res.statusText}: ${text.slice(0, 200)}`);
|
||
}
|
||
return res.json();
|
||
}
|
||
|
||
async function typesenseImport(docs) {
|
||
if (docs.length === 0) return { success: 0, error: 0, errors: [] };
|
||
|
||
const body = docs.map(d => JSON.stringify(d)).join('\n');
|
||
const url = `${CFG.typesenseUrl}/collections/${CFG.typesenseCollection}/documents/import?action=upsert`;
|
||
const res = await fetch(url, {
|
||
method: 'POST',
|
||
headers: {
|
||
'X-TYPESENSE-API-KEY': CFG.typesenseKey,
|
||
'Content-Type': 'text/plain',
|
||
},
|
||
body,
|
||
});
|
||
|
||
const text = await res.text();
|
||
|
||
let success = 0, error = 0;
|
||
const errors = [];
|
||
|
||
if (!text.startsWith('{')) {
|
||
throw new Error(`Typesense import devolvió formato inesperado: ${text.slice(0, 200)}`);
|
||
}
|
||
|
||
for (const line of text.trim().split('\n')) {
|
||
if (!line.trim()) continue;
|
||
try {
|
||
const result = JSON.parse(line);
|
||
if (result.success) success++;
|
||
else { error++; errors.push(result.error || 'desconocido'); }
|
||
} catch (e) {
|
||
error++;
|
||
errors.push(`error parseando línea: ${line.slice(0, 100)}`);
|
||
}
|
||
}
|
||
|
||
return { success, error, errors };
|
||
}
|
||
|
||
// ── Helpers de transformación ────────────────────────────────────
|
||
|
||
function toUnixTimestamp(dateStr) {
|
||
if (!dateStr) return undefined;
|
||
const ts = Math.floor(new Date(dateStr).getTime() / 1000);
|
||
return isNaN(ts) ? undefined : ts;
|
||
}
|
||
|
||
function getThumbnail(material) {
|
||
return material.thumbnail?.filename_disk ?? null;
|
||
}
|
||
|
||
// ── Transformación ───────────────────────────────────────────────
|
||
// Replica la lógica del n8n (formateo a Typesense)
|
||
|
||
function transformMaterial(material) {
|
||
const translations = material.translations || [];
|
||
const docs = [];
|
||
|
||
for (const t of translations) {
|
||
if (!t.title) continue;
|
||
|
||
const doc = {
|
||
id: String(material.id) + '_' + t.languages_code,
|
||
material_id: material.id,
|
||
title: t.title,
|
||
thumbnail: getThumbnail(material),
|
||
slug: t.slug || String(material.id),
|
||
locale: t.languages_code,
|
||
type: material.type,
|
||
private: !!t.private,
|
||
images: (t.files_images || []).map(item => ({
|
||
file: item.directus_files_id?.filename_disk || '',
|
||
downloadname: '',
|
||
})).filter(img => img.file),
|
||
files: (t.files || []).map(item => ({
|
||
filename: item.file?.filename_disk || '',
|
||
downloadname: item.name || '',
|
||
type: item.tipo || '',
|
||
})).filter(f => f.filename),
|
||
related: (material.material_relacionado || []).map(item => {
|
||
const val = typeof item === 'object' ? (item.related_print_materials_id ?? item) : item;
|
||
return parseInt(val, 10);
|
||
}).filter(n => !isNaN(n)),
|
||
};
|
||
|
||
const printCategory = (material.print_category || [])
|
||
.map(item => String(typeof item === 'object' ? (item.print_categories_id ?? item.id ?? item) : item))
|
||
.join(',') || undefined;
|
||
if (printCategory) doc['print-category'] = printCategory;
|
||
|
||
if (t.slug_personalizado) doc['custom-slug'] = t.slug_personalizado;
|
||
if (material.subtype) doc.subtype = material.subtype;
|
||
if (material.number != null) doc.number = material.number;
|
||
if (t.notas) doc.notes = t.notas;
|
||
|
||
const ts = toUnixTimestamp(material.date);
|
||
if (ts !== undefined) doc.date = ts;
|
||
|
||
docs.push(doc);
|
||
}
|
||
|
||
return docs;
|
||
}
|
||
|
||
// ── Fetch desde Directus ─────────────────────────────────────────
|
||
|
||
async function fetchAllMaterials() {
|
||
const allItems = [];
|
||
let page = 1;
|
||
let hasMore = true;
|
||
|
||
const fields = [
|
||
'*',
|
||
'thumbnail.filename_disk',
|
||
'translations.*',
|
||
'translations.files.*',
|
||
'translations.files.file.filename_disk',
|
||
'translations.files_images.*',
|
||
'translations.files_images.directus_files_id.filename_disk',
|
||
'material_relacionado.related_print_materials_id',
|
||
'print_category.print_categories_id',
|
||
];
|
||
|
||
const fieldsParam = fields.map(f => `fields[]=${encodeURIComponent(f)}`).join('&');
|
||
|
||
const limit = CFG.limit > 0 ? Math.min(CFG.limit, CFG.pageLimit) : CFG.pageLimit;
|
||
const offset = CFG.offset || 0;
|
||
let remaining = CFG.limit > 0 ? CFG.limit : Infinity;
|
||
|
||
while (hasMore && remaining > 0) {
|
||
const currentLimit = Math.min(limit, remaining);
|
||
const pageOffset = offset + ((page - 1) * currentLimit);
|
||
const qs = `print_materials?limit=${currentLimit}&page=${page}&offset=${pageOffset}&${fieldsParam}`;
|
||
|
||
log(COLORS.blue(` → Directus: page ${page} (limit ${currentLimit}, offset ${pageOffset})...`));
|
||
|
||
const json = await directusFetch(qs);
|
||
const items = json.data || [];
|
||
|
||
if (items.length === 0) {
|
||
hasMore = false;
|
||
break;
|
||
}
|
||
|
||
for (const item of items) {
|
||
allItems.push(item);
|
||
remaining--;
|
||
}
|
||
|
||
page++;
|
||
if (items.length < currentLimit) hasMore = false;
|
||
}
|
||
|
||
return allItems;
|
||
}
|
||
|
||
// ── Main ─────────────────────────────────────────────────────────
|
||
|
||
async function main() {
|
||
log('');
|
||
log(COLORS.bold(COLORS.cyan('════════════════════════════════════════')));
|
||
log(COLORS.bold(COLORS.cyan(' REINDEX PRINT-MATERIALS → TYPESENSE')));
|
||
log(COLORS.bold(COLORS.cyan('════════════════════════════════════════')));
|
||
log('');
|
||
|
||
// ── Validación ────────────────────────────────────────
|
||
const missing = [];
|
||
if (!CFG.directusUrl) missing.push('DIRECTUS_URL');
|
||
if (!CFG.directusToken) missing.push('DIRECTUS_TOKEN');
|
||
if (!CFG.typesenseUrl) missing.push('TYPESENSE_URL');
|
||
if (!CFG.typesenseKey) missing.push('TYPESENSE_ADMIN_KEY');
|
||
|
||
if (missing.length > 0) {
|
||
log(COLORS.red(` ❌ Faltan variables: ${missing.join(', ')}`));
|
||
log(COLORS.gray(` Copia .env.example a .env y completa los valores`));
|
||
process.exit(1);
|
||
}
|
||
|
||
log(` Tiposense: ${COLORS.blue(CFG.typesenseUrl)}/collections/${COLORS.cyan(CFG.typesenseCollection)}`);
|
||
log(` Directus: ${COLORS.blue(CFG.directusUrl)}`);
|
||
if (CFG.limit > 0) log(` Límite: ${COLORS.yellow(fmtNum(CFG.limit))} registros`);
|
||
if (CFG.offset > 0) log(` Offset: ${COLORS.yellow(fmtNum(CFG.offset))} (saltando los primeros)`);
|
||
if (CFG.dryRun) log(` ${COLORS.magenta('Modo: DRY RUN — no se escribirá nada en Typesense')}`);
|
||
log('');
|
||
|
||
// ── 1. Fetch desde Directus ───────────────────────────
|
||
log(COLORS.bold('Fase 1: Leyendo datos desde Directus...'));
|
||
let materials;
|
||
try {
|
||
materials = await fetchAllMaterials();
|
||
} catch (err) {
|
||
log(COLORS.red(` ❌ Error leyendo Directus: ${err.message}`));
|
||
process.exit(1);
|
||
}
|
||
|
||
log(COLORS.green(` ✅ ${fmtNum(materials.length)} print_materials leídos (${elapsed()})`));
|
||
log('');
|
||
|
||
// ── 2. Transformar ────────────────────────────────────
|
||
log(COLORS.bold('Fase 2: Transformando datos...'));
|
||
const documents = [];
|
||
let materialsWithContent = 0;
|
||
|
||
for (const material of materials) {
|
||
const docs = transformMaterial(material);
|
||
documents.push(...docs);
|
||
if (docs.length > 0) materialsWithContent++;
|
||
}
|
||
|
||
log(COLORS.green(` ✅ ${fmtNum(documents.length)} documentos generados desde ${fmtNum(materialsWithContent)} materiales`));
|
||
log('');
|
||
|
||
// ── Dry-run: salir ────────────────────────────────────
|
||
if (CFG.dryRun) {
|
||
log(COLORS.magenta('╔══════════════════════════════════════╗'));
|
||
log(COLORS.magenta('║ DRY RUN — nada se importó ║'));
|
||
log(COLORS.magenta('╚══════════════════════════════════════╝'));
|
||
log('');
|
||
if (documents.length > 0) {
|
||
log(COLORS.gray(' Primer documento (sample):'));
|
||
log(COLORS.gray(` ${JSON.stringify(documents[0], null, 2).slice(0, 800)}`));
|
||
}
|
||
log('');
|
||
return;
|
||
}
|
||
|
||
// ── 3. Importar a Typesense ───────────────────────────
|
||
log(COLORS.bold(`Fase 3: Importando a Typesense (batches de ${CFG.batchSize})...`));
|
||
log(COLORS.gray(' (upsert: si el id existe se actualiza, si no se crea)'));
|
||
|
||
let totalOk = 0, totalFail = 0;
|
||
const allErrors = [];
|
||
|
||
for (let i = 0; i < documents.length; i += CFG.batchSize) {
|
||
const batch = documents.slice(i, i + CFG.batchSize);
|
||
const label = `[${fmtNum(i + 1)}–${fmtNum(Math.min(i + CFG.batchSize, documents.length))}/${fmtNum(documents.length)}]`;
|
||
try {
|
||
const result = await typesenseImport(batch);
|
||
totalOk += result.success;
|
||
totalFail += result.error;
|
||
allErrors.push(...result.errors);
|
||
const color = result.error > 0 ? COLORS.yellow : COLORS.green;
|
||
log(color(` ${label}: ${fmtNum(result.success)} ok, ${fmtNum(result.error)} fail`));
|
||
} catch (err) {
|
||
log(COLORS.red(` ${label} ERROR: ${err.message}`));
|
||
totalFail += batch.length;
|
||
}
|
||
}
|
||
|
||
log('');
|
||
|
||
// ── Reporte final ─────────────────────────────────────
|
||
const hasErrors = totalFail > 0;
|
||
const titleColor = hasErrors ? COLORS.yellow : COLORS.green;
|
||
|
||
log(titleColor(COLORS.bold('════════════════════════════════════════')));
|
||
log(titleColor(COLORS.bold(' REPORTE FINAL')));
|
||
log(titleColor(COLORS.bold('════════════════════════════════════════')));
|
||
log('');
|
||
log(` Directus:`);
|
||
log(` print_materials: ${COLORS.cyan(fmtNum(materials.length))}`);
|
||
log(` con traducciones: ${COLORS.cyan(fmtNum(materialsWithContent))}`);
|
||
log('');
|
||
log(` Typesense (${CFG.typesenseCollection}):`);
|
||
const ok = COLORS.green(fmtNum(totalOk));
|
||
const fail = totalFail > 0 ? COLORS.red(fmtNum(totalFail)) : COLORS.green(fmtNum(totalFail));
|
||
log(` ${ok} importados, ${fail} errores`);
|
||
log('');
|
||
log(` ${COLORS.bold('Duración:')} ${elapsed()}`);
|
||
log('');
|
||
|
||
if (allErrors.length > 0) {
|
||
log(COLORS.yellow(COLORS.bold(' ⚠️ Errores de Typesense (primeros 10):')));
|
||
for (const err of allErrors.slice(0, 10)) {
|
||
log(COLORS.red(` - ${err}`));
|
||
}
|
||
if (allErrors.length > 10) {
|
||
log(COLORS.gray(` ... y ${allErrors.length - 10} más`));
|
||
}
|
||
log('');
|
||
}
|
||
|
||
if (documents.length > 0) {
|
||
log(COLORS.gray(' Muestra — primer documento importado:'));
|
||
log(COLORS.gray(` id: ${documents[0].id}`));
|
||
log(COLORS.gray(` material_id: ${documents[0].material_id}`));
|
||
log(COLORS.gray(` title: ${(documents[0].title ?? '').slice(0, 60)}`));
|
||
log(COLORS.gray(` locale: ${documents[0].locale}`));
|
||
log(COLORS.gray(` type: ${documents[0].type}`));
|
||
log(COLORS.gray(` private: ${documents[0].private}`));
|
||
log(COLORS.gray(` files: ${(documents[0].files ?? []).length}`));
|
||
log(COLORS.gray(` images: ${(documents[0].images ?? []).length}`));
|
||
log(COLORS.gray(` related: ${JSON.stringify(documents[0].related ?? [])}`));
|
||
log('');
|
||
}
|
||
|
||
log(COLORS.bold(hasErrors ? COLORS.yellow(' ⚠️ Proceso completado con errores') : COLORS.green(' ✅ Proceso completado exitosamente')));
|
||
log('');
|
||
}
|
||
|
||
main().catch(err => {
|
||
log(COLORS.red(`\n ❌ Error fatal: ${err.message}`));
|
||
log(err.stack);
|
||
process.exit(1);
|
||
});
|