LGCC
/
api
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed bulk for files

This commit is contained in:
Julio Ruiz 2026-07-11 06:56:42 -05:00
parent 8cf9a8052d
commit 550ede702c
4 changed files with 116 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
carpa_json_to_markdown/document_update.mjs
View File

@ -35,24 +35,25 @@ async function processJsonLines(filePath) {
// Parse the line as JSON // Parse the line as JSON
const data = JSON.parse(line); const data = JSON.parse(line);
//if( data.id != 62643 ) continue; // Skip until we find the specific ID //if( data.id != 'es-2956' ) continue; // Skip until we find the specific ID
console.log('Processing: ', data.title); const doc = await createDocument(data);
const doc_id = doc.id
const doc_id = await createDocument(data); console.log('Processing: ', doc.code);
console.log('Document created with ID:', doc_id); console.log('Document created with ID:', doc_id);
if ('body' in data && typeof data.body === 'string' && hasParagraphs(data.body).length > 0 && data.body.length > 0) { if ('body' in data && typeof data.body === 'string' && hasParagraphs(data.body).length > 0 && data.body.length > 0) {
// Split the body into paragraphs using <p> tags // Split the body into paragraphs using <p> tags
const paragraphs = data.body.split(/<\/?p>/).map(p => p.trim()).filter(Boolean); const paragraphs = data.body.split(/<\/?p>/).map(p => p.trim()).filter(Boolean);
const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type); const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type, doc.code);
//console.log('Total paragraphs found:', paragraphs); //console.log('Total BODY paragraphs found:', paragraphs);
} else { } else {
if( data.body !== undefined ){ if( data.body !== undefined ){
const paragraphs = data.body.split('\n\n').map(p => p.trim()).filter(Boolean); const paragraphs = data.body.split('\n\n').map(p => p.trim()).filter(Boolean);
const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type); const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type, doc.code);
//console.log('Total paragraphs found:', paragraphs); //console.log('Total NO BODY paragraphs found:', paragraphs);
} }
} }
} catch (error) { } catch (error) {
@ -75,11 +76,11 @@ function hasParagraphs(htmlString) {
return dom.window.document.querySelectorAll('p'); return dom.window.document.querySelectorAll('p');
} }
//processJsonLines('./input/activities_wp_ES.json'); processJsonLines('./input/activities_translations_PT.json');
processJsonLines('./input/conferences_wp_ES.json'); //processJsonLines('./input/conferences_wp_ES.json');
// Function to create paragraphs in Typesense // Function to create paragraphs in Typesense
async function createParagraphs(documentId, paragraphs, locale, type) { async function createParagraphs(documentId, paragraphs, locale, type, code) {
let lineCount = 0; let lineCount = 0;
let jsonl = ''; let jsonl = '';
for (const para of paragraphs) { for (const para of paragraphs) {
@ -96,21 +97,25 @@ async function createParagraphs(documentId, paragraphs, locale, type) {
} }
let paragraph = { let paragraph = {
document_id: documentId, staging_activities_id: documentId,
raw: fixedHtml, html: fixedHtml,
text: fixedHtml.replace(/<[^>]+>/g, ''), // Remove HTML tags for search text: fixedHtml.replace(/<[^>]+>/g, ''), // Remove HTML tags for search
number: lineCount + 1, number: lineCount + 1,
locale: locale, locale: locale,
type: type, type: type,
id: locale + '-' + documentId + '-' + (lineCount + 1) // Unique ID for the paragraph code: code,
id: documentId + '-' + (lineCount + 1) // Unique ID for the paragraph
}; };
//console.log( code );
jsonl += JSON.stringify(paragraph) + '\n'; jsonl += JSON.stringify(paragraph) + '\n';
lineCount++; lineCount++;
} }
let doc = await client.collections('paragraphs').documents().import(jsonl, {action: 'upsert'}) let doc = await client.collections('staging_activities_paragraphs').documents().import(jsonl, {action: 'upsert'})
return doc; // Return the number of paragraphs created return doc; // Return the number of paragraphs created
} }
@ -134,14 +139,14 @@ async function createDocument(data){
draft: data.draft, draft: data.draft,
thumbnail: data.thumbnail, thumbnail: data.thumbnail,
files: { files: {
youtube: data.youtube, youtube: data.files.youtube,
video: data.video || null, video: data.files.video || null,
audio: data.audio || null, audio: data.files.audio || null,
booklet: data.booklet || null, booklet: data.files.booklet || null,
simple: data.simple || null, simple: data.files.simple || null,
}, },
directus: "", directus: data.id.toString(),
wp: data.id.toString(), wp: data.wp.toString(),
rm: data.rm, rm: data.rm,
private: data.private || false, private: data.private || false,
slug: data.slug || null, slug: data.slug || null,
@ -150,8 +155,8 @@ async function createDocument(data){
month: data.month, month: data.month,
}; };
let doc = await client.collections('documents').documents().upsert(document); let doc = await client.collections('staging_activities').documents().upsert(document);
return doc.id; return doc;
} }

91
carpa_json_to_markdown/index_directus.js
View File

@ -94,6 +94,7 @@ async function generateJson( type ) {
let fields = []; let fields = [];
fields['activities_translations'] = [ fields['activities_translations'] = [
'id',
'activities_id.id', 'activities_id.id',
'activities_id.wpid', 'activities_id.wpid',
'title', 'title',
@ -104,6 +105,7 @@ async function generateJson( type ) {
'activities_id.state', 'activities_id.state',
'activities_id.country', 'activities_id.country',
'activities_id.duration', 'activities_id.duration',
'interventions.title',
'interventions.text', 'interventions.text',
'activities_id.private', 'activities_id.private',
'activities_id.thumbnail.filename_disk', 'activities_id.thumbnail.filename_disk',
@ -116,8 +118,10 @@ async function generateJson( type ) {
] ]
fields['conferences_translations'] = [ fields['conferences_translations'] = [
'id',
'conferences_id.id', 'conferences_id.id',
'title', 'title',
'conferences_id.title',
'conferences_id.date', 'conferences_id.date',
'conferences_id.activity', 'conferences_id.activity',
'conferences_id.place', 'conferences_id.place',
@ -151,41 +155,42 @@ async function generateJson( type ) {
let nitem = {} let nitem = {}
if (type == 'activities_translations') { if (type == 'activities_translations') {
nitem.language = LOCALE;
//nitem.id = item.activities_id?.id nitem.locale = LOCALE;
//nitem.type = 'activities' nitem.id = `${nitem.locale}-${item.id}`
nitem.title = item.title nitem.title = item.title ? item.title : item.interventions[0]?.title
nitem.type = 'activities'
nitem.timestamp = dayjs(item.activities_id?.date).unix() nitem.timestamp = dayjs(item.activities_id?.date).unix()
nitem.date = ''; nitem.date = dayjs(item.activities_id?.date).format('YYYY-MM-DD')
nitem.activity = item.activities_id?.activity; nitem.activity = item.activities_id?.activity;
nitem.duration = item.activities_id?.duration ?? 0; nitem.duration = item.activities_id?.duration ?? 0;
//nitem.body = he.decode(striptags(item.interventions[0]?.text)) || '' nitem.body = item.interventions[0]?.text;
nitem.place = item.activities_id?.place || null; nitem.place = item.activities_id?.place || null;
nitem.city = item.activities_id?.city || null; nitem.city = item.activities_id?.city || null;
nitem.state = item.activities_id?.state || null; nitem.state = item.activities_id?.state || null;
nitem.country = item.activities_id?.country || null; nitem.country = item.activities_id?.country || null;
nitem.body = (item.interventions[0]?.text?true:false); //nitem.body = (item.interventions[0]?.text?true:false);
nitem.thumbnail = item.activities_id?.thumbnail?.filename_disk nitem.thumbnail = item.activities_id?.thumbnail?.filename_disk
//nitem.files = {} nitem.files = {}
nitem.youtube = item.youtube nitem.files.youtube = item.youtube
nitem.video = item.privateVideo?.filename_disk nitem.files.video = item.privateVideo?.filename_disk
nitem.audio = item.mp3?.filename_disk nitem.files.audio = item.mp3?.filename_disk
nitem.booklet = item.pdf_booklet?.filename_disk nitem.files.booklet = item.pdf_booklet?.filename_disk
nitem.simple = item.pdf?.filename_disk nitem.files.simple = item.pdf?.filename_disk
nitem.directus = item.activities_id?.id; nitem.directus = item.activities_id?.id;
nitem.wp = item.activities_id?.wpid; nitem.wp = item.activities_id?.wpid;
nitem.typesense = true; nitem.typesense = true;
nitem.private = item.activities_id?.private == 1 ? true : false; nitem.private = item.activities_id?.private == 1 ? true : false;
nitem.slug = item.slug nitem.slug = slugify(nitem.title)
//nitem.year = dayjs(item.activities_id?.date).year().toString() nitem.year = dayjs(item.activities_id?.date).year().toString()
//nitem.month = nitem.year + " > " + (dayjs(item.activities_id?.date).month() + 1).toString().padStart(2, "0") nitem.month = (dayjs(item.activities_id?.date).month() + 1).toString().padStart(2, "0")
// nitem.menu_country = nitem.country // nitem.menu_country = nitem.country
// nitem.menu_state = nitem.country + " > " + nitem.state // nitem.menu_state = nitem.country + " > " + nitem.state
@ -199,16 +204,22 @@ async function generateJson( type ) {
} }
if (type == 'conferences_translations') { if (type == 'conferences_translations') {
nitem.language = LOCALE;
//nitem.id = item.conferences_id?.id.toString() //if( !item.conferences_id?.activity ) return false
//nitem.type = 'conferences' if( !item.title && !item.conferences_id?.title ) return false
nitem.title = item.title
nitem.locale = LOCALE;
nitem.code = dayjs(item.conferences_id?.date).format('YYYYMMDD') + '-' + (item.conferences_id?.activity ?? 0),
nitem.id = `${nitem.locale}-${item.id}`
nitem.type = 'conferences'
nitem.title = item.title ? item.title : item.conferences_id?.title
//nitem.body = he.decode(striptags(item.text_published)) || '' //nitem.body = he.decode(striptags(item.text_published)) || ''
//nitem.title = item.title //nitem.title = item.title
nitem.body = item.html
nitem.timestamp = dayjs(item.conferences_id?.date).unix() nitem.timestamp = dayjs(item.conferences_id?.date).unix()
nitem.date = ''; nitem.date = '';
nitem.activity = item.conferences_id?.activity; nitem.activity = item.conferences_id?.activity ?? 0;
nitem.duration = item.conferences_id?.duration ?? 0; nitem.duration = item.conferences_id?.duration ?? 0;
//nitem.body = he.decode(striptags(item.interventions[0]?.text)) || '' //nitem.body = he.decode(striptags(item.interventions[0]?.text)) || ''
@ -217,19 +228,19 @@ async function generateJson( type ) {
nitem.state = item.conferences_id?.state || null; nitem.state = item.conferences_id?.state || null;
nitem.country = item.conferences_id?.country || null; nitem.country = item.conferences_id?.country || null;
nitem.body = (item.text_published?true:false); //nitem.body = (item.text_published?true:false);
nitem.thumbnail = item.conferences_id?.thumbnail?.filename_disk nitem.thumbnail = item.conferences_id?.thumbnail?.filename_disk
//nitem.files = {} nitem.files = {}
nitem.youtube = item.youtube nitem.files.youtube = item.youtube
nitem.video = item.privateVideo?.filename_disk nitem.files.video = item.privateVideo?.filename_disk
nitem.audio = item.mp3?.filename_disk nitem.files.audio = item.mp3?.filename_disk
nitem.booklet = item.pdf_booklet?.filename_disk nitem.files.booklet = item.pdf_booklet?.filename_disk
nitem.simple = item.pdf?.filename_disk nitem.files.simple = item.pdf?.filename_disk
nitem.directus = item.conferences_id?.id.toString() nitem.directus = item.conferences_id?.id.toString()
nitem.wp = ''; nitem.wp = item.conferences_id?.wpid;
nitem.typesense = true; nitem.typesense = true;
nitem.private = item.conferences_id?.public == 0 ? true : false; nitem.private = item.conferences_id?.public == 0 ? true : false;
@ -244,12 +255,12 @@ async function generateJson( type ) {
//writeFile(jsonlData, type) //writeFile(jsonlData, type)
console.log() console.log()
if( nitems.length > 0 ){ if( nitems.length > 0 ){
let csv = parse(nitems); //let csv = parse(nitems);
jsonlData += csv //jsonlData += csv
//let csv = nitems.map(row => fields.map(fieldName => JSON.stringify(row[fieldName], replacer)).join(',')); //let csv = nitems.map(row => fields.map(fieldName => JSON.stringify(row[fieldName], replacer)).join(','));
//return csv; //return csv;
//jsonlData += nitems.filter(item => item !== false).map(JSON.stringify).join('\n') jsonlData += nitems.filter(item => item !== false).map(JSON.stringify).join('\n')
jsonlData += '\n';
//console.log( csv ); //console.log( csv );
//console.log( documents.length + " documents to write for year " + year + " and type " + type ); //console.log( documents.length + " documents to write for year " + year + " and type " + type );
writeFile(jsonlData, type) writeFile(jsonlData, type)
@ -260,7 +271,7 @@ async function generateJson( type ) {
} }
function writeFile(jsonlData, type) { function writeFile(jsonlData, type) {
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.csv`, jsonlData, (err) => { fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.json`, jsonlData, (err) => {
if (err) { if (err) {
console.error("Error writing file:", err); console.error("Error writing file:", err);
return; return;
@ -269,6 +280,18 @@ function writeFile(jsonlData, type) {
}); });
} }
function slugify(str) {
if(!str) return 'no-title'
return str
.toLowerCase() // Convert to lowercase
.trim() // Remove leading/trailing whitespace
.normalize('NFD') // Split accented characters into base letters and diacritics
.replace(/[\u0300-\u036f]/g, '') // Remove diacritics (accents)
.replace(/[^a-z0-9\s-]/g, '') // Remove non-alphanumeric characters (except spaces and hyphens)
.replace(/[\s-]+/g, '-') // Replace multiple spaces or hyphens with a single hyphen
.replace(/^-+|-+$/g, ''); // Remove leading or trailing hyphens
}
//generateJson( 'conferences', 1974); //generateJson( 'conferences', 1974);
//generateJson( 'activities' , 2021 ); //generateJson( 'activities' , 2021 );

44
carpa_json_to_markdown/index_files.js
View File

@ -178,29 +178,30 @@ async function generateFiles( type ) {
let nitem = {} let nitem = {}
if (type == 'activities_translations') { if (type == 'activities_translations') {
nitem.locale = item.languages_code //nitem.locale = item.languages_code
nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}` //nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}`
nitem.id = item.activities_id?.id nitem.id = `${item.languages_code}-${item.activities_id?.wpid}`
nitem.title = item.title /*nitem.title = item.title
nitem.activities_title = item.activities_id?.title nitem.activities_title = item.activities_id?.title
nitem.date = item.activities_id?.date; nitem.date = item.activities_id?.date;
nitem.activity = item.activities_id?.activity; nitem.activity = item.activities_id?.activity;*/
nitem.thumbnail = item.activities_id?.thumbnail?.filename_download //nitem.thumbnail = item.activities_id?.thumbnail?.filename_download
nitem.thumbnail_file = item.activities_id?.thumbnail?.filename_disk //nitem.thumbnail_file = item.activities_id?.thumbnail?.filename_disk
nitem.thumbnail_id = item.activities_id?.thumbnail?.id nitem.thumbnail_id = item.activities_id?.thumbnail?.filename_disk
nitem.video = item.privateVideo?.filename_download //nitem.video = item.privateVideo?.filename_download
nitem.video_file = item.privateVideo?.filename_disk //nitem.video_file = item.privateVideo?.filename_disk
nitem.video_id = item.privateVideo?.id nitem.files = {}
nitem.audio = item.mp3?.filename_download nitem.files.video = item.privateVideo?.filename_disk
nitem.audio_file = item.mp3?.filename_disk //nitem.audio = item.mp3?.filename_download
nitem.audio_id = item.mp3?.id //nitem.audio_file = item.mp3?.filename_disk
nitem.booklet = item.pdf_booklet?.filename_download nitem.files.audio = item.mp3?.filename_disk
nitem.booklet_file = item.pdf_booklet?.filename_disk //nitem.booklet = item.pdf_booklet?.filename_download
nitem.booklet_id = item.pdf_booklet?.id //nitem.booklet_file = item.pdf_booklet?.filename_disk
nitem.simple = item.pdf?.filename_download nitem.files.booklet = item.pdf_booklet?.filename_disk
nitem.simple_file = item.pdf?.filename_disk //nitem.simple = item.pdf?.filename_download
nitem.simple_id = item.pdf?.id //nitem.simple_file = item.pdf?.filename_disk
nitem.files.simple = item.pdf?.filename_disk
} }
if (type == 'conferences_translations') { if (type == 'conferences_translations') {
@ -283,7 +284,8 @@ async function generateFiles( type ) {
} }
if(type=='materiales_didacticos_translations'){ if(type=='materiales_didacticos_translations'){
nitem.id = item.title nitem.id = item.id.toString()
nitem.title = item.title
nitem.locale = item.languages_code nitem.locale = item.languages_code
nitem.featured_image = item.featured_image?.filename_disk nitem.featured_image = item.featured_image?.filename_disk
nitem.materiales_didacticos_id = item.materiales_didacticos_id?.id nitem.materiales_didacticos_id = item.materiales_didacticos_id?.id

15
carpa_json_to_markdown/index_wp.js
View File

@ -98,7 +98,7 @@ async function generateJson( type, year ) {
url = `https://actividadeswp.carpa.com/v3/actividades/?f=texts&locale=${LOCALE}&year=${year}&limit=1000`; url = `https://actividadeswp.carpa.com/v3/actividades/?f=texts&locale=${LOCALE}&year=${year}&limit=1000`;
} else { } else {
//url = `https://conferenciaswp.carpa.com/v3/conferencias/?f=algolia&locale=${LOCALE}&year=${year}&limit=1000&cache=false`; //url = `https://conferenciaswp.carpa.com/v3/conferencias/?f=algolia&locale=${LOCALE}&year=${year}&limit=1000&cache=false`;
url = `http://localhost:10018/v3/conferencias/?f=typesense&locale=${LOCALE}&year=${year}`; url = `http://conferences.local/v3/conferencias/?f=typesense&locale=${LOCALE}&year=${year}`;
} }
request(url, options, async (error, res, body) => { request(url, options, async (error, res, body) => {
@ -112,13 +112,14 @@ async function generateJson( type, year ) {
// do something with JSON, using the 'body' variable // do something with JSON, using the 'body' variable
let items = [] let items = []
console.log( body.length + " items found for year " + year + " and type " + type ); console.log( body.result.conferencias.length + " items found for year " + year + " and type " + type );
console.log( 'URL', url )
items = body; items = body.result.conferencias;
if(items.length > 0) { if(items.length > 0) {
items.map((item) => { items.map((item) => {
if( item.id == null || item.id == undefined || item.id == '' ) { if( item._id == null || item._id == undefined || item._id == '' ) {
console.log( "Item has no id, skipping." ); console.log( "Item has no id, skipping." );
return false; return false;
}}) }})
@ -167,7 +168,7 @@ async function generateJson( type, year ) {
if (type == 'conferences') { if (type == 'conferences') {
nitem.locale = LOCALE; nitem.locale = LOCALE;
nitem.id = item.id.toString() nitem.id = item._id.toString()
nitem.type = 'conferences' nitem.type = 'conferences'
nitem.title = item.title nitem.title = item.title
nitem.timestamp = item.timestamp nitem.timestamp = item.timestamp
@ -180,7 +181,7 @@ async function generateJson( type, year ) {
nitem.city = item.city || null; nitem.city = item.city || null;
nitem.state = item.state || null; nitem.state = item.state || null;
nitem.country = item.country || null; nitem.country = item.country || null;
nitem.body = item.body; //nitem.body = item.body;
nitem.thumbnail = item.thumbnail nitem.thumbnail = item.thumbnail
nitem.youtube = item.files?.youtube nitem.youtube = item.files?.youtube
nitem.video = item.files?.video nitem.video = item.files?.video
@ -188,7 +189,7 @@ async function generateJson( type, year ) {
nitem.booklet = item.files?.pdf nitem.booklet = item.files?.pdf
nitem.simple = item.files?.pdf_simple nitem.simple = item.files?.pdf_simple
nitem.directus = ''; nitem.directus = '';
nitem.wp = item.id.toString(); nitem.wp = item._id.toString();
nitem.typesense = true; nitem.typesense = true;
nitem.rm = item.rm; nitem.rm = item.rm;
nitem.private = item.private == 1 ? true : false; nitem.private = item.private == 1 ? true : false;