LGCC
/
api
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed bulk for files

This commit is contained in:
Julio Ruiz 2026-07-11 06:56:42 -05:00
parent 8cf9a8052d
commit 550ede702c
4 changed files with 116 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
carpa_json_to_markdown/document_update.mjs
View File

@ -35,24 +35,25 @@ async function processJsonLines(filePath) {
// Parse the line as JSON
const data = JSON.parse(line);
//if( data.id != 62643 ) continue; // Skip until we find the specific ID
//if( data.id != 'es-2956' ) continue; // Skip until we find the specific ID
console.log('Processing: ', data.title);
const doc = await createDocument(data);
const doc_id = doc.id
const doc_id = await createDocument(data);
console.log('Processing: ', doc.code);
console.log('Document created with ID:', doc_id);
if ('body' in data && typeof data.body === 'string' && hasParagraphs(data.body).length > 0 && data.body.length > 0) {
// Split the body into paragraphs using <p> tags
const paragraphs = data.body.split(/<\/?p>/).map(p => p.trim()).filter(Boolean);
const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type);
//console.log('Total paragraphs found:', paragraphs);
const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type, doc.code);
//console.log('Total BODY paragraphs found:', paragraphs);
} else {
if( data.body !== undefined ){
const paragraphs = data.body.split('\n\n').map(p => p.trim()).filter(Boolean);
const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type);
//console.log('Total paragraphs found:', paragraphs);
const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type, doc.code);
//console.log('Total NO BODY paragraphs found:', paragraphs);
}
}
} catch (error) {
@ -75,11 +76,11 @@ function hasParagraphs(htmlString) {
return dom.window.document.querySelectorAll('p');
}
//processJsonLines('./input/activities_wp_ES.json');
processJsonLines('./input/conferences_wp_ES.json');
processJsonLines('./input/activities_translations_PT.json');
//processJsonLines('./input/conferences_wp_ES.json');
// Function to create paragraphs in Typesense
async function createParagraphs(documentId, paragraphs, locale, type) {
async function createParagraphs(documentId, paragraphs, locale, type, code) {
let lineCount = 0;
let jsonl = '';
for (const para of paragraphs) {
@ -96,21 +97,25 @@ async function createParagraphs(documentId, paragraphs, locale, type) {
}
let paragraph = {
document_id: documentId,
raw: fixedHtml,
staging_activities_id: documentId,
html: fixedHtml,
text: fixedHtml.replace(/<[^>]+>/g, ''), // Remove HTML tags for search
number: lineCount + 1,
locale: locale,
type: type,
id: locale + '-' + documentId + '-' + (lineCount + 1) // Unique ID for the paragraph
code: code,
id: documentId + '-' + (lineCount + 1) // Unique ID for the paragraph
};
//console.log( code );
jsonl += JSON.stringify(paragraph) + '\n';
lineCount++;
}
let doc = await client.collections('paragraphs').documents().import(jsonl, {action: 'upsert'})
let doc = await client.collections('staging_activities_paragraphs').documents().import(jsonl, {action: 'upsert'})
return doc; // Return the number of paragraphs created
}
@ -134,14 +139,14 @@ async function createDocument(data){
draft: data.draft,
thumbnail: data.thumbnail,
files: {
youtube: data.youtube,
video: data.video || null,
audio: data.audio || null,
booklet: data.booklet || null,
simple: data.simple || null,
youtube: data.files.youtube,
video: data.files.video || null,
audio: data.files.audio || null,
booklet: data.files.booklet || null,
simple: data.files.simple || null,
},
directus: "",
wp: data.id.toString(),
directus: data.id.toString(),
wp: data.wp.toString(),
rm: data.rm,
private: data.private || false,
slug: data.slug || null,
@ -150,8 +155,8 @@ async function createDocument(data){
month: data.month,
};
let doc = await client.collections('documents').documents().upsert(document);
let doc = await client.collections('staging_activities').documents().upsert(document);
return doc.id;
return doc;
}

91
carpa_json_to_markdown/index_directus.js
View File

@ -94,6 +94,7 @@ async function generateJson( type ) {
let fields = [];
fields['activities_translations'] = [
'id',
'activities_id.id',
'activities_id.wpid',
'title',
@ -104,6 +105,7 @@ async function generateJson( type ) {
'activities_id.state',
'activities_id.country',
'activities_id.duration',
'interventions.title',
'interventions.text',
'activities_id.private',
'activities_id.thumbnail.filename_disk',
@ -116,8 +118,10 @@ async function generateJson( type ) {
]
fields['conferences_translations'] = [
'id',
'conferences_id.id',
'title',
'conferences_id.title',
'conferences_id.date',
'conferences_id.activity',
'conferences_id.place',
@ -151,41 +155,42 @@ async function generateJson( type ) {
let nitem = {}
if (type == 'activities_translations') {
nitem.language = LOCALE;
//nitem.id = item.activities_id?.id
//nitem.type = 'activities'
nitem.title = item.title
nitem.locale = LOCALE;
nitem.id = `${nitem.locale}-${item.id}`
nitem.title = item.title ? item.title : item.interventions[0]?.title
nitem.type = 'activities'
nitem.timestamp = dayjs(item.activities_id?.date).unix()
nitem.date = '';
nitem.date = dayjs(item.activities_id?.date).format('YYYY-MM-DD')
nitem.activity = item.activities_id?.activity;
nitem.duration = item.activities_id?.duration ?? 0;
//nitem.body = he.decode(striptags(item.interventions[0]?.text)) || ''
nitem.body = item.interventions[0]?.text;
nitem.place = item.activities_id?.place || null;
nitem.city = item.activities_id?.city || null;
nitem.state = item.activities_id?.state || null;
nitem.country = item.activities_id?.country || null;
nitem.body = (item.interventions[0]?.text?true:false);
//nitem.body = (item.interventions[0]?.text?true:false);
nitem.thumbnail = item.activities_id?.thumbnail?.filename_disk
//nitem.files = {}
nitem.youtube = item.youtube
nitem.video = item.privateVideo?.filename_disk
nitem.audio = item.mp3?.filename_disk
nitem.booklet = item.pdf_booklet?.filename_disk
nitem.simple = item.pdf?.filename_disk
nitem.files = {}
nitem.files.youtube = item.youtube
nitem.files.video = item.privateVideo?.filename_disk
nitem.files.audio = item.mp3?.filename_disk
nitem.files.booklet = item.pdf_booklet?.filename_disk
nitem.files.simple = item.pdf?.filename_disk
nitem.directus = item.activities_id?.id;
nitem.wp = item.activities_id?.wpid;
nitem.typesense = true;
nitem.private = item.activities_id?.private == 1 ? true : false;
nitem.slug = item.slug
nitem.slug = slugify(nitem.title)
//nitem.year = dayjs(item.activities_id?.date).year().toString()
//nitem.month = nitem.year + " > " + (dayjs(item.activities_id?.date).month() + 1).toString().padStart(2, "0")
nitem.year = dayjs(item.activities_id?.date).year().toString()
nitem.month = (dayjs(item.activities_id?.date).month() + 1).toString().padStart(2, "0")
// nitem.menu_country = nitem.country
// nitem.menu_state = nitem.country + " > " + nitem.state
@ -199,16 +204,22 @@ async function generateJson( type ) {
}
if (type == 'conferences_translations') {
nitem.language = LOCALE;
//nitem.id = item.conferences_id?.id.toString()
//nitem.type = 'conferences'
nitem.title = item.title
//if( !item.conferences_id?.activity ) return false
if( !item.title && !item.conferences_id?.title ) return false
nitem.locale = LOCALE;
nitem.code = dayjs(item.conferences_id?.date).format('YYYYMMDD') + '-' + (item.conferences_id?.activity ?? 0),
nitem.id = `${nitem.locale}-${item.id}`
nitem.type = 'conferences'
nitem.title = item.title ? item.title : item.conferences_id?.title
//nitem.body = he.decode(striptags(item.text_published)) || ''
//nitem.title = item.title
nitem.body = item.html
nitem.timestamp = dayjs(item.conferences_id?.date).unix()
nitem.date = '';
nitem.activity = item.conferences_id?.activity;
nitem.activity = item.conferences_id?.activity ?? 0;
nitem.duration = item.conferences_id?.duration ?? 0;
//nitem.body = he.decode(striptags(item.interventions[0]?.text)) || ''
@ -217,19 +228,19 @@ async function generateJson( type ) {
nitem.state = item.conferences_id?.state || null;
nitem.country = item.conferences_id?.country || null;
nitem.body = (item.text_published?true:false);
//nitem.body = (item.text_published?true:false);
nitem.thumbnail = item.conferences_id?.thumbnail?.filename_disk
//nitem.files = {}
nitem.youtube = item.youtube
nitem.video = item.privateVideo?.filename_disk
nitem.audio = item.mp3?.filename_disk
nitem.booklet = item.pdf_booklet?.filename_disk
nitem.simple = item.pdf?.filename_disk
nitem.files = {}
nitem.files.youtube = item.youtube
nitem.files.video = item.privateVideo?.filename_disk
nitem.files.audio = item.mp3?.filename_disk
nitem.files.booklet = item.pdf_booklet?.filename_disk
nitem.files.simple = item.pdf?.filename_disk
nitem.directus = item.conferences_id?.id.toString()
nitem.wp = '';
nitem.wp = item.conferences_id?.wpid;
nitem.typesense = true;
nitem.private = item.conferences_id?.public == 0 ? true : false;
@ -244,12 +255,12 @@ async function generateJson( type ) {
//writeFile(jsonlData, type)
console.log()
if( nitems.length > 0 ){
let csv = parse(nitems);
jsonlData += csv
//let csv = parse(nitems);
//jsonlData += csv
//let csv = nitems.map(row => fields.map(fieldName => JSON.stringify(row[fieldName], replacer)).join(','));
//return csv;
//jsonlData += nitems.filter(item => item !== false).map(JSON.stringify).join('\n')
jsonlData += nitems.filter(item => item !== false).map(JSON.stringify).join('\n')
jsonlData += '\n';
//console.log( csv );
//console.log( documents.length + " documents to write for year " + year + " and type " + type );
writeFile(jsonlData, type)
@ -260,7 +271,7 @@ async function generateJson( type ) {
}
function writeFile(jsonlData, type) {
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.csv`, jsonlData, (err) => {
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.json`, jsonlData, (err) => {
if (err) {
console.error("Error writing file:", err);
return;
@ -269,6 +280,18 @@ function writeFile(jsonlData, type) {
});
}
function slugify(str) {
if(!str) return 'no-title'
return str
.toLowerCase() // Convert to lowercase
.trim() // Remove leading/trailing whitespace
.normalize('NFD') // Split accented characters into base letters and diacritics
.replace(/[\u0300-\u036f]/g, '') // Remove diacritics (accents)
.replace(/[^a-z0-9\s-]/g, '') // Remove non-alphanumeric characters (except spaces and hyphens)
.replace(/[\s-]+/g, '-') // Replace multiple spaces or hyphens with a single hyphen
.replace(/^-+|-+$/g, ''); // Remove leading or trailing hyphens
}
//generateJson( 'conferences', 1974);
//generateJson( 'activities' , 2021 );

44
carpa_json_to_markdown/index_files.js
View File

@ -178,29 +178,30 @@ async function generateFiles( type ) {
let nitem = {}
if (type == 'activities_translations') {
nitem.locale = item.languages_code
nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}`
nitem.id = item.activities_id?.id
nitem.title = item.title
//nitem.locale = item.languages_code
//nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}`
nitem.id = `${item.languages_code}-${item.activities_id?.wpid}`
/*nitem.title = item.title
nitem.activities_title = item.activities_id?.title
nitem.date = item.activities_id?.date;
nitem.activity = item.activities_id?.activity;
nitem.activity = item.activities_id?.activity;*/
nitem.thumbnail = item.activities_id?.thumbnail?.filename_download
nitem.thumbnail_file = item.activities_id?.thumbnail?.filename_disk
nitem.thumbnail_id = item.activities_id?.thumbnail?.id
nitem.video = item.privateVideo?.filename_download
nitem.video_file = item.privateVideo?.filename_disk
nitem.video_id = item.privateVideo?.id
nitem.audio = item.mp3?.filename_download
nitem.audio_file = item.mp3?.filename_disk
nitem.audio_id = item.mp3?.id
nitem.booklet = item.pdf_booklet?.filename_download
nitem.booklet_file = item.pdf_booklet?.filename_disk
nitem.booklet_id = item.pdf_booklet?.id
nitem.simple = item.pdf?.filename_download
nitem.simple_file = item.pdf?.filename_disk
nitem.simple_id = item.pdf?.id
//nitem.thumbnail = item.activities_id?.thumbnail?.filename_download
//nitem.thumbnail_file = item.activities_id?.thumbnail?.filename_disk
nitem.thumbnail_id = item.activities_id?.thumbnail?.filename_disk
//nitem.video = item.privateVideo?.filename_download
//nitem.video_file = item.privateVideo?.filename_disk
nitem.files = {}
nitem.files.video = item.privateVideo?.filename_disk
//nitem.audio = item.mp3?.filename_download
//nitem.audio_file = item.mp3?.filename_disk
nitem.files.audio = item.mp3?.filename_disk
//nitem.booklet = item.pdf_booklet?.filename_download
//nitem.booklet_file = item.pdf_booklet?.filename_disk
nitem.files.booklet = item.pdf_booklet?.filename_disk
//nitem.simple = item.pdf?.filename_download
//nitem.simple_file = item.pdf?.filename_disk
nitem.files.simple = item.pdf?.filename_disk
}
if (type == 'conferences_translations') {
@ -283,7 +284,8 @@ async function generateFiles( type ) {
}
if(type=='materiales_didacticos_translations'){
nitem.id = item.title
nitem.id = item.id.toString()
nitem.title = item.title
nitem.locale = item.languages_code
nitem.featured_image = item.featured_image?.filename_disk
nitem.materiales_didacticos_id = item.materiales_didacticos_id?.id

15
carpa_json_to_markdown/index_wp.js
View File

@ -98,7 +98,7 @@ async function generateJson( type, year ) {
url = `https://actividadeswp.carpa.com/v3/actividades/?f=texts&locale=${LOCALE}&year=${year}&limit=1000`;
} else {
//url = `https://conferenciaswp.carpa.com/v3/conferencias/?f=algolia&locale=${LOCALE}&year=${year}&limit=1000&cache=false`;
url = `http://localhost:10018/v3/conferencias/?f=typesense&locale=${LOCALE}&year=${year}`;
url = `http://conferences.local/v3/conferencias/?f=typesense&locale=${LOCALE}&year=${year}`;
}
request(url, options, async (error, res, body) => {
@ -112,13 +112,14 @@ async function generateJson( type, year ) {
// do something with JSON, using the 'body' variable
let items = []
console.log( body.length + " items found for year " + year + " and type " + type );
console.log( body.result.conferencias.length + " items found for year " + year + " and type " + type );
console.log( 'URL', url )
items = body;
items = body.result.conferencias;
if(items.length > 0) {
items.map((item) => {
if( item.id == null || item.id == undefined || item.id == '' ) {
if( item._id == null || item._id == undefined || item._id == '' ) {
console.log( "Item has no id, skipping." );
return false;
}})
@ -167,7 +168,7 @@ async function generateJson( type, year ) {
if (type == 'conferences') {
nitem.locale = LOCALE;
nitem.id = item.id.toString()
nitem.id = item._id.toString()
nitem.type = 'conferences'
nitem.title = item.title
nitem.timestamp = item.timestamp
@ -180,7 +181,7 @@ async function generateJson( type, year ) {
nitem.city = item.city || null;
nitem.state = item.state || null;
nitem.country = item.country || null;
nitem.body = item.body;
//nitem.body = item.body;
nitem.thumbnail = item.thumbnail
nitem.youtube = item.files?.youtube
nitem.video = item.files?.video
@ -188,7 +189,7 @@ async function generateJson( type, year ) {
nitem.booklet = item.files?.pdf
nitem.simple = item.files?.pdf_simple
nitem.directus = '';
nitem.wp = item.id.toString();
nitem.wp = item._id.toString();
nitem.typesense = true;
nitem.rm = item.rm;
nitem.private = item.private == 1 ? true : false;