diff --git a/carpa_json_to_markdown/document_update.mjs b/carpa_json_to_markdown/document_update.mjs index fdcf6c9..03f7c5a 100644 --- a/carpa_json_to_markdown/document_update.mjs +++ b/carpa_json_to_markdown/document_update.mjs @@ -35,24 +35,25 @@ async function processJsonLines(filePath) { // Parse the line as JSON const data = JSON.parse(line); - //if( data.id != 62643 ) continue; // Skip until we find the specific ID + //if( data.id != 'es-2956' ) continue; // Skip until we find the specific ID - console.log('Processing: ', data.title); + const doc = await createDocument(data); + const doc_id = doc.id - const doc_id = await createDocument(data); + console.log('Processing: ', doc.code); console.log('Document created with ID:', doc_id); if ('body' in data && typeof data.body === 'string' && hasParagraphs(data.body).length > 0 && data.body.length > 0) { // Split the body into paragraphs using
tags const paragraphs = data.body.split(/<\/?p>/).map(p => p.trim()).filter(Boolean); - const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type); - //console.log('Total paragraphs found:', paragraphs); + const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type, doc.code); + //console.log('Total BODY paragraphs found:', paragraphs); } else { if( data.body !== undefined ){ const paragraphs = data.body.split('\n\n').map(p => p.trim()).filter(Boolean); - const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type); - //console.log('Total paragraphs found:', paragraphs); + const totalParagraphs = await createParagraphs(doc_id, paragraphs, data.locale, data.type, doc.code); + //console.log('Total NO BODY paragraphs found:', paragraphs); } } } catch (error) { @@ -75,11 +76,11 @@ function hasParagraphs(htmlString) { return dom.window.document.querySelectorAll('p'); } -//processJsonLines('./input/activities_wp_ES.json'); -processJsonLines('./input/conferences_wp_ES.json'); +processJsonLines('./input/activities_translations_PT.json'); +//processJsonLines('./input/conferences_wp_ES.json'); // Function to create paragraphs in Typesense -async function createParagraphs(documentId, paragraphs, locale, type) { +async function createParagraphs(documentId, paragraphs, locale, type, code) { let lineCount = 0; let jsonl = ''; for (const para of paragraphs) { @@ -96,21 +97,25 @@ async function createParagraphs(documentId, paragraphs, locale, type) { } let paragraph = { - document_id: documentId, - raw: fixedHtml, + staging_activities_id: documentId, + html: fixedHtml, text: fixedHtml.replace(/<[^>]+>/g, ''), // Remove HTML tags for search number: lineCount + 1, locale: locale, type: type, - id: locale + '-' + documentId + '-' + (lineCount + 1) // Unique ID for the paragraph + code: code, + id: documentId + '-' + (lineCount + 1) // Unique ID for the paragraph }; + //console.log( code ); + jsonl += JSON.stringify(paragraph) + '\n'; lineCount++; } - let doc = await client.collections('paragraphs').documents().import(jsonl, {action: 'upsert'}) + let doc = await client.collections('staging_activities_paragraphs').documents().import(jsonl, {action: 'upsert'}) + return doc; // Return the number of paragraphs created } @@ -134,14 +139,14 @@ async function createDocument(data){ draft: data.draft, thumbnail: data.thumbnail, files: { - youtube: data.youtube, - video: data.video || null, - audio: data.audio || null, - booklet: data.booklet || null, - simple: data.simple || null, + youtube: data.files.youtube, + video: data.files.video || null, + audio: data.files.audio || null, + booklet: data.files.booklet || null, + simple: data.files.simple || null, }, - directus: "", - wp: data.id.toString(), + directus: data.id.toString(), + wp: data.wp.toString(), rm: data.rm, private: data.private || false, slug: data.slug || null, @@ -150,8 +155,8 @@ async function createDocument(data){ month: data.month, }; - let doc = await client.collections('documents').documents().upsert(document); + let doc = await client.collections('staging_activities').documents().upsert(document); - return doc.id; + return doc; } \ No newline at end of file diff --git a/carpa_json_to_markdown/index_directus.js b/carpa_json_to_markdown/index_directus.js index 3183d14..2882486 100644 --- a/carpa_json_to_markdown/index_directus.js +++ b/carpa_json_to_markdown/index_directus.js @@ -94,6 +94,7 @@ async function generateJson( type ) { let fields = []; fields['activities_translations'] = [ + 'id', 'activities_id.id', 'activities_id.wpid', 'title', @@ -104,6 +105,7 @@ async function generateJson( type ) { 'activities_id.state', 'activities_id.country', 'activities_id.duration', + 'interventions.title', 'interventions.text', 'activities_id.private', 'activities_id.thumbnail.filename_disk', @@ -116,8 +118,10 @@ async function generateJson( type ) { ] fields['conferences_translations'] = [ + 'id', 'conferences_id.id', 'title', + 'conferences_id.title', 'conferences_id.date', 'conferences_id.activity', 'conferences_id.place', @@ -151,41 +155,42 @@ async function generateJson( type ) { let nitem = {} if (type == 'activities_translations') { - nitem.language = LOCALE; - //nitem.id = item.activities_id?.id - //nitem.type = 'activities' - nitem.title = item.title + + nitem.locale = LOCALE; + nitem.id = `${nitem.locale}-${item.id}` + nitem.title = item.title ? item.title : item.interventions[0]?.title + nitem.type = 'activities' nitem.timestamp = dayjs(item.activities_id?.date).unix() - nitem.date = ''; + nitem.date = dayjs(item.activities_id?.date).format('YYYY-MM-DD') nitem.activity = item.activities_id?.activity; nitem.duration = item.activities_id?.duration ?? 0; - //nitem.body = he.decode(striptags(item.interventions[0]?.text)) || '' + nitem.body = item.interventions[0]?.text; nitem.place = item.activities_id?.place || null; nitem.city = item.activities_id?.city || null; nitem.state = item.activities_id?.state || null; nitem.country = item.activities_id?.country || null; - nitem.body = (item.interventions[0]?.text?true:false); + //nitem.body = (item.interventions[0]?.text?true:false); nitem.thumbnail = item.activities_id?.thumbnail?.filename_disk - //nitem.files = {} - nitem.youtube = item.youtube - nitem.video = item.privateVideo?.filename_disk - nitem.audio = item.mp3?.filename_disk - nitem.booklet = item.pdf_booklet?.filename_disk - nitem.simple = item.pdf?.filename_disk + nitem.files = {} + nitem.files.youtube = item.youtube + nitem.files.video = item.privateVideo?.filename_disk + nitem.files.audio = item.mp3?.filename_disk + nitem.files.booklet = item.pdf_booklet?.filename_disk + nitem.files.simple = item.pdf?.filename_disk nitem.directus = item.activities_id?.id; nitem.wp = item.activities_id?.wpid; nitem.typesense = true; nitem.private = item.activities_id?.private == 1 ? true : false; - nitem.slug = item.slug + nitem.slug = slugify(nitem.title) - //nitem.year = dayjs(item.activities_id?.date).year().toString() - //nitem.month = nitem.year + " > " + (dayjs(item.activities_id?.date).month() + 1).toString().padStart(2, "0") + nitem.year = dayjs(item.activities_id?.date).year().toString() + nitem.month = (dayjs(item.activities_id?.date).month() + 1).toString().padStart(2, "0") // nitem.menu_country = nitem.country // nitem.menu_state = nitem.country + " > " + nitem.state @@ -199,16 +204,22 @@ async function generateJson( type ) { } if (type == 'conferences_translations') { - nitem.language = LOCALE; - //nitem.id = item.conferences_id?.id.toString() - //nitem.type = 'conferences' - nitem.title = item.title + + //if( !item.conferences_id?.activity ) return false + if( !item.title && !item.conferences_id?.title ) return false + + nitem.locale = LOCALE; + nitem.code = dayjs(item.conferences_id?.date).format('YYYYMMDD') + '-' + (item.conferences_id?.activity ?? 0), + nitem.id = `${nitem.locale}-${item.id}` + nitem.type = 'conferences' + nitem.title = item.title ? item.title : item.conferences_id?.title //nitem.body = he.decode(striptags(item.text_published)) || '' //nitem.title = item.title + nitem.body = item.html nitem.timestamp = dayjs(item.conferences_id?.date).unix() nitem.date = ''; - nitem.activity = item.conferences_id?.activity; + nitem.activity = item.conferences_id?.activity ?? 0; nitem.duration = item.conferences_id?.duration ?? 0; //nitem.body = he.decode(striptags(item.interventions[0]?.text)) || '' @@ -217,19 +228,19 @@ async function generateJson( type ) { nitem.state = item.conferences_id?.state || null; nitem.country = item.conferences_id?.country || null; - nitem.body = (item.text_published?true:false); + //nitem.body = (item.text_published?true:false); nitem.thumbnail = item.conferences_id?.thumbnail?.filename_disk - //nitem.files = {} - nitem.youtube = item.youtube - nitem.video = item.privateVideo?.filename_disk - nitem.audio = item.mp3?.filename_disk - nitem.booklet = item.pdf_booklet?.filename_disk - nitem.simple = item.pdf?.filename_disk + nitem.files = {} + nitem.files.youtube = item.youtube + nitem.files.video = item.privateVideo?.filename_disk + nitem.files.audio = item.mp3?.filename_disk + nitem.files.booklet = item.pdf_booklet?.filename_disk + nitem.files.simple = item.pdf?.filename_disk nitem.directus = item.conferences_id?.id.toString() - nitem.wp = ''; + nitem.wp = item.conferences_id?.wpid; nitem.typesense = true; nitem.private = item.conferences_id?.public == 0 ? true : false; @@ -244,12 +255,12 @@ async function generateJson( type ) { //writeFile(jsonlData, type) console.log() if( nitems.length > 0 ){ - let csv = parse(nitems); - jsonlData += csv + //let csv = parse(nitems); + //jsonlData += csv //let csv = nitems.map(row => fields.map(fieldName => JSON.stringify(row[fieldName], replacer)).join(',')); //return csv; - //jsonlData += nitems.filter(item => item !== false).map(JSON.stringify).join('\n') - + jsonlData += nitems.filter(item => item !== false).map(JSON.stringify).join('\n') + jsonlData += '\n'; //console.log( csv ); //console.log( documents.length + " documents to write for year " + year + " and type " + type ); writeFile(jsonlData, type) @@ -260,7 +271,7 @@ async function generateJson( type ) { } function writeFile(jsonlData, type) { - fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.csv`, jsonlData, (err) => { + fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.json`, jsonlData, (err) => { if (err) { console.error("Error writing file:", err); return; @@ -269,6 +280,18 @@ function writeFile(jsonlData, type) { }); } +function slugify(str) { + if(!str) return 'no-title' + return str + .toLowerCase() // Convert to lowercase + .trim() // Remove leading/trailing whitespace + .normalize('NFD') // Split accented characters into base letters and diacritics + .replace(/[\u0300-\u036f]/g, '') // Remove diacritics (accents) + .replace(/[^a-z0-9\s-]/g, '') // Remove non-alphanumeric characters (except spaces and hyphens) + .replace(/[\s-]+/g, '-') // Replace multiple spaces or hyphens with a single hyphen + .replace(/^-+|-+$/g, ''); // Remove leading or trailing hyphens +} + //generateJson( 'conferences', 1974); //generateJson( 'activities' , 2021 ); diff --git a/carpa_json_to_markdown/index_files.js b/carpa_json_to_markdown/index_files.js index 299c052..4bb8752 100644 --- a/carpa_json_to_markdown/index_files.js +++ b/carpa_json_to_markdown/index_files.js @@ -178,29 +178,30 @@ async function generateFiles( type ) { let nitem = {} if (type == 'activities_translations') { - nitem.locale = item.languages_code - nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}` - nitem.id = item.activities_id?.id - nitem.title = item.title + //nitem.locale = item.languages_code + //nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}` + nitem.id = `${item.languages_code}-${item.activities_id?.wpid}` + /*nitem.title = item.title nitem.activities_title = item.activities_id?.title nitem.date = item.activities_id?.date; - nitem.activity = item.activities_id?.activity; + nitem.activity = item.activities_id?.activity;*/ - nitem.thumbnail = item.activities_id?.thumbnail?.filename_download - nitem.thumbnail_file = item.activities_id?.thumbnail?.filename_disk - nitem.thumbnail_id = item.activities_id?.thumbnail?.id - nitem.video = item.privateVideo?.filename_download - nitem.video_file = item.privateVideo?.filename_disk - nitem.video_id = item.privateVideo?.id - nitem.audio = item.mp3?.filename_download - nitem.audio_file = item.mp3?.filename_disk - nitem.audio_id = item.mp3?.id - nitem.booklet = item.pdf_booklet?.filename_download - nitem.booklet_file = item.pdf_booklet?.filename_disk - nitem.booklet_id = item.pdf_booklet?.id - nitem.simple = item.pdf?.filename_download - nitem.simple_file = item.pdf?.filename_disk - nitem.simple_id = item.pdf?.id + //nitem.thumbnail = item.activities_id?.thumbnail?.filename_download + //nitem.thumbnail_file = item.activities_id?.thumbnail?.filename_disk + nitem.thumbnail_id = item.activities_id?.thumbnail?.filename_disk + //nitem.video = item.privateVideo?.filename_download + //nitem.video_file = item.privateVideo?.filename_disk + nitem.files = {} + nitem.files.video = item.privateVideo?.filename_disk + //nitem.audio = item.mp3?.filename_download + //nitem.audio_file = item.mp3?.filename_disk + nitem.files.audio = item.mp3?.filename_disk + //nitem.booklet = item.pdf_booklet?.filename_download + //nitem.booklet_file = item.pdf_booklet?.filename_disk + nitem.files.booklet = item.pdf_booklet?.filename_disk + //nitem.simple = item.pdf?.filename_download + //nitem.simple_file = item.pdf?.filename_disk + nitem.files.simple = item.pdf?.filename_disk } if (type == 'conferences_translations') { @@ -283,7 +284,8 @@ async function generateFiles( type ) { } if(type=='materiales_didacticos_translations'){ - nitem.id = item.title + nitem.id = item.id.toString() + nitem.title = item.title nitem.locale = item.languages_code nitem.featured_image = item.featured_image?.filename_disk nitem.materiales_didacticos_id = item.materiales_didacticos_id?.id diff --git a/carpa_json_to_markdown/index_wp.js b/carpa_json_to_markdown/index_wp.js index 9a9b221..902e5e9 100644 --- a/carpa_json_to_markdown/index_wp.js +++ b/carpa_json_to_markdown/index_wp.js @@ -98,7 +98,7 @@ async function generateJson( type, year ) { url = `https://actividadeswp.carpa.com/v3/actividades/?f=texts&locale=${LOCALE}&year=${year}&limit=1000`; } else { //url = `https://conferenciaswp.carpa.com/v3/conferencias/?f=algolia&locale=${LOCALE}&year=${year}&limit=1000&cache=false`; - url = `http://localhost:10018/v3/conferencias/?f=typesense&locale=${LOCALE}&year=${year}`; + url = `http://conferences.local/v3/conferencias/?f=typesense&locale=${LOCALE}&year=${year}`; } request(url, options, async (error, res, body) => { @@ -112,13 +112,14 @@ async function generateJson( type, year ) { // do something with JSON, using the 'body' variable let items = [] - console.log( body.length + " items found for year " + year + " and type " + type ); + console.log( body.result.conferencias.length + " items found for year " + year + " and type " + type ); + console.log( 'URL', url ) - items = body; + items = body.result.conferencias; if(items.length > 0) { items.map((item) => { - if( item.id == null || item.id == undefined || item.id == '' ) { + if( item._id == null || item._id == undefined || item._id == '' ) { console.log( "Item has no id, skipping." ); return false; }}) @@ -167,7 +168,7 @@ async function generateJson( type, year ) { if (type == 'conferences') { nitem.locale = LOCALE; - nitem.id = item.id.toString() + nitem.id = item._id.toString() nitem.type = 'conferences' nitem.title = item.title nitem.timestamp = item.timestamp @@ -180,7 +181,7 @@ async function generateJson( type, year ) { nitem.city = item.city || null; nitem.state = item.state || null; nitem.country = item.country || null; - nitem.body = item.body; + //nitem.body = item.body; nitem.thumbnail = item.thumbnail nitem.youtube = item.files?.youtube nitem.video = item.files?.video @@ -188,7 +189,7 @@ async function generateJson( type, year ) { nitem.booklet = item.files?.pdf nitem.simple = item.files?.pdf_simple nitem.directus = ''; - nitem.wp = item.id.toString(); + nitem.wp = item._id.toString(); nitem.typesense = true; nitem.rm = item.rm; nitem.private = item.private == 1 ? true : false;