LGCC
/
api
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updating indexer

This commit is contained in:
Julio Ruiz 2026-02-03 00:05:21 -05:00
parent a937fc96f3
commit 155db6157d
2 changed files with 95 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
carpa_json_to_markdown/index.js
View File

@ -60,28 +60,28 @@ function generateMarkdown(type) {
fs.writeFileSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}/${month}/${conference.date}-${conference.activity}.md`, output) fs.writeFileSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}/${month}/${conference.date}-${conference.activity}.md`, output)
//Add month count for summary //Add month count for summary
const m = conference.date.substring(5,7); const m = conference.date.substring(5, 7);
if( !months.hasOwnProperty( m ) ){ if (!months.hasOwnProperty(m)) {
months[m] = [] months[m] = []
} }
months[m].push( m ) months[m].push(m)
}) })
let monthsArray = []; let monthsArray = [];
for ( const month in months) { for (const month in months) {
let mObject = {}; let mObject = {};
mObject.month = month; mObject.month = month;
mObject.count = months[month].length; mObject.count = months[month].length;
monthsArray.push( mObject ); monthsArray.push(mObject);
} }
//Sort monthsarray //Sort monthsarray
monthsArray.sort((a,b) => b.month - a.month); // b - a for reverse sort monthsArray.sort((a, b) => b.month - a.month); // b - a for reverse sort
yearObject.months = monthsArray; yearObject.months = monthsArray;
summary.push( yearObject ); summary.push(yearObject);
}) })
@ -93,48 +93,6 @@ async function generateJson( type ) {
let options = { json: true }; let options = { json: true };
let fields = []; let fields = [];
fields['conferences'] = [
'id',
'title',
'date',
'activity',
'place',
'city',
'state',
'country',
'duration',
'translations.text_published',
'public',
'thumbnail.filename_disk',
'translations.slug',
'translations.youtube',
'translations.video.filename_disk',
'translations.audio.filename_disk',
'translations.pdf.filename_disk',
'translations.pdf_simple.filename_disk',
]
fields['activities'] = [
'id',
'title',
'date',
'activity',
'place',
'city',
'state',
'country',
'duration',
'translations.interventions.text',
'private',
'thumbnail.filename_disk',
'translations.slug',
'translations.youtube',
'translations.privateVideo.filename_disk',
'translations.mp3.filename_disk',
'translations.pdf_booklet.filename_disk',
'translations.pdf.filename_disk',
]
fields['activities_translations'] = [ fields['activities_translations'] = [
'activities_id.id', 'activities_id.id',
'title', 'title',
@ -179,8 +137,6 @@ async function generateJson( type ) {
let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&filter[languages_code]=${LOCALE}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=10000`; let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&filter[languages_code]=${LOCALE}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=10000`;
console.log( url )
request(url, options, (error, res, body) => { request(url, options, (error, res, body) => {
if (error) { if (error) {
return console.log(error) return console.log(error)
@ -193,10 +149,12 @@ async function generateJson( type ) {
let nitems = items.map((item) => { let nitems = items.map((item) => {
let nitem = {} let nitem = {}
if( type == 'activities_translations' ){ if (type == 'activities_translations') {
nitem.id = item.activities_id?.id nitem.id = item.activities_id?.id
nitem.type = 'activities'
nitem.title = item.title nitem.title = item.title
nitem.body = he.decode(striptags(item.interventions[0]?.text)) || '' //nitem.body = he.decode(striptags(item.interventions[0]?.text)) || ''
nitem.body = item.interventions[0]?.text
nitem.private = item.activities_id?.private == 1 ? true : false; nitem.private = item.activities_id?.private == 1 ? true : false;
nitem.files = {} nitem.files = {}
nitem.files.youtube = item.youtube nitem.files.youtube = item.youtube
@ -213,22 +171,29 @@ async function generateJson( type ) {
nitem.duration = item.activities_id?.duration ?? 0; nitem.duration = item.activities_id?.duration ?? 0;
nitem.year = dayjs(item.activities_id?.date).year().toString() nitem.year = dayjs(item.activities_id?.date).year().toString()
nitem.month = nitem.year + " > " + (dayjs(item.activities_id?.date).month()+1).toString().padStart(2,"0") nitem.month = nitem.year + " > " + (dayjs(item.activities_id?.date).month() + 1).toString().padStart(2, "0")
nitem.menu_lvl0 = nitem.country nitem.menu_country = nitem.country
nitem.menu_lvl1 = nitem.country + " > " + nitem.state nitem.menu_state = nitem.country + " > " + nitem.state
nitem.menu_lvl2 = nitem.country + " > " + nitem.state + " > " + nitem.city nitem.menu_city = nitem.country + " > " + nitem.state + " > " + nitem.city
nitem.menu_lvl3 = nitem.country + " > " + nitem.state + " > " + nitem.city + " > " + nitem.place nitem.menu_place = nitem.country + " > " + nitem.state + " > " + nitem.city + " > " + nitem.place
nitem.date = dayjs(item.activities_id?.date).unix() nitem.date = dayjs(item.activities_id?.date).unix()
nitem.thumbnail = item.activities_id?.thumbnail?.filename_disk nitem.thumbnail = item.activities_id?.thumbnail?.filename_disk
//Filter out anything before 25/12/2021
if( nitem.date < 1640408400 ) {
return false
}
} }
if( type == 'conferences_translations' ){ if (type == 'conferences_translations') {
nitem.id = item.conferences_id?.id.toString() nitem.id = item.conferences_id?.id.toString()
nitem.type = 'conferences'
nitem.title = item.title nitem.title = item.title
nitem.body = he.decode(striptags(item.text_published)) || '' //nitem.body = he.decode(striptags(item.text_published)) || ''
nitem.body = item.text_published
nitem.private = item.conferences_id?.public == 0 ? true : false; nitem.private = item.conferences_id?.public == 0 ? true : false;
nitem.files = {} nitem.files = {}
nitem.files.youtube = item.youtube nitem.files.youtube = item.youtube
@ -245,71 +210,32 @@ async function generateJson( type ) {
nitem.duration = item.conferences_id?.duration ?? 0; nitem.duration = item.conferences_id?.duration ?? 0;
nitem.year = dayjs(item.conferences_id?.date).year().toString() nitem.year = dayjs(item.conferences_id?.date).year().toString()
nitem.month = nitem.year + " > " + (dayjs(item.conferences_id?.date).month()+1).toString().padStart(2,"0") nitem.month = nitem.year + " > " + (dayjs(item.conferences_id?.date).month() + 1).toString().padStart(2, "0")
nitem.menu_lvl0 = nitem.country nitem.menu_country = nitem.country
nitem.menu_lvl1 = nitem.country + " > " + nitem.state nitem.menu_state = nitem.country + " > " + nitem.state
nitem.menu_lvl2 = nitem.country + " > " + nitem.state + " > " + nitem.city nitem.menu_city = nitem.country + " > " + nitem.state + " > " + nitem.city
nitem.menu_lvl3 = nitem.country + " > " + nitem.state + " > " + nitem.city + " > " + nitem.place nitem.menu_place = nitem.country + " > " + nitem.state + " > " + nitem.city + " > " + nitem.place
nitem.date = dayjs(item.conferences_id?.date).unix() nitem.date = dayjs(item.conferences_id?.date).unix()
nitem.thumbnail = item.conferences_id?.thumbnail?.filename_disk nitem.thumbnail = item.conferences_id?.thumbnail?.filename_disk
} }
if (nitem.title && nitem.id) {
// if(type=='conferences'){
// item.id = item.id.toString();
// item.body = he.decode(striptags(item.translations[0]?.text_published)) || ''
// item.private = item.public == 0 ? true : false;
// item.files = {}
// item.files.youtube = item.translations[0]?.youtube
// item.files.video = item.translations[0]?.video?.filename_disk
// item.files.audio = item.translations[0]?.audio?.filename_disk
// item.files.booklet = item.translations[0]?.pdf?.filename_disk
// item.files.simple = item.translations[0]?.pdf_simple?.filename_disk
// delete item.public
// }
// item.slug = item.translations[0]?.slug
// item.place = item.place || null;
// item.city = item.city || null;
// item.state = item.state || null;
// item.country = item.country || null;
// item.duration = item.duration ?? 0;
// item.year = dayjs(item.date).year().toString()
// item.month = item.year + " > " + (dayjs(item.date).month()+1).toString().padStart(2,"0")
// item.menu_lvl0 = item.country
// item.menu_lvl1 = item.country + " > " + item.state
// item.menu_lvl2 = item.country + " > " + item.state + " > " + item.city
// item.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place
// item.date = dayjs(item.date).unix()
// item.thumbnail = item.thumbnail?.filename_disk
// delete item.translations;
if( nitem.title && nitem.id ){
return nitem; return nitem;
} }
}) })
// if (!fs.existsSync(`./${DATA_INPUT_FOLDER}/${type}`)) {
// fs.mkdirSync(`./${DATA_INPUT_FOLDER}/${type}`);
// }
jsonlData += nitems.filter(item => item).map(JSON.stringify).join('\n') jsonlData += nitems.filter(item => item).map(JSON.stringify).join('\n')
writeFile( jsonlData , type ) writeFile(jsonlData, type)
}; };
}); });
} }
function writeFile( jsonlData, type ){ function writeFile(jsonlData, type) {
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.json`, jsonlData, (err) => { fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.json`, jsonlData, (err) => {
if (err) { if (err) {
console.error("Error writing file:", err); console.error("Error writing file:", err);
@ -334,5 +260,6 @@ function writeFile( jsonlData, type ){
// } // }
//for( let year = 2021; year < 2027; year++){ //for( let year = 2021; year < 2027; year++){
generateJson( 'activities_translations' ); //generateJson( 'activities_translations' );
generateJson( 'conferences_translations' );
//} //}

13
typesense/schema/activities_base.json
View File

@ -6,6 +6,11 @@
"type": "string", "type": "string",
"facet": false "facet": false
}, },
{
"name": "type",
"type": "string",
"facet": false
},
{ {
"name": "title", "name": "title",
"type": "string", "type": "string",
@ -75,25 +80,25 @@
"facet":true "facet":true
}, },
{ {
"name": "menu_lvl0", "name": "menu_country",
"type": "string", "type": "string",
"facet":true, "facet":true,
"optional": true "optional": true
}, },
{ {
"name": "menu_lvl1", "name": "menu_state",
"type": "string", "type": "string",
"facet":true, "facet":true,
"optional": true "optional": true
}, },
{ {
"name": "menu_lvl2", "name": "menu_city",
"type": "string", "type": "string",
"facet":true, "facet":true,
"optional": true "optional": true
}, },
{ {
"name": "menu_lvl3", "name": "menu_place",
"type": "string", "type": "string",
"facet":true, "facet":true,
"optional": true "optional": true