Update indexer
Modified indexer to query _translation tables directly so that we can pull only translated versions of activities and conferences
This commit is contained in:
Julio Ruiz
parent
2faede3348
commit
a3a16eb006
|
|
@ -18,6 +18,7 @@ let jsonlData = '';
|
||||||
const DATA_INPUT_FOLDER = './input';
|
const DATA_INPUT_FOLDER = './input';
|
||||||
const DATA_OUTPUT_FOLDER = './output';
|
const DATA_OUTPUT_FOLDER = './output';
|
||||||
const TEMPLATES_FOLDER = './templates';
|
const TEMPLATES_FOLDER = './templates';
|
||||||
|
const LOCALE = 'es';
|
||||||
|
|
||||||
function generateMarkdown(type) {
|
function generateMarkdown(type) {
|
||||||
const folderPath = `${DATA_INPUT_FOLDER}/${type}`;
|
const folderPath = `${DATA_INPUT_FOLDER}/${type}`;
|
||||||
|
|
@ -88,7 +89,7 @@ function generateMarkdown(type) {
|
||||||
console.log(`Summary created successfully.`);
|
console.log(`Summary created successfully.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function generateJson( type, year ) {
|
async function generateJson( type ) {
|
||||||
let options = { json: true };
|
let options = { json: true };
|
||||||
|
|
||||||
let fields = [];
|
let fields = [];
|
||||||
|
|
@ -134,7 +135,49 @@ async function generateJson( type, year ) {
|
||||||
'translations.pdf.filename_disk',
|
'translations.pdf.filename_disk',
|
||||||
]
|
]
|
||||||
|
|
||||||
let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&deep[translations][_filter][languages_code][_eq]=es&filter[year(date)]=${year}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=1000&sort=-date`;
|
fields['activities_translations'] = [
|
||||||
|
'activities_id.id',
|
||||||
|
'title',
|
||||||
|
'activities_id.date',
|
||||||
|
'activities_id.activity',
|
||||||
|
'activities_id.place',
|
||||||
|
'activities_id.city',
|
||||||
|
'activities_id.state',
|
||||||
|
'activities_id.country',
|
||||||
|
'activities_id.duration',
|
||||||
|
'interventions.text',
|
||||||
|
'activities_id.private',
|
||||||
|
'activities_id.thumbnail.filename_disk',
|
||||||
|
'slug',
|
||||||
|
'youtube',
|
||||||
|
'privateVideo.filename_disk',
|
||||||
|
'mp3.filename_disk',
|
||||||
|
'pdf_booklet.filename_disk',
|
||||||
|
'pdf.filename_disk'
|
||||||
|
]
|
||||||
|
|
||||||
|
fields['conferences_translations'] = [
|
||||||
|
'conferences_id.id',
|
||||||
|
'title',
|
||||||
|
'conferences_id.date',
|
||||||
|
'conferences_id.activity',
|
||||||
|
'conferences_id.place',
|
||||||
|
'conferences_id.city',
|
||||||
|
'conferences_id.state',
|
||||||
|
'conferences_id.country',
|
||||||
|
'conferences_id.duration',
|
||||||
|
'text_published',
|
||||||
|
'conferences_id.public',
|
||||||
|
'conferences_id.thumbnail.filename_disk',
|
||||||
|
'slug',
|
||||||
|
'youtube',
|
||||||
|
'video.filename_disk',
|
||||||
|
'audio.filename_disk',
|
||||||
|
'pdf.filename_disk',
|
||||||
|
'pdf_simple.filename_disk',
|
||||||
|
]
|
||||||
|
|
||||||
|
let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&filter[languages_code]=${LOCALE}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=10000`;
|
||||||
|
|
||||||
console.log( url )
|
console.log( url )
|
||||||
|
|
||||||
|
|
@ -147,60 +190,118 @@ async function generateJson( type, year ) {
|
||||||
// do something with JSON, using the 'body' variable
|
// do something with JSON, using the 'body' variable
|
||||||
const items = body.data;
|
const items = body.data;
|
||||||
|
|
||||||
items.map((item) => {
|
let nitems = items.map((item) => {
|
||||||
if(type=='activities'){
|
let nitem = {}
|
||||||
item.body = he.decode(striptags(item.translations[0]?.interventions[0]?.text)) || ''
|
|
||||||
item.private = item.private == 1 ? true : false;
|
if( type == 'activities_translations' ){
|
||||||
item.files = {}
|
nitem.id = item.activities_id?.id
|
||||||
item.files.youtube = item.translations[0]?.youtube
|
nitem.title = item.title
|
||||||
item.files.video = item.translations[0]?.privateVideo?.filename_disk
|
nitem.body = he.decode(striptags(item.interventions[0]?.text)) || ''
|
||||||
item.files.audio = item.translations[0]?.mp3?.filename_disk
|
nitem.private = item.activities_id?.private == 1 ? true : false;
|
||||||
item.files.booklet = item.translations[0]?.pdf_booklet?.filename_disk
|
nitem.files = {}
|
||||||
item.files.simple = item.translations[0]?.pdf?.filename_disk
|
nitem.files.youtube = item.youtube
|
||||||
|
nitem.files.video = item.privateVideo?.filename_disk
|
||||||
|
nitem.files.audio = item.mp3?.filename_disk
|
||||||
|
nitem.files.booklet = item.pdf_booklet?.filename_disk
|
||||||
|
nitem.files.simple = item.pdf?.filename_disk
|
||||||
|
nitem.slug = item.slug
|
||||||
|
nitem.place = item.activities_id?.place || null;
|
||||||
|
nitem.city = item.activities_id?.city || null;
|
||||||
|
nitem.state = item.activities_id?.state || null;
|
||||||
|
nitem.country = item.activities_id?.country || null;
|
||||||
|
|
||||||
|
nitem.duration = item.activities_id?.duration ?? 0;
|
||||||
|
|
||||||
|
nitem.year = dayjs(item.activities_id?.date).year().toString()
|
||||||
|
nitem.month = item.year + " > " + (dayjs(item.activities_id?.date).month()+1).toString().padStart(2,"0")
|
||||||
|
|
||||||
|
nitem.menu_lvl0 = item.country
|
||||||
|
nitem.menu_lvl1 = item.country + " > " + item.state
|
||||||
|
nitem.menu_lvl2 = item.country + " > " + item.state + " > " + item.city
|
||||||
|
nitem.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place
|
||||||
|
|
||||||
|
nitem.date = dayjs(item.activities_id?.date).unix()
|
||||||
|
|
||||||
|
nitem.thumbnail = item.activities_id?.thumbnail?.filename_disk
|
||||||
}
|
}
|
||||||
|
|
||||||
if(type=='conferences'){
|
if( type == 'conferences_translations' ){
|
||||||
item.id = item.id.toString();
|
nitem.id = item.conferences_id?.id
|
||||||
item.body = he.decode(striptags(item.translations[0]?.text_published)) || ''
|
nitem.title = item.title
|
||||||
item.private = item.public == 0 ? true : false;
|
nitem.body = he.decode(striptags(item.text_published)) || ''
|
||||||
item.files = {}
|
nitem.private = item.conferences_id?.public == 0 ? true : false;
|
||||||
item.files.youtube = item.translations[0]?.youtube
|
nitem.files = {}
|
||||||
item.files.video = item.translations[0]?.video?.filename_disk
|
nitem.files.youtube = item.youtube
|
||||||
item.files.audio = item.translations[0]?.audio?.filename_disk
|
nitem.files.video = item.privateVideo?.filename_disk
|
||||||
item.files.booklet = item.translations[0]?.pdf?.filename_disk
|
nitem.files.audio = item.mp3?.filename_disk
|
||||||
item.files.simple = item.translations[0]?.pdf_simple?.filename_disk
|
nitem.files.booklet = item.pdf_booklet?.filename_disk
|
||||||
delete item.public
|
nitem.files.simple = item.pdf?.filename_disk
|
||||||
|
nitem.slug = item.slug
|
||||||
|
nitem.place = item.conferences_id?.place || null;
|
||||||
|
nitem.city = item.conferences_id?.city || null;
|
||||||
|
nitem.state = item.conferences_id?.state || null;
|
||||||
|
nitem.country = item.conferences_id?.country || null;
|
||||||
|
|
||||||
|
nitem.duration = item.conferences_id?.duration ?? 0;
|
||||||
|
|
||||||
|
nitem.year = dayjs(item.conferences_id?.date).year().toString()
|
||||||
|
nitem.month = item.year + " > " + (dayjs(item.conferences_id?.date).month()+1).toString().padStart(2,"0")
|
||||||
|
|
||||||
|
nitem.menu_lvl0 = item.country
|
||||||
|
nitem.menu_lvl1 = item.country + " > " + item.state
|
||||||
|
nitem.menu_lvl2 = item.country + " > " + item.state + " > " + item.city
|
||||||
|
nitem.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place
|
||||||
|
|
||||||
|
nitem.date = dayjs(item.conferences_id?.date).unix()
|
||||||
|
|
||||||
|
nitem.thumbnail = item.conferences_id?.thumbnail?.filename_disk
|
||||||
}
|
}
|
||||||
|
|
||||||
item.slug = item.translations[0]?.slug
|
// if(type=='conferences'){
|
||||||
item.place = item.place || null;
|
// item.id = item.id.toString();
|
||||||
item.city = item.city || null;
|
// item.body = he.decode(striptags(item.translations[0]?.text_published)) || ''
|
||||||
item.state = item.state || null;
|
// item.private = item.public == 0 ? true : false;
|
||||||
item.country = item.country || null;
|
// item.files = {}
|
||||||
|
// item.files.youtube = item.translations[0]?.youtube
|
||||||
|
// item.files.video = item.translations[0]?.video?.filename_disk
|
||||||
|
// item.files.audio = item.translations[0]?.audio?.filename_disk
|
||||||
|
// item.files.booklet = item.translations[0]?.pdf?.filename_disk
|
||||||
|
// item.files.simple = item.translations[0]?.pdf_simple?.filename_disk
|
||||||
|
// delete item.public
|
||||||
|
// }
|
||||||
|
|
||||||
item.duration = item.duration ?? 0;
|
// item.slug = item.translations[0]?.slug
|
||||||
|
// item.place = item.place || null;
|
||||||
|
// item.city = item.city || null;
|
||||||
|
// item.state = item.state || null;
|
||||||
|
// item.country = item.country || null;
|
||||||
|
|
||||||
item.year = dayjs(item.date).year().toString()
|
// item.duration = item.duration ?? 0;
|
||||||
item.month = item.year + " > " + (dayjs(item.date).month()+1).toString().padStart(2,"0")
|
|
||||||
|
|
||||||
item.menu_lvl0 = item.country
|
// item.year = dayjs(item.date).year().toString()
|
||||||
item.menu_lvl1 = item.country + " > " + item.state
|
// item.month = item.year + " > " + (dayjs(item.date).month()+1).toString().padStart(2,"0")
|
||||||
item.menu_lvl2 = item.country + " > " + item.state + " > " + item.city
|
|
||||||
item.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place
|
|
||||||
|
|
||||||
item.date = dayjs(item.date).unix()
|
// item.menu_lvl0 = item.country
|
||||||
|
// item.menu_lvl1 = item.country + " > " + item.state
|
||||||
|
// item.menu_lvl2 = item.country + " > " + item.state + " > " + item.city
|
||||||
|
// item.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place
|
||||||
|
|
||||||
item.thumbnail = item.thumbnail?.filename_disk
|
// item.date = dayjs(item.date).unix()
|
||||||
|
|
||||||
|
// item.thumbnail = item.thumbnail?.filename_disk
|
||||||
|
|
||||||
|
// delete item.translations;
|
||||||
|
if( nitem.title && nitem.id ){
|
||||||
|
return nitem;
|
||||||
|
}
|
||||||
|
|
||||||
delete item.translations;
|
|
||||||
return item;
|
|
||||||
})
|
})
|
||||||
|
|
||||||
if (!fs.existsSync(`./${DATA_INPUT_FOLDER}/${type}`)) {
|
// if (!fs.existsSync(`./${DATA_INPUT_FOLDER}/${type}`)) {
|
||||||
fs.mkdirSync(`./${DATA_INPUT_FOLDER}/${type}`);
|
// fs.mkdirSync(`./${DATA_INPUT_FOLDER}/${type}`);
|
||||||
}
|
// }
|
||||||
|
|
||||||
jsonlData += items.map(JSON.stringify).join('\n')
|
jsonlData += nitems.filter(item => item).map(JSON.stringify).join('\n')
|
||||||
|
|
||||||
writeFile( jsonlData , type )
|
writeFile( jsonlData , type )
|
||||||
};
|
};
|
||||||
|
|
@ -209,7 +310,7 @@ async function generateJson( type, year ) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function writeFile( jsonlData, type ){
|
function writeFile( jsonlData, type ){
|
||||||
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}.json`, jsonlData, (err) => {
|
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.json`, jsonlData, (err) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
console.error("Error writing file:", err);
|
console.error("Error writing file:", err);
|
||||||
return;
|
return;
|
||||||
|
|
@ -232,6 +333,6 @@ function writeFile( jsonlData, type ){
|
||||||
// generateJson( 'conferences', year );
|
// generateJson( 'conferences', year );
|
||||||
// }
|
// }
|
||||||
|
|
||||||
for( let year = 2021; year < 2027; year++){
|
//for( let year = 2021; year < 2027; year++){
|
||||||
generateJson( 'activities', year );
|
generateJson( 'conferences_translations' );
|
||||||
}
|
//}
|
||||||
Loading…
Reference in New Issue