338 lines
11 KiB
JavaScript
338 lines
11 KiB
JavaScript
|
|
//import fetch from 'node-fetch'; // For ES Modules
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const striptags = require('striptags');
|
|
const dayjs = require('dayjs');
|
|
const he = require('he');
|
|
const jsonTojsonl = require('json-to-jsonl');
|
|
|
|
const fetch = require('node-fetch'); // For CommonJS
|
|
const { start } = require('repl');
|
|
const request = require('request');
|
|
let { render } = require("mustache");
|
|
|
|
let jsonlData = '';
|
|
|
|
const DATA_INPUT_FOLDER = './input';
|
|
const DATA_OUTPUT_FOLDER = './output';
|
|
const TEMPLATES_FOLDER = './templates';
|
|
const LOCALE = 'pt';
|
|
|
|
function generateMarkdown(type) {
|
|
const folderPath = `${DATA_INPUT_FOLDER}/${type}`;
|
|
const files = fs.readdirSync(folderPath);
|
|
const jsonFiles = files.filter(file => file.endsWith('.json'));
|
|
|
|
const summary = [];
|
|
|
|
jsonFiles.forEach(file => {
|
|
const startYear = path.basename(file, '.json');
|
|
const filePath = path.join(folderPath, file);
|
|
const content = fs.readFileSync(filePath, 'utf8');
|
|
const data = JSON.parse(content);
|
|
|
|
//Init year for summaries
|
|
yearObject = {};
|
|
yearObject.year = startYear;
|
|
yearObject.count = data.length;
|
|
//Init months object for summary
|
|
let months = {};
|
|
|
|
let template = fs.readFileSync(`./${TEMPLATES_FOLDER}/${type}.md`).toString()
|
|
|
|
data.map((conference) => {
|
|
let month = conference.date.substring(5, 7);
|
|
let output = render(template, conference)
|
|
|
|
if (!fs.existsSync(`./${DATA_OUTPUT_FOLDER}/${type}`)) {
|
|
fs.mkdirSync(`./${DATA_OUTPUT_FOLDER}/${type}`);
|
|
}
|
|
|
|
if (!fs.existsSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}`)) {
|
|
fs.mkdirSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}`);
|
|
}
|
|
|
|
if (!fs.existsSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}/${month}`)) {
|
|
fs.mkdirSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}/${month}`);
|
|
}
|
|
fs.writeFileSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}/${month}/${conference.date}-${conference.activity}.md`, output)
|
|
|
|
//Add month count for summary
|
|
const m = conference.date.substring(5,7);
|
|
if( !months.hasOwnProperty( m ) ){
|
|
months[m] = []
|
|
}
|
|
months[m].push( m )
|
|
})
|
|
|
|
let monthsArray = [];
|
|
|
|
for ( const month in months) {
|
|
let mObject = {};
|
|
mObject.month = month;
|
|
mObject.count = months[month].length;
|
|
monthsArray.push( mObject );
|
|
}
|
|
|
|
//Sort monthsarray
|
|
monthsArray.sort((a,b) => b.month - a.month); // b - a for reverse sort
|
|
|
|
yearObject.months = monthsArray;
|
|
|
|
summary.push( yearObject );
|
|
|
|
})
|
|
|
|
fs.writeFileSync(`./${DATA_OUTPUT_FOLDER}/${type}/${type}.json`, JSON.stringify(summary.reverse(), null, 2));
|
|
console.log(`Summary created successfully.`);
|
|
}
|
|
|
|
async function generateJson( type ) {
|
|
let options = { json: true };
|
|
|
|
let fields = [];
|
|
fields['conferences'] = [
|
|
'id',
|
|
'title',
|
|
'date',
|
|
'activity',
|
|
'place',
|
|
'city',
|
|
'state',
|
|
'country',
|
|
'duration',
|
|
'translations.text_published',
|
|
'public',
|
|
'thumbnail.filename_disk',
|
|
'translations.slug',
|
|
'translations.youtube',
|
|
'translations.video.filename_disk',
|
|
'translations.audio.filename_disk',
|
|
'translations.pdf.filename_disk',
|
|
'translations.pdf_simple.filename_disk',
|
|
]
|
|
|
|
fields['activities'] = [
|
|
'id',
|
|
'title',
|
|
'date',
|
|
'activity',
|
|
'place',
|
|
'city',
|
|
'state',
|
|
'country',
|
|
'duration',
|
|
'translations.interventions.text',
|
|
'private',
|
|
'thumbnail.filename_disk',
|
|
'translations.slug',
|
|
'translations.youtube',
|
|
'translations.privateVideo.filename_disk',
|
|
'translations.mp3.filename_disk',
|
|
'translations.pdf_booklet.filename_disk',
|
|
'translations.pdf.filename_disk',
|
|
]
|
|
|
|
fields['activities_translations'] = [
|
|
'activities_id.id',
|
|
'title',
|
|
'activities_id.date',
|
|
'activities_id.activity',
|
|
'activities_id.place',
|
|
'activities_id.city',
|
|
'activities_id.state',
|
|
'activities_id.country',
|
|
'activities_id.duration',
|
|
'interventions.text',
|
|
'activities_id.private',
|
|
'activities_id.thumbnail.filename_disk',
|
|
'slug',
|
|
'youtube',
|
|
'privateVideo.filename_disk',
|
|
'mp3.filename_disk',
|
|
'pdf_booklet.filename_disk',
|
|
'pdf.filename_disk'
|
|
]
|
|
|
|
fields['conferences_translations'] = [
|
|
'conferences_id.id',
|
|
'title',
|
|
'conferences_id.date',
|
|
'conferences_id.activity',
|
|
'conferences_id.place',
|
|
'conferences_id.city',
|
|
'conferences_id.state',
|
|
'conferences_id.country',
|
|
'conferences_id.duration',
|
|
'text_published',
|
|
'conferences_id.public',
|
|
'conferences_id.thumbnail.filename_disk',
|
|
'slug',
|
|
'youtube',
|
|
'video.filename_disk',
|
|
'audio.filename_disk',
|
|
'pdf.filename_disk',
|
|
'pdf_simple.filename_disk',
|
|
]
|
|
|
|
let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&filter[languages_code]=${LOCALE}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=10000`;
|
|
|
|
console.log( url )
|
|
|
|
request(url, options, (error, res, body) => {
|
|
if (error) {
|
|
return console.log(error)
|
|
};
|
|
|
|
if (!error && res.statusCode == 200) {
|
|
// do something with JSON, using the 'body' variable
|
|
const items = body.data;
|
|
|
|
let nitems = items.map((item) => {
|
|
let nitem = {}
|
|
|
|
if( type == 'activities_translations' ){
|
|
nitem.id = item.activities_id?.id
|
|
nitem.title = item.title
|
|
nitem.body = he.decode(striptags(item.interventions[0]?.text)) || ''
|
|
nitem.private = item.activities_id?.private == 1 ? true : false;
|
|
nitem.files = {}
|
|
nitem.files.youtube = item.youtube
|
|
nitem.files.video = item.privateVideo?.filename_disk
|
|
nitem.files.audio = item.mp3?.filename_disk
|
|
nitem.files.booklet = item.pdf_booklet?.filename_disk
|
|
nitem.files.simple = item.pdf?.filename_disk
|
|
nitem.slug = item.slug
|
|
nitem.place = item.activities_id?.place || null;
|
|
nitem.city = item.activities_id?.city || null;
|
|
nitem.state = item.activities_id?.state || null;
|
|
nitem.country = item.activities_id?.country || null;
|
|
|
|
nitem.duration = item.activities_id?.duration ?? 0;
|
|
|
|
nitem.year = dayjs(item.activities_id?.date).year().toString()
|
|
nitem.month = nitem.year + " > " + (dayjs(item.activities_id?.date).month()+1).toString().padStart(2,"0")
|
|
|
|
nitem.menu_lvl0 = nitem.country
|
|
nitem.menu_lvl1 = nitem.country + " > " + nitem.state
|
|
nitem.menu_lvl2 = nitem.country + " > " + nitem.state + " > " + nitem.city
|
|
nitem.menu_lvl3 = nitem.country + " > " + nitem.state + " > " + nitem.city + " > " + nitem.place
|
|
|
|
nitem.date = dayjs(item.activities_id?.date).unix()
|
|
|
|
nitem.thumbnail = item.activities_id?.thumbnail?.filename_disk
|
|
}
|
|
|
|
if( type == 'conferences_translations' ){
|
|
nitem.id = item.conferences_id?.id.toString()
|
|
nitem.title = item.title
|
|
nitem.body = he.decode(striptags(item.text_published)) || ''
|
|
nitem.private = item.conferences_id?.public == 0 ? true : false;
|
|
nitem.files = {}
|
|
nitem.files.youtube = item.youtube
|
|
nitem.files.video = item.privateVideo?.filename_disk
|
|
nitem.files.audio = item.mp3?.filename_disk
|
|
nitem.files.booklet = item.pdf_booklet?.filename_disk
|
|
nitem.files.simple = item.pdf?.filename_disk
|
|
nitem.slug = item.slug
|
|
nitem.place = item.conferences_id?.place || null;
|
|
nitem.city = item.conferences_id?.city || null;
|
|
nitem.state = item.conferences_id?.state || null;
|
|
nitem.country = item.conferences_id?.country || null;
|
|
|
|
nitem.duration = item.conferences_id?.duration ?? 0;
|
|
|
|
nitem.year = dayjs(item.conferences_id?.date).year().toString()
|
|
nitem.month = nitem.year + " > " + (dayjs(item.conferences_id?.date).month()+1).toString().padStart(2,"0")
|
|
|
|
nitem.menu_lvl0 = nitem.country
|
|
nitem.menu_lvl1 = nitem.country + " > " + nitem.state
|
|
nitem.menu_lvl2 = nitem.country + " > " + nitem.state + " > " + nitem.city
|
|
nitem.menu_lvl3 = nitem.country + " > " + nitem.state + " > " + nitem.city + " > " + nitem.place
|
|
|
|
nitem.date = dayjs(item.conferences_id?.date).unix()
|
|
|
|
nitem.thumbnail = item.conferences_id?.thumbnail?.filename_disk
|
|
}
|
|
|
|
// if(type=='conferences'){
|
|
// item.id = item.id.toString();
|
|
// item.body = he.decode(striptags(item.translations[0]?.text_published)) || ''
|
|
// item.private = item.public == 0 ? true : false;
|
|
// item.files = {}
|
|
// item.files.youtube = item.translations[0]?.youtube
|
|
// item.files.video = item.translations[0]?.video?.filename_disk
|
|
// item.files.audio = item.translations[0]?.audio?.filename_disk
|
|
// item.files.booklet = item.translations[0]?.pdf?.filename_disk
|
|
// item.files.simple = item.translations[0]?.pdf_simple?.filename_disk
|
|
// delete item.public
|
|
// }
|
|
|
|
// item.slug = item.translations[0]?.slug
|
|
// item.place = item.place || null;
|
|
// item.city = item.city || null;
|
|
// item.state = item.state || null;
|
|
// item.country = item.country || null;
|
|
|
|
// item.duration = item.duration ?? 0;
|
|
|
|
// item.year = dayjs(item.date).year().toString()
|
|
// item.month = item.year + " > " + (dayjs(item.date).month()+1).toString().padStart(2,"0")
|
|
|
|
// item.menu_lvl0 = item.country
|
|
// item.menu_lvl1 = item.country + " > " + item.state
|
|
// item.menu_lvl2 = item.country + " > " + item.state + " > " + item.city
|
|
// item.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place
|
|
|
|
// item.date = dayjs(item.date).unix()
|
|
|
|
// item.thumbnail = item.thumbnail?.filename_disk
|
|
|
|
// delete item.translations;
|
|
if( nitem.title && nitem.id ){
|
|
return nitem;
|
|
}
|
|
|
|
})
|
|
|
|
// if (!fs.existsSync(`./${DATA_INPUT_FOLDER}/${type}`)) {
|
|
// fs.mkdirSync(`./${DATA_INPUT_FOLDER}/${type}`);
|
|
// }
|
|
|
|
jsonlData += nitems.filter(item => item).map(JSON.stringify).join('\n')
|
|
|
|
writeFile( jsonlData , type )
|
|
};
|
|
});
|
|
|
|
}
|
|
|
|
function writeFile( jsonlData, type ){
|
|
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_${LOCALE.toLocaleUpperCase()}.json`, jsonlData, (err) => {
|
|
if (err) {
|
|
console.error("Error writing file:", err);
|
|
return;
|
|
}
|
|
console.log("File has been written successfully! : " + type);
|
|
});
|
|
}
|
|
|
|
|
|
//generateJson( 'conferences', 1974);
|
|
//generateJson( 'activities' , 2021 );
|
|
|
|
// setTimeout( () => {
|
|
// generateMarkdown( 'conferences' )
|
|
// }, 5000 );
|
|
//generateMarkdown( 'activities' );
|
|
|
|
//generateMarkdown( 'conferences' );
|
|
// for( let year = 1974; year < 2019; year++){
|
|
// generateJson( 'conferences', year );
|
|
// }
|
|
|
|
//for( let year = 2021; year < 2027; year++){
|
|
generateJson( 'activities_translations' );
|
|
//}
|