api/carpa_json_to_markdown/index.js

//import fetch from 'node-fetch'; // For ES Modules
const fs = require('fs');
const path = require('path');

const striptags = require('striptags');
const dayjs = require('dayjs');
const he = require('he');
const jsonTojsonl = require('json-to-jsonl');

const fetch = require('node-fetch'); // For CommonJS
const { start } = require('repl');
const request = require('request');
let { render } = require("mustache");

let jsonlData = '';

const DATA_INPUT_FOLDER = './input';
const DATA_OUTPUT_FOLDER = './output';
const TEMPLATES_FOLDER = './templates';

function generateMarkdown(type) {
  const folderPath = `${DATA_INPUT_FOLDER}/${type}`;
  const files = fs.readdirSync(folderPath);
  const jsonFiles = files.filter(file => file.endsWith('.json'));

  const summary = [];

  jsonFiles.forEach(file => {
    const startYear = path.basename(file, '.json');
    const filePath = path.join(folderPath, file);
    const content = fs.readFileSync(filePath, 'utf8');
    const data = JSON.parse(content);

    //Init year for summaries
    yearObject = {};
    yearObject.year = startYear;
    yearObject.count = data.length;
    //Init months object for summary
    let months = {};

    let template = fs.readFileSync(`./${TEMPLATES_FOLDER}/${type}.md`).toString()

    data.map((conference) => {
      let month = conference.date.substring(5, 7);
      let output = render(template, conference)

      if (!fs.existsSync(`./${DATA_OUTPUT_FOLDER}/${type}`)) {
        fs.mkdirSync(`./${DATA_OUTPUT_FOLDER}/${type}`);
      }

      if (!fs.existsSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}`)) {
        fs.mkdirSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}`);
      }

      if (!fs.existsSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}/${month}`)) {
        fs.mkdirSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}/${month}`);
      }
      fs.writeFileSync(`./${DATA_OUTPUT_FOLDER}/${type}/${startYear}/${month}/${conference.date}-${conference.activity}.md`, output)

      //Add month count for summary
      const m = conference.date.substring(5,7);
       if( !months.hasOwnProperty( m ) ){
            months[m] = []
       }
       months[m].push( m )
    })

    let monthsArray = [];
    
    for ( const month in months) {
        let mObject = {};
        mObject.month = month;
        mObject.count = months[month].length;
        monthsArray.push( mObject );
    }

    //Sort monthsarray
    monthsArray.sort((a,b) => b.month - a.month); // b - a for reverse sort

    yearObject.months = monthsArray;

    summary.push( yearObject );

  })

  fs.writeFileSync(`./${DATA_OUTPUT_FOLDER}/${type}/${type}.json`, JSON.stringify(summary.reverse(), null, 2));
  console.log(`Summary created successfully.`);
}

async function generateJson( type, year ) {
  let options = { json: true };

  let fields = [];
  fields['conferences'] = [
    'id',
    'title',
    'date',
    'activity',
    'place',
    'city',
    'state',
    'country',
    'duration',
    'translations.text_published',
    'public',
    'thumbnail.filename_disk',
    'translations.slug',
    'translations.youtube',
    'translations.video.filename_disk',
    'translations.audio.filename_disk',
    'translations.pdf.filename_disk',
    'translations.pdf_simple.filename_disk',
  ]

  fields['activities'] = [
    'id',
    'title',
    'date',
    'activity',
    'place',
    'city',
    'state',
    'country',
    'duration',
    'translations.interventions.text',
    'private',
    'thumbnail.filename_disk',
    'translations.slug',
    'translations.youtube',
    'translations.privateVideo.filename_disk',
    'translations.mp3.filename_disk',
    'translations.pdf_booklet.filename_disk',
    'translations.pdf.filename_disk',
  ]

  let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&deep[translations][_filter][languages_code][_eq]=es&filter[year(date)]=${year}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=1000&sort=-date`;

  console.log( url )

  request(url, options, (error, res, body) => {
    if (error) {
      return console.log(error)
    };

    if (!error && res.statusCode == 200) {
      // do something with JSON, using the 'body' variable
      const items = body.data;

      items.map((item) => {
        if(type=='activities'){
          item.body = he.decode(striptags(item.translations[0]?.interventions[0]?.text)) || ''
          item.private = item.private == 1 ? true : false;
          item.files = {}
          item.files.youtube = item.translations[0]?.youtube
          item.files.video = item.translations[0]?.privateVideo?.filename_disk
          item.files.audio = item.translations[0]?.mp3?.filename_disk
          item.files.booklet = item.translations[0]?.pdf_booklet?.filename_disk
          item.files.simple = item.translations[0]?.pdf?.filename_disk
        }

        if(type=='conferences'){
          item.id = item.id.toString();
          item.body = he.decode(striptags(item.translations[0]?.text_published)) || ''
          item.private = item.public == 0 ? true : false;
          item.files = {}
          item.files.youtube = item.translations[0]?.youtube
          item.files.video = item.translations[0]?.video?.filename_disk
          item.files.audio = item.translations[0]?.audio?.filename_disk
          item.files.booklet = item.translations[0]?.pdf?.filename_disk
          item.files.simple = item.translations[0]?.pdf_simple?.filename_disk
          delete item.public
        }
        
        item.slug = item.translations[0]?.slug
        item.place = item.place || null;
        item.city = item.city || null;
        item.state = item.state || null;
        item.country = item.country || null;

        item.duration = item.duration ?? 0;

        item.year = dayjs(item.date).year().toString()
        item.month = item.year + " > " + (dayjs(item.date).month()+1).toString().padStart(2,"0")

        item.menu_lvl0 = item.country
        item.menu_lvl1 = item.country + " > " + item.state
        item.menu_lvl2 = item.country + " > " + item.state + " > " + item.city
        item.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place

        item.date = dayjs(item.date).unix()

        item.thumbnail = item.thumbnail?.filename_disk

        delete item.translations;
        return item;
      })

      if (!fs.existsSync(`./${DATA_INPUT_FOLDER}/${type}`)) {
        fs.mkdirSync(`./${DATA_INPUT_FOLDER}/${type}`);
      }

      jsonlData += items.map(JSON.stringify).join('\n')

      writeFile( jsonlData , type )
    };
  });

}

function writeFile( jsonlData, type ){
  fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}.json`, jsonlData, (err) => {
      if (err) {
        console.error("Error writing file:", err);
        return;
      }
      console.log("File has been written successfully! : " + type);
    });
}


//generateJson( 'conferences', 1974);
//generateJson( 'activities' , 2021 );

// setTimeout( () => {
//   generateMarkdown( 'conferences' )
// }, 5000 );
//generateMarkdown( 'activities' );

//generateMarkdown( 'conferences' );
// for( let year = 1974; year < 2019; year++){
//   generateJson( 'conferences', year );
// }

for( let year = 2021; year < 2027; year++){
  generateJson( 'activities', year );
}