Node File creator

carpa_json_to_markdown/index_files.js

@@ -0,0 +1,143 @@
+
+//import fetch from 'node-fetch'; // For ES Modules
+const fs = require('fs');
+const path = require('path');
+const { parse } = require('json2csv');
+const striptags = require('striptags');
+const dayjs = require('dayjs');
+const he = require('he');
+const jsonTojsonl = require('json-to-jsonl');
+
+const fetch = require('node-fetch'); // For CommonJS
+const { start } = require('repl');
+const request = require('request');
+let { render } = require("mustache");
+
+let jsonlData = '';
+
+const DATA_INPUT_FOLDER = './input';
+const DATA_OUTPUT_FOLDER = './output';
+const TEMPLATES_FOLDER = './templates';
+const DIRECTUS_BUCKET = 'https://sjc1.vultrobjects.com/directus/'
+
+function isUndefined(variable){
+    if( typeof(variable) === 'undefined'){
+        return true
+    } else {
+        return false
+    }
+}
+
+async function generateFiles( type ) {
+  let options = { json: true };
+
+  let fields = [];
+
+  fields['activities_translations'] = [
+    "id",
+    "title",
+    "languages_code",
+    "mp3.filename_disk",
+    "mp3.filename_download",
+    "pdf_booklet.filename_disk",
+    "pdf_booklet.filename_download",
+    "pdf.filename_disk",
+    "pdf.filename_download",
+    "activities_id.id",
+    "activities_id.title",
+    "activities_id.date",
+    "activities_id.activity",
+    "activities_id.thumbnail.filename_disk",
+    "activities_id.thumbnail.filename_download"
+  ]
+
+  fields['conferences_translations'] = [
+    "id",
+    "title",
+    "conferences_id.title",
+    "conferences_id.date",
+    "conferences_id.activity",
+    "languages_code",
+    "video.filename_disk",
+    "video.filename_download",
+    "audio.filename_disk",
+    "audio.filename_download",
+    "pdf.filename_disk",
+    "pdf.filename_download",
+    "pdf_simple.filename_disk",
+    "pdf_simple.filename_download",
+    "conferences_id.thumbnail.filename_disk",
+    "conferences_id.thumbnail.filename_download"
+  ]
+
+  let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=100000&sort`;
+
+  request(url, options, (error, res, body) => {
+    if (error) {
+      return console.log(error)
+    };
+
+    if (!error && res.statusCode == 200) {
+      // do something with JSON, using the 'body' variable
+      const items = body.data;
+
+      let nitems = items.map((item) => {
+        let nitem = {}
+
+        if (type == 'activities_translations') {
+          nitem.locale = item.languages_code
+          nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}`
+          nitem.id = item.activities_id?.id
+          nitem.title = item.title
+          nitem.activities_title = item.activities_id?.title
+          nitem.date = item.activities_id?.date;
+          nitem.activity = item.activities_id?.activity;
+          
+          nitem.thumbnail = !isUndefined(item.activities_id?.thumbnail?.filename_disk) ? DIRECTUS_BUCKET + item.activities_id?.thumbnail?.filename_disk : ''
+
+          nitem.video = !isUndefined(item.privateVideo?.filename_disk) ? DIRECTUS_BUCKET + item.privateVideo?.filename_disk : ''
+          nitem.audio = !isUndefined(item.mp3?.filename_disk) ? DIRECTUS_BUCKET + item.mp3?.filename_disk : ''
+          nitem.booklet = !isUndefined(item.pdf_booklet?.filename_disk) ? DIRECTUS_BUCKET + item.pdf_booklet?.filename_disk : ''
+          nitem.simple = !isUndefined(item.pdf?.filename_disk) ? DIRECTUS_BUCKET + item.pdf?.filename_disk : ''
+        }
+
+        if (type == 'conferences_translations') {
+          nitem.locale = item.languages_code
+          nitem.id = item.conferences_id?.id
+          nitem.title = item.title
+          nitem.conferences_title = item.conferences_id?.title
+          nitem.date = item.conferences_id?.date;
+          nitem.activity = item.conferences_id?.activity;
+          
+          nitem.thumbnail = !isUndefined(item.conferences_id?.thumbnail?.filename_disk) ? DIRECTUS_BUCKET + item.conferences_id?.thumbnail?.filename_disk : ''
+
+          nitem.video = !isUndefined(item.video?.filename_disk) ? DIRECTUS_BUCKET + item.video?.filename_disk : ''
+          nitem.audio = !isUndefined(item.audio?.filename_disk) ? DIRECTUS_BUCKET + item.audio?.filename_disk : ''
+          nitem.simple = !isUndefined(item.pdf?.filename_disk) ? DIRECTUS_BUCKET + item.pdf?.filename_disk : ''
+          nitem.booklet = !isUndefined(item.pdf_simple?.filename_disk) ? DIRECTUS_BUCKET + item.pdf_simple?.filename_disk : ''
+        }
+
+        return nitem;
+      })
+      
+      if( nitems.length > 0 ){
+        let csv = parse(nitems);
+        jsonlData += csv
+        writeFile(jsonlData, type)
+        }
+    };
+  });
+
+}
+
+function writeFile(jsonlData, type) {
+  fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_files.csv`, jsonlData, (err) => {
+    if (err) {
+      console.error("Error writing file:", err);
+      return;
+    }
+    console.log("File has been written successfully! : " + type);
+  });
+}
+
+generateFiles( 'conferences_translations' );

carpa_json_to_markdown/index_files_conferences.js

@@ -0,0 +1,86 @@
+
+//import fetch from 'node-fetch'; // For ES Modules
+const fs = require('fs');
+const path = require('path');
+const { parse } = require('json2csv');
+const striptags = require('striptags');
+const dayjs = require('dayjs');
+const he = require('he');
+const jsonTojsonl = require('json-to-jsonl');
+
+const fetch = require('node-fetch'); // For CommonJS
+const { start } = require('repl');
+const request = require('request');
+let { render } = require("mustache");
+
+let jsonlData = '';
+
+const DATA_INPUT_FOLDER = './input';
+const DATA_OUTPUT_FOLDER = './output';
+const TEMPLATES_FOLDER = './templates';
+const DIRECTUS_BUCKET = 'https://ewr1.vultrobjects.com/lgcc-conferences/'
+const LOCALE = 'pt'
+
+function isUndefined(variable){
+    if( typeof(variable) === 'undefined'){
+        return true
+    } else {
+        return false
+    }
+}
+
+async function generateFiles( type ) {
+  let options = { json: true };
+
+  let url = `http://conferences.local:10003/v3/conferencias/?f=id_list&cache=true&locale=${LOCALE}`;
+
+  request(url, options, (error, res, body) => {
+    if (error) {
+      return console.log(error)
+    };
+
+    if (!error && res.statusCode == 200) {
+      // do something with JSON, using the 'body' variable
+      const items = body.result.conferencias;
+
+      let nitems = items.map((item) => {
+        let nitem = {}
+
+        nitem.locale = LOCALE
+        nitem.code = `${LOCALE}-${dayjs(item.date).format('YYYYMMDD')}-${item.activity}`
+        nitem.id = item.wpid
+        nitem.title = item.title
+        nitem.date = item.date;
+        nitem.activity = item.activity;
+        
+        nitem.thumbnail = !isUndefined(item.thumbnail) ? DIRECTUS_BUCKET + item.thumbnail : ''
+
+        nitem.video = !isUndefined(item.video) ? item.video : ''
+        nitem.audio = !isUndefined(item.audio) ? item.audio : ''
+        nitem.booklet = !isUndefined(item.pdf) ? item.pdf : ''
+        nitem.simple = !isUndefined(item.pdf_simple) ? item.pdf_simple : ''
+      
+        return nitem;
+      })
+      
+      if( nitems.length > 0 ){
+        let csv = parse(nitems);
+        jsonlData += csv
+        writeFile(jsonlData, type)
+        }
+    };
+  });
+
+}
+
+function writeFile(jsonlData, type) {
+  fs.writeFile(`./${DATA_INPUT_FOLDER}/conferences_${LOCALE}_files.csv`, jsonlData, (err) => {
+    if (err) {
+      console.error("Error writing file:", err);
+      return;
+    }
+    console.log("File has been written successfully! : " + type);
+  });
+}
+
+generateFiles( 'conferences' );