diff --git a/carpa_json_to_markdown/index_files.js b/carpa_json_to_markdown/index_files.js new file mode 100644 index 0000000..5e84a4b --- /dev/null +++ b/carpa_json_to_markdown/index_files.js @@ -0,0 +1,143 @@ + +//import fetch from 'node-fetch'; // For ES Modules +const fs = require('fs'); +const path = require('path'); +const { parse } = require('json2csv'); +const striptags = require('striptags'); +const dayjs = require('dayjs'); +const he = require('he'); +const jsonTojsonl = require('json-to-jsonl'); + +const fetch = require('node-fetch'); // For CommonJS +const { start } = require('repl'); +const request = require('request'); +let { render } = require("mustache"); + +let jsonlData = ''; + +const DATA_INPUT_FOLDER = './input'; +const DATA_OUTPUT_FOLDER = './output'; +const TEMPLATES_FOLDER = './templates'; +const DIRECTUS_BUCKET = 'https://sjc1.vultrobjects.com/directus/' + +function isUndefined(variable){ + if( typeof(variable) === 'undefined'){ + return true + } else { + return false + } +} + +async function generateFiles( type ) { + let options = { json: true }; + + let fields = []; + + fields['activities_translations'] = [ + "id", + "title", + "languages_code", + "mp3.filename_disk", + "mp3.filename_download", + "pdf_booklet.filename_disk", + "pdf_booklet.filename_download", + "pdf.filename_disk", + "pdf.filename_download", + "activities_id.id", + "activities_id.title", + "activities_id.date", + "activities_id.activity", + "activities_id.thumbnail.filename_disk", + "activities_id.thumbnail.filename_download" + ] + + fields['conferences_translations'] = [ + "id", + "title", + "conferences_id.title", + "conferences_id.date", + "conferences_id.activity", + "languages_code", + "video.filename_disk", + "video.filename_download", + "audio.filename_disk", + "audio.filename_download", + "pdf.filename_disk", + "pdf.filename_download", + "pdf_simple.filename_disk", + "pdf_simple.filename_download", + "conferences_id.thumbnail.filename_disk", + "conferences_id.thumbnail.filename_download" + ] + + let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=100000&sort`; + + request(url, options, (error, res, body) => { + if (error) { + return console.log(error) + }; + + if (!error && res.statusCode == 200) { + // do something with JSON, using the 'body' variable + const items = body.data; + + let nitems = items.map((item) => { + let nitem = {} + + if (type == 'activities_translations') { + nitem.locale = item.languages_code + nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}` + nitem.id = item.activities_id?.id + nitem.title = item.title + nitem.activities_title = item.activities_id?.title + nitem.date = item.activities_id?.date; + nitem.activity = item.activities_id?.activity; + + nitem.thumbnail = !isUndefined(item.activities_id?.thumbnail?.filename_disk) ? DIRECTUS_BUCKET + item.activities_id?.thumbnail?.filename_disk : '' + + nitem.video = !isUndefined(item.privateVideo?.filename_disk) ? DIRECTUS_BUCKET + item.privateVideo?.filename_disk : '' + nitem.audio = !isUndefined(item.mp3?.filename_disk) ? DIRECTUS_BUCKET + item.mp3?.filename_disk : '' + nitem.booklet = !isUndefined(item.pdf_booklet?.filename_disk) ? DIRECTUS_BUCKET + item.pdf_booklet?.filename_disk : '' + nitem.simple = !isUndefined(item.pdf?.filename_disk) ? DIRECTUS_BUCKET + item.pdf?.filename_disk : '' + } + + if (type == 'conferences_translations') { + nitem.locale = item.languages_code + nitem.id = item.conferences_id?.id + nitem.title = item.title + nitem.conferences_title = item.conferences_id?.title + nitem.date = item.conferences_id?.date; + nitem.activity = item.conferences_id?.activity; + + nitem.thumbnail = !isUndefined(item.conferences_id?.thumbnail?.filename_disk) ? DIRECTUS_BUCKET + item.conferences_id?.thumbnail?.filename_disk : '' + + nitem.video = !isUndefined(item.video?.filename_disk) ? DIRECTUS_BUCKET + item.video?.filename_disk : '' + nitem.audio = !isUndefined(item.audio?.filename_disk) ? DIRECTUS_BUCKET + item.audio?.filename_disk : '' + nitem.simple = !isUndefined(item.pdf?.filename_disk) ? DIRECTUS_BUCKET + item.pdf?.filename_disk : '' + nitem.booklet = !isUndefined(item.pdf_simple?.filename_disk) ? DIRECTUS_BUCKET + item.pdf_simple?.filename_disk : '' + } + + return nitem; + }) + + if( nitems.length > 0 ){ + let csv = parse(nitems); + jsonlData += csv + writeFile(jsonlData, type) + } + }; + }); + +} + +function writeFile(jsonlData, type) { + fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_files.csv`, jsonlData, (err) => { + if (err) { + console.error("Error writing file:", err); + return; + } + console.log("File has been written successfully! : " + type); + }); +} + +generateFiles( 'conferences_translations' ); \ No newline at end of file diff --git a/carpa_json_to_markdown/index_files_conferences.js b/carpa_json_to_markdown/index_files_conferences.js new file mode 100644 index 0000000..c58c137 --- /dev/null +++ b/carpa_json_to_markdown/index_files_conferences.js @@ -0,0 +1,86 @@ + +//import fetch from 'node-fetch'; // For ES Modules +const fs = require('fs'); +const path = require('path'); +const { parse } = require('json2csv'); +const striptags = require('striptags'); +const dayjs = require('dayjs'); +const he = require('he'); +const jsonTojsonl = require('json-to-jsonl'); + +const fetch = require('node-fetch'); // For CommonJS +const { start } = require('repl'); +const request = require('request'); +let { render } = require("mustache"); + +let jsonlData = ''; + +const DATA_INPUT_FOLDER = './input'; +const DATA_OUTPUT_FOLDER = './output'; +const TEMPLATES_FOLDER = './templates'; +const DIRECTUS_BUCKET = 'https://ewr1.vultrobjects.com/lgcc-conferences/' +const LOCALE = 'pt' + +function isUndefined(variable){ + if( typeof(variable) === 'undefined'){ + return true + } else { + return false + } +} + +async function generateFiles( type ) { + let options = { json: true }; + + let url = `http://conferences.local:10003/v3/conferencias/?f=id_list&cache=true&locale=${LOCALE}`; + + request(url, options, (error, res, body) => { + if (error) { + return console.log(error) + }; + + if (!error && res.statusCode == 200) { + // do something with JSON, using the 'body' variable + const items = body.result.conferencias; + + let nitems = items.map((item) => { + let nitem = {} + + nitem.locale = LOCALE + nitem.code = `${LOCALE}-${dayjs(item.date).format('YYYYMMDD')}-${item.activity}` + nitem.id = item.wpid + nitem.title = item.title + nitem.date = item.date; + nitem.activity = item.activity; + + nitem.thumbnail = !isUndefined(item.thumbnail) ? DIRECTUS_BUCKET + item.thumbnail : '' + + nitem.video = !isUndefined(item.video) ? item.video : '' + nitem.audio = !isUndefined(item.audio) ? item.audio : '' + nitem.booklet = !isUndefined(item.pdf) ? item.pdf : '' + nitem.simple = !isUndefined(item.pdf_simple) ? item.pdf_simple : '' + + return nitem; + }) + + if( nitems.length > 0 ){ + let csv = parse(nitems); + jsonlData += csv + writeFile(jsonlData, type) + } + }; + }); + +} + +function writeFile(jsonlData, type) { + fs.writeFile(`./${DATA_INPUT_FOLDER}/conferences_${LOCALE}_files.csv`, jsonlData, (err) => { + if (err) { + console.error("Error writing file:", err); + return; + } + console.log("File has been written successfully! : " + type); + }); +} + +generateFiles( 'conferences' ); \ No newline at end of file