Node File creator
This commit is contained in:
Julio Ruiz
parent
e83122945a
commit
c9a0f35ac1
|
|
@ -0,0 +1,143 @@
|
|||
|
||||
//import fetch from 'node-fetch'; // For ES Modules
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { parse } = require('json2csv');
|
||||
const striptags = require('striptags');
|
||||
const dayjs = require('dayjs');
|
||||
const he = require('he');
|
||||
const jsonTojsonl = require('json-to-jsonl');
|
||||
|
||||
const fetch = require('node-fetch'); // For CommonJS
|
||||
const { start } = require('repl');
|
||||
const request = require('request');
|
||||
let { render } = require("mustache");
|
||||
|
||||
let jsonlData = '';
|
||||
|
||||
const DATA_INPUT_FOLDER = './input';
|
||||
const DATA_OUTPUT_FOLDER = './output';
|
||||
const TEMPLATES_FOLDER = './templates';
|
||||
const DIRECTUS_BUCKET = 'https://sjc1.vultrobjects.com/directus/'
|
||||
|
||||
function isUndefined(variable){
|
||||
if( typeof(variable) === 'undefined'){
|
||||
return true
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function generateFiles( type ) {
|
||||
let options = { json: true };
|
||||
|
||||
let fields = [];
|
||||
|
||||
fields['activities_translations'] = [
|
||||
"id",
|
||||
"title",
|
||||
"languages_code",
|
||||
"mp3.filename_disk",
|
||||
"mp3.filename_download",
|
||||
"pdf_booklet.filename_disk",
|
||||
"pdf_booklet.filename_download",
|
||||
"pdf.filename_disk",
|
||||
"pdf.filename_download",
|
||||
"activities_id.id",
|
||||
"activities_id.title",
|
||||
"activities_id.date",
|
||||
"activities_id.activity",
|
||||
"activities_id.thumbnail.filename_disk",
|
||||
"activities_id.thumbnail.filename_download"
|
||||
]
|
||||
|
||||
fields['conferences_translations'] = [
|
||||
"id",
|
||||
"title",
|
||||
"conferences_id.title",
|
||||
"conferences_id.date",
|
||||
"conferences_id.activity",
|
||||
"languages_code",
|
||||
"video.filename_disk",
|
||||
"video.filename_download",
|
||||
"audio.filename_disk",
|
||||
"audio.filename_download",
|
||||
"pdf.filename_disk",
|
||||
"pdf.filename_download",
|
||||
"pdf_simple.filename_disk",
|
||||
"pdf_simple.filename_download",
|
||||
"conferences_id.thumbnail.filename_disk",
|
||||
"conferences_id.thumbnail.filename_download"
|
||||
]
|
||||
|
||||
let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=100000&sort`;
|
||||
|
||||
request(url, options, (error, res, body) => {
|
||||
if (error) {
|
||||
return console.log(error)
|
||||
};
|
||||
|
||||
if (!error && res.statusCode == 200) {
|
||||
// do something with JSON, using the 'body' variable
|
||||
const items = body.data;
|
||||
|
||||
let nitems = items.map((item) => {
|
||||
let nitem = {}
|
||||
|
||||
if (type == 'activities_translations') {
|
||||
nitem.locale = item.languages_code
|
||||
nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}`
|
||||
nitem.id = item.activities_id?.id
|
||||
nitem.title = item.title
|
||||
nitem.activities_title = item.activities_id?.title
|
||||
nitem.date = item.activities_id?.date;
|
||||
nitem.activity = item.activities_id?.activity;
|
||||
|
||||
nitem.thumbnail = !isUndefined(item.activities_id?.thumbnail?.filename_disk) ? DIRECTUS_BUCKET + item.activities_id?.thumbnail?.filename_disk : ''
|
||||
|
||||
nitem.video = !isUndefined(item.privateVideo?.filename_disk) ? DIRECTUS_BUCKET + item.privateVideo?.filename_disk : ''
|
||||
nitem.audio = !isUndefined(item.mp3?.filename_disk) ? DIRECTUS_BUCKET + item.mp3?.filename_disk : ''
|
||||
nitem.booklet = !isUndefined(item.pdf_booklet?.filename_disk) ? DIRECTUS_BUCKET + item.pdf_booklet?.filename_disk : ''
|
||||
nitem.simple = !isUndefined(item.pdf?.filename_disk) ? DIRECTUS_BUCKET + item.pdf?.filename_disk : ''
|
||||
}
|
||||
|
||||
if (type == 'conferences_translations') {
|
||||
nitem.locale = item.languages_code
|
||||
nitem.id = item.conferences_id?.id
|
||||
nitem.title = item.title
|
||||
nitem.conferences_title = item.conferences_id?.title
|
||||
nitem.date = item.conferences_id?.date;
|
||||
nitem.activity = item.conferences_id?.activity;
|
||||
|
||||
nitem.thumbnail = !isUndefined(item.conferences_id?.thumbnail?.filename_disk) ? DIRECTUS_BUCKET + item.conferences_id?.thumbnail?.filename_disk : ''
|
||||
|
||||
nitem.video = !isUndefined(item.video?.filename_disk) ? DIRECTUS_BUCKET + item.video?.filename_disk : ''
|
||||
nitem.audio = !isUndefined(item.audio?.filename_disk) ? DIRECTUS_BUCKET + item.audio?.filename_disk : ''
|
||||
nitem.simple = !isUndefined(item.pdf?.filename_disk) ? DIRECTUS_BUCKET + item.pdf?.filename_disk : ''
|
||||
nitem.booklet = !isUndefined(item.pdf_simple?.filename_disk) ? DIRECTUS_BUCKET + item.pdf_simple?.filename_disk : ''
|
||||
}
|
||||
|
||||
return nitem;
|
||||
})
|
||||
|
||||
if( nitems.length > 0 ){
|
||||
let csv = parse(nitems);
|
||||
jsonlData += csv
|
||||
writeFile(jsonlData, type)
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
function writeFile(jsonlData, type) {
|
||||
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_files.csv`, jsonlData, (err) => {
|
||||
if (err) {
|
||||
console.error("Error writing file:", err);
|
||||
return;
|
||||
}
|
||||
console.log("File has been written successfully! : " + type);
|
||||
});
|
||||
}
|
||||
|
||||
generateFiles( 'conferences_translations' );
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
|
||||
//import fetch from 'node-fetch'; // For ES Modules
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { parse } = require('json2csv');
|
||||
const striptags = require('striptags');
|
||||
const dayjs = require('dayjs');
|
||||
const he = require('he');
|
||||
const jsonTojsonl = require('json-to-jsonl');
|
||||
|
||||
const fetch = require('node-fetch'); // For CommonJS
|
||||
const { start } = require('repl');
|
||||
const request = require('request');
|
||||
let { render } = require("mustache");
|
||||
|
||||
let jsonlData = '';
|
||||
|
||||
const DATA_INPUT_FOLDER = './input';
|
||||
const DATA_OUTPUT_FOLDER = './output';
|
||||
const TEMPLATES_FOLDER = './templates';
|
||||
const DIRECTUS_BUCKET = 'https://ewr1.vultrobjects.com/lgcc-conferences/'
|
||||
const LOCALE = 'pt'
|
||||
|
||||
function isUndefined(variable){
|
||||
if( typeof(variable) === 'undefined'){
|
||||
return true
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function generateFiles( type ) {
|
||||
let options = { json: true };
|
||||
|
||||
let url = `http://conferences.local:10003/v3/conferencias/?f=id_list&cache=true&locale=${LOCALE}`;
|
||||
|
||||
request(url, options, (error, res, body) => {
|
||||
if (error) {
|
||||
return console.log(error)
|
||||
};
|
||||
|
||||
if (!error && res.statusCode == 200) {
|
||||
// do something with JSON, using the 'body' variable
|
||||
const items = body.result.conferencias;
|
||||
|
||||
let nitems = items.map((item) => {
|
||||
let nitem = {}
|
||||
|
||||
nitem.locale = LOCALE
|
||||
nitem.code = `${LOCALE}-${dayjs(item.date).format('YYYYMMDD')}-${item.activity}`
|
||||
nitem.id = item.wpid
|
||||
nitem.title = item.title
|
||||
nitem.date = item.date;
|
||||
nitem.activity = item.activity;
|
||||
|
||||
nitem.thumbnail = !isUndefined(item.thumbnail) ? DIRECTUS_BUCKET + item.thumbnail : ''
|
||||
|
||||
nitem.video = !isUndefined(item.video) ? item.video : ''
|
||||
nitem.audio = !isUndefined(item.audio) ? item.audio : ''
|
||||
nitem.booklet = !isUndefined(item.pdf) ? item.pdf : ''
|
||||
nitem.simple = !isUndefined(item.pdf_simple) ? item.pdf_simple : ''
|
||||
|
||||
return nitem;
|
||||
})
|
||||
|
||||
if( nitems.length > 0 ){
|
||||
let csv = parse(nitems);
|
||||
jsonlData += csv
|
||||
writeFile(jsonlData, type)
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
function writeFile(jsonlData, type) {
|
||||
fs.writeFile(`./${DATA_INPUT_FOLDER}/conferences_${LOCALE}_files.csv`, jsonlData, (err) => {
|
||||
if (err) {
|
||||
console.error("Error writing file:", err);
|
||||
return;
|
||||
}
|
||||
console.log("File has been written successfully! : " + type);
|
||||
});
|
||||
}
|
||||
|
||||
generateFiles( 'conferences' );
|
||||
Loading…
Reference in New Issue