391 lines
12 KiB
JavaScript
391 lines
12 KiB
JavaScript
|
|
//import fetch from 'node-fetch'; // For ES Modules
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const { parse } = require('json2csv');
|
|
const striptags = require('striptags');
|
|
const dayjs = require('dayjs');
|
|
const he = require('he');
|
|
const jsonTojsonl = require('json-to-jsonl');
|
|
|
|
const fetch = require('node-fetch'); // For CommonJS
|
|
const { start } = require('repl');
|
|
const request = require('request');
|
|
let { render } = require("mustache");
|
|
|
|
let jsonlData = '';
|
|
|
|
const DATA_INPUT_FOLDER = './input';
|
|
const DATA_OUTPUT_FOLDER = './output';
|
|
const TEMPLATES_FOLDER = './templates';
|
|
const DIRECTUS_BUCKET = 'https://sjc1.vultrobjects.com/directus/'
|
|
const LOCALE = 'es';
|
|
|
|
function isUndefined(variable){
|
|
if( typeof(variable) === 'undefined'){
|
|
return true
|
|
} else {
|
|
return false
|
|
}
|
|
}
|
|
|
|
async function generateFiles( type ) {
|
|
let options = { json: true };
|
|
|
|
let fields = [];
|
|
|
|
fields['activities_translations'] = [
|
|
"id",
|
|
"title",
|
|
"languages_code",
|
|
"mp3.filename_disk",
|
|
"mp3.filename_download",
|
|
"mp3.id",
|
|
"pdf_booklet.filename_disk",
|
|
"pdf_booklet.filename_download",
|
|
"pdf_booklet.id",
|
|
"pdf.filename_disk",
|
|
"pdf.filename_download",
|
|
"pdf.id",
|
|
"activities_id.id",
|
|
"activities_id.title",
|
|
"activities_id.date",
|
|
"activities_id.activity",
|
|
"activities_id.thumbnail.id",
|
|
"activities_id.thumbnail.filename_disk",
|
|
"activities_id.thumbnail.filename_download"
|
|
]
|
|
|
|
fields['conferences_translations'] = [
|
|
"id",
|
|
"title",
|
|
"conferences_id.title",
|
|
"conferences_id.date",
|
|
"conferences_id.activity",
|
|
"languages_code",
|
|
"video.filename_disk",
|
|
"video.filename_download",
|
|
"video.id",
|
|
"audio.filename_disk",
|
|
"audio.filename_download",
|
|
"audio.id",
|
|
"pdf.filename_disk",
|
|
"pdf.filename_download",
|
|
"pdf.id",
|
|
"pdf_simple.filename_disk",
|
|
"pdf_simple.filename_download",
|
|
"pdf_simple.id",
|
|
"conferences_id.thumbnail.filename_disk",
|
|
"conferences_id.thumbnail.filename_download",
|
|
"conferences_id.id"
|
|
]
|
|
|
|
fields['sermons'] = [
|
|
"id",
|
|
"title_jbp",
|
|
"title_spanish",
|
|
"title_english",
|
|
"sermon_code",
|
|
"files.directus_files_id.id",
|
|
"files.directus_files_id.filename_disk",
|
|
"files.directus_files_id.filename_download"
|
|
]
|
|
|
|
fields['clases_biblicas_translations'] = [
|
|
"id",
|
|
"languages_code",
|
|
"title",
|
|
"youtube,slug",
|
|
"featured_image.id",
|
|
"featured_image.filename_disk",
|
|
"featured_image.filename_download",
|
|
"bible_study",
|
|
"slides.directus_files_id.id",
|
|
"slides.directus_files_id.filename_disk",
|
|
"slides.directus_files_id.filename_download",
|
|
"audios.directus_files_id.id",
|
|
"audios.directus_files_id.filename_disk",
|
|
"audios.directus_files_id.filename_download",
|
|
"crafts.directus_files_id.id",
|
|
"crafts.directus_files_id.filename_download",
|
|
"crafts.directus_files_id.filename_disk"
|
|
]
|
|
|
|
fields['materiales_didacticos_translations'] = [
|
|
"id",
|
|
"languages_code",
|
|
"title",
|
|
"featured_image.id",
|
|
"featured_image.filename_disk",
|
|
"featured_image.filename_download",
|
|
"bible_studies.bible_study",
|
|
"priting_url",
|
|
"slug",
|
|
"pdf_simple.directus_files_id.id",
|
|
"pdf_simple.directus_files_id.filename_disk",
|
|
"pdf_simple.directus_files_id.filename_download",
|
|
"pdf_booklet.directus_files_id.id",
|
|
"pdf_booklet.directus_files_id.filename_disk",
|
|
"pdf_booklet.directus_files_id.filename_download"
|
|
]
|
|
|
|
fields['print_materials_translations_1'] = [
|
|
"id",
|
|
"title",
|
|
"slug",
|
|
"slug_personalizado",
|
|
"private",
|
|
"notas",
|
|
"languages_code.code",
|
|
"files.file.id",
|
|
"files.file.filename_disk",
|
|
"files.file.filename_download",
|
|
"files.tipo",
|
|
"files.name",
|
|
"files_images.directus_files_id.id",
|
|
"files_images.directus_files_id.filename_disk",
|
|
"print_materials_id.thumbnail.filename_disk",
|
|
"files_images.directus_files_id.filename_download"
|
|
]
|
|
|
|
fields['ebn'] = [
|
|
"id"
|
|
]
|
|
|
|
let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=100000`;
|
|
|
|
if( type=='sermons' ){
|
|
url += '&filter[files][_gte]=1'
|
|
}
|
|
|
|
console.log( url )
|
|
|
|
request(url, options, (error, res, body) => {
|
|
if (error) {
|
|
return console.log(error)
|
|
};
|
|
|
|
if (!error && res.statusCode == 200) {
|
|
// do something with JSON, using the 'body' variable
|
|
const items = body.data;
|
|
|
|
let fileList = ''
|
|
|
|
let nitems = items.map((item) => {
|
|
let nitem = {}
|
|
|
|
if (type == 'activities_translations') {
|
|
nitem.locale = item.languages_code
|
|
nitem.code = `${item.languages_code}-${dayjs(item.activities_id?.date).format('YYYYMMDD')}-${item.activities_id?.activity}`
|
|
nitem.id = item.activities_id?.id
|
|
nitem.title = item.title
|
|
nitem.activities_title = item.activities_id?.title
|
|
nitem.date = item.activities_id?.date;
|
|
nitem.activity = item.activities_id?.activity;
|
|
|
|
nitem.thumbnail = item.activities_id?.thumbnail?.filename_download
|
|
nitem.thumbnail_file = item.activities_id?.thumbnail?.filename_disk
|
|
nitem.thumbnail_id = item.activities_id?.thumbnail?.id
|
|
nitem.video = item.privateVideo?.filename_download
|
|
nitem.video_file = item.privateVideo?.filename_disk
|
|
nitem.video_id = item.privateVideo?.id
|
|
nitem.audio = item.mp3?.filename_download
|
|
nitem.audio_file = item.mp3?.filename_disk
|
|
nitem.audio_id = item.mp3?.id
|
|
nitem.booklet = item.pdf_booklet?.filename_download
|
|
nitem.booklet_file = item.pdf_booklet?.filename_disk
|
|
nitem.booklet_id = item.pdf_booklet?.id
|
|
nitem.simple = item.pdf?.filename_download
|
|
nitem.simple_file = item.pdf?.filename_disk
|
|
nitem.simple_id = item.pdf?.id
|
|
}
|
|
|
|
if (type == 'conferences_translations') {
|
|
nitem.locale = item.languages_code
|
|
nitem.id = item.conferences_id?.id
|
|
nitem.title = item.title
|
|
nitem.conferences_title = item.conferences_id?.title
|
|
nitem.date = item.conferences_id?.date;
|
|
nitem.activity = item.conferences_id?.activity;
|
|
|
|
nitem.thumbnail = item.conferences_id?.thumbnail?.filename_download
|
|
nitem.thumbnail_file = item.conferences_id?.thumbnail?.filename_disk
|
|
nitem.thumbnail_id = item.conferences_id?.thumbnail?.id
|
|
|
|
nitem.video = item.video?.filename_download
|
|
nitem.video_file = item.video?.filename_disk
|
|
nitem.video_id = item.video?.id
|
|
|
|
nitem.audio = item.audio?.filename_download
|
|
nitem.audio_file = item.audio?.filename_disk
|
|
nitem.audio_id = item.audio?.id
|
|
|
|
nitem.simple = item.pdf?.filename_download
|
|
nitem.simple_file = item.pdf?.filename_disk
|
|
nitem.simple_id = item.pdf?.id
|
|
|
|
nitem.booklet = item.pdf_simple?.filename_download
|
|
nitem.booklet_file = item.pdf_simple?.filename_disk
|
|
nitem.booklet_id = item.pdf_simple?.id
|
|
}
|
|
|
|
if( type=='sermons'){
|
|
nitem.locale = 'es'
|
|
nitem.id = item.id
|
|
nitem.title_jbp = item.title_jbp
|
|
nitem.title_spanish = item.title_spanish
|
|
nitem.title_english = item.title_english
|
|
nitem.sermon_code = item.sermon_code
|
|
nitem.file_url = DIRECTUS_BUCKET + item.files?.[0]?.directus_files_id?.filename_disk
|
|
nitem.file_id = item.files?.[0]?.directus_files_id?.id
|
|
nitem.filename_disk = item.files?.[0]?.directus_files_id?.filename_disk
|
|
nitem.filename_download = item.files?.[0]?.directus_files_id?.filename_download
|
|
}
|
|
|
|
if(type=='clases_biblicas_translations'){
|
|
nitem.id = item.id.toString()
|
|
nitem.locale = item.languages_code
|
|
nitem.title = item.title
|
|
nitem.youtube = item.youtube
|
|
nitem.slug = item.slug
|
|
nitem.featured_image = item.featured_image.filename_disk
|
|
nitem.bible_study = item.bible_study
|
|
nitem.slides = []
|
|
item.slides?.map((file)=>{
|
|
const fileObject = {
|
|
id: file.directus_files_id?.id,
|
|
filename: file.directus_files_id?.filename_disk,
|
|
download: file.directus_files_id?.filename_download
|
|
}
|
|
nitem.slides.push(fileObject)
|
|
})
|
|
nitem.audios = []
|
|
item.audios?.map((file)=>{
|
|
const fileObject = {
|
|
id: file.directus_files_id?.id,
|
|
filename: file.directus_files_id?.filename_disk,
|
|
download: file.directus_files_id?.filename_download
|
|
}
|
|
nitem.audios.push(fileObject)
|
|
})
|
|
nitem.crafts = []
|
|
item.crafts?.map((file)=>{
|
|
const fileObject = {
|
|
id: file.directus_files_id?.id,
|
|
filename: file.directus_files_id?.filename_disk,
|
|
download: file.directus_files_id?.filename_download
|
|
}
|
|
nitem.crafts.push(fileObject)
|
|
})
|
|
}
|
|
|
|
if(type=='materiales_didacticos_translations'){
|
|
nitem.id = item.title
|
|
nitem.locale = item.languages_code
|
|
nitem.featured_image = item.featured_image.filename_disk
|
|
nitem.bible_studies = []
|
|
item.bible_studies?.map((item) => {
|
|
let bstudy = {}
|
|
bstudy.id = item.bible_study
|
|
nitem.bible_studies.push( bstudy )
|
|
})
|
|
nitem.printing_url = item.priting_url
|
|
nitem.slug = item.slug
|
|
nitem.pdf_simple = []
|
|
item.pdf_simple?.map((file)=>{
|
|
const fileObject = {
|
|
id: file.directus_files_id?.id,
|
|
filename: file.directus_files_id?.filename_disk,
|
|
download: file.directus_files_id?.filename_download
|
|
}
|
|
nitem.pdf_simple.push(fileObject)
|
|
})
|
|
nitem.pdf_booklet = []
|
|
item.pdf_booklet?.map((file)=>{
|
|
const fileObject = {
|
|
id: file.directus_files_id?.id,
|
|
filename: file.directus_files_id?.filename_disk,
|
|
download: file.directus_files_id?.filename_download
|
|
}
|
|
nitem.pdf_booklet.push(fileObject)
|
|
})
|
|
}
|
|
|
|
if(type=='print_materials_translations_1'){
|
|
nitem.id = item.id
|
|
nitem.title = item.title
|
|
nitem.slug = item.slug
|
|
nitem.custom_slug = item.slug_personalizado
|
|
nitem.private = item.private
|
|
nitem.notes = item.notas
|
|
nitem.locale = item.languages_code.code
|
|
nitem.files = []
|
|
item.files?.map((file)=>{
|
|
console.log('File',file)
|
|
const fileObject = {
|
|
id: file.file?.id,
|
|
filename: file.file?.filename_disk,
|
|
download: file.file?.filename_download,
|
|
type: file.tipo,
|
|
name: file.name
|
|
}
|
|
nitem.files.push(fileObject)
|
|
})
|
|
nitem.files_images = []
|
|
item.files_images?.map((file)=>{
|
|
const fileObject = {
|
|
id: file.directus_files_id?.id,
|
|
filename: file.directus_files_id?.filename_disk,
|
|
download: file.directus_files_id?.filename_download
|
|
}
|
|
nitem.files_images.push(fileObject)
|
|
})
|
|
nitem.thumbnail = item.print_materials_id?.thumbnail?.filename_disk
|
|
}
|
|
//console.log(fileList)
|
|
return nitem;
|
|
})
|
|
|
|
//writeFile( fileList , type)
|
|
if( nitems.length > 0 ){
|
|
// let csv = parse(nitems);
|
|
// jsonlData += csv
|
|
|
|
jsonlData += nitems.filter(item => item !== false).map(JSON.stringify).join('\n')
|
|
jsonlData += '\n';
|
|
|
|
writeFile(jsonlData, type)
|
|
}
|
|
};
|
|
});
|
|
|
|
}
|
|
|
|
function flattenObject(obj){
|
|
const flattened = {}
|
|
|
|
Object.keys(obj).forEach((key) => {
|
|
const value = obj[key]
|
|
|
|
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
|
Object.assign(flattened, flattenObject(value))
|
|
} else {
|
|
flattened[key] = value
|
|
}
|
|
})
|
|
|
|
return flattened
|
|
}
|
|
|
|
function writeFile(jsonlData, type) {
|
|
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}_files.json`, jsonlData, (err) => {
|
|
if (err) {
|
|
console.error("Error writing file:", err);
|
|
return;
|
|
}
|
|
console.log("File has been written successfully! : " + type);
|
|
});
|
|
}
|
|
|
|
//generateFiles( 'clases_biblicas_translations' );
|
|
generateFiles( 'clases_biblicas_translations' ); |