diff --git a/carpa_json_to_markdown/index.js b/carpa_json_to_markdown/index.js index 69e280b..8c289f5 100644 --- a/carpa_json_to_markdown/index.js +++ b/carpa_json_to_markdown/index.js @@ -13,6 +13,8 @@ const { start } = require('repl'); const request = require('request'); let { render } = require("mustache"); +let jsonlData = ''; + const DATA_INPUT_FOLDER = './input'; const DATA_OUTPUT_FOLDER = './output'; const TEMPLATES_FOLDER = './templates'; @@ -102,6 +104,13 @@ async function generateJson( type, year ) { 'duration', 'translations.text_published', 'public', + 'thumbnail.filename_disk', + 'translations.slug', + 'translations.youtube', + 'translations.video.filename_disk', + 'translations.audio.filename_disk', + 'translations.pdf.filename_disk', + 'translations.pdf_simple.filename_disk', ] fields['activities'] = [ @@ -116,10 +125,18 @@ async function generateJson( type, year ) { 'duration', 'translations.interventions.text', 'private', + 'thumbnail.filename_disk', + 'translations.slug', + 'translations.youtube', + 'translations.privateVideo.filename_disk', + 'translations.mp3.filename_disk', + 'translations.pdf_booklet.filename_disk', + 'translations.pdf.filename_disk', ] let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&deep[translations][_filter][languages_code][_eq]=es&filter[year(date)]=${year}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=1000&sort=-date`; + console.log( url ) request(url, options, (error, res, body) => { if (error) { @@ -134,15 +151,28 @@ async function generateJson( type, year ) { if(type=='activities'){ item.body = he.decode(striptags(item.translations[0]?.interventions[0]?.text)) || '' item.private = item.private == 1 ? true : false; + item.files = {} + item.files.youtube = item.translations[0]?.youtube + item.files.video = item.translations[0]?.privateVideo?.filename_disk + item.files.audio = item.translations[0]?.mp3?.filename_disk + item.files.booklet = item.translations[0]?.pdf_booklet?.filename_disk + item.files.simple = item.translations[0]?.pdf?.filename_disk } if(type=='conferences'){ item.id = item.id.toString(); item.body = he.decode(striptags(item.translations[0]?.text_published)) || '' item.private = item.public == 0 ? true : false; + item.files = {} + item.files.youtube = item.translations[0]?.youtube + item.files.video = item.translations[0]?.video?.filename_disk + item.files.audio = item.translations[0]?.audio?.filename_disk + item.files.booklet = item.translations[0]?.pdf?.filename_disk + item.files.simple = item.translations[0]?.pdf_simple?.filename_disk delete item.public } - + + item.slug = item.translations[0]?.slug item.place = item.place || null; item.city = item.city || null; item.state = item.state || null; @@ -150,11 +180,18 @@ async function generateJson( type, year ) { item.duration = item.duration ?? 0; - item.year = dayjs(item.date).year() - item.month = dayjs(item.date).month()+1 + item.year = dayjs(item.date).year().toString() + item.month = item.year + " > " + (dayjs(item.date).month()+1).toString().padStart(2,"0") + + item.menu_lvl0 = item.country + item.menu_lvl1 = item.country + " > " + item.state + item.menu_lvl2 = item.country + " > " + item.state + " > " + item.city + item.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place item.date = dayjs(item.date).unix() + item.thumbnail = item.thumbnail?.filename_disk + delete item.translations; return item; }) @@ -163,20 +200,24 @@ async function generateJson( type, year ) { fs.mkdirSync(`./${DATA_INPUT_FOLDER}/${type}`); } - const jsonlData = items.map(JSON.stringify).join('\n'); + jsonlData += items.map(JSON.stringify).join('\n') - fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}/${year}.json`, jsonlData, (err) => { - if (err) { - console.error("Error writing file:", err); - return; - } - console.log("File has been written successfully! : " + year); - }); + writeFile( jsonlData , type ) }; }); } +function writeFile( jsonlData, type ){ + fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}.json`, jsonlData, (err) => { + if (err) { + console.error("Error writing file:", err); + return; + } + console.log("File has been written successfully! : " + type); + }); +} + //generateJson( 'conferences', 1974); //generateJson( 'activities' , 2021 ); @@ -187,10 +228,10 @@ async function generateJson( type, year ) { //generateMarkdown( 'activities' ); //generateMarkdown( 'conferences' ); -for( let year = 1974; year < 2019; year++){ - generateJson( 'conferences', year ); -} +// for( let year = 1974; year < 2019; year++){ +// generateJson( 'conferences', year ); +// } -for( let year = 2021; year < 2026; year++){ +for( let year = 2021; year < 2027; year++){ generateJson( 'activities', year ); } \ No newline at end of file diff --git a/typesense/docker-compose.yml b/typesense/docker-compose.yml index 84a6de0..eb4eac8 100644 --- a/typesense/docker-compose.yml +++ b/typesense/docker-compose.yml @@ -1,18 +1,17 @@ -version: "3" services: - typesense: - image: typesense/typesense:0.25.0.rc36 - entrypoint: sh -c "/opt/typesense-server --data-dir /data --api-key=abc --enable-cors" - ports: - - "8108:8108" - volumes: - - typesense-data:/data + # typesense: + # image: typesense/typesense:0.29 + # entrypoint: sh -c "/opt/typesense-server --data-dir /data --api-key=abc --enable-cors" + # ports: + # - "8108:8108" + # volumes: + # - typesense-data:/data admin_dashboard: container_name: admin_dashboard image: ghcr.io/lewynation/typesense-admin-dashboard:latest restart: always ports: - 3005:3000 -volumes: - typesense-data: - driver: local \ No newline at end of file +# volumes: +# typesense-data: +# driver: local \ No newline at end of file diff --git a/typesense/schema/activities_base.json b/typesense/schema/activities_base.json index 807035e..eab3fbf 100644 --- a/typesense/schema/activities_base.json +++ b/typesense/schema/activities_base.json @@ -66,13 +66,79 @@ }, { "name": "year", - "type": "int32", + "type": "string", "facet":true }, { "name": "month", - "type": "int32", + "type": "string", "facet":true + }, + { + "name": "menu_lvl0", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "menu_lvl1", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "menu_lvl2", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "menu_lvl3", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "thumbnail", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "slug", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "youtube", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "video", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "audio", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "booklet", + "type": "string", + "facet":true, + "optional": true + }, + { + "name": "simple", + "type": "string", + "facet":true, + "optional": true } ], "default_sorting_field": "date"