Fixed indexer to produce one file per type.

Added thumbnails, slugs, files into the search index.

carpa_json_to_markdown/index.js

@@ -13,6 +13,8 @@ const { start } = require('repl');
 const request = require('request');
 let { render } = require("mustache");
 
+let jsonlData = '';
+
 const DATA_INPUT_FOLDER = './input';
 const DATA_OUTPUT_FOLDER = './output';
 const TEMPLATES_FOLDER = './templates';
@@ -102,6 +104,13 @@ async function generateJson( type, year ) {
     'duration',
     'translations.text_published',
     'public',
+    'thumbnail.filename_disk',
+    'translations.slug',
+    'translations.youtube',
+    'translations.video.filename_disk',
+    'translations.audio.filename_disk',
+    'translations.pdf.filename_disk',
+    'translations.pdf_simple.filename_disk',
   ]
 
   fields['activities'] = [
@@ -116,10 +125,18 @@ async function generateJson( type, year ) {
     'duration',
     'translations.interventions.text',
     'private',
+    'thumbnail.filename_disk',
+    'translations.slug',
+    'translations.youtube',
+    'translations.privateVideo.filename_disk',
+    'translations.mp3.filename_disk',
+    'translations.pdf_booklet.filename_disk',
+    'translations.pdf.filename_disk',
   ]
 
   let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&deep[translations][_filter][languages_code][_eq]=es&filter[year(date)]=${year}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=1000&sort=-date`;
 
+  console.log( url )
 
   request(url, options, (error, res, body) => {
     if (error) {
@@ -134,15 +151,28 @@ async function generateJson( type, year ) {
         if(type=='activities'){
           item.body = he.decode(striptags(item.translations[0]?.interventions[0]?.text)) || ''
           item.private = item.private == 1 ? true : false;
+          item.files = {}
+          item.files.youtube = item.translations[0]?.youtube
+          item.files.video = item.translations[0]?.privateVideo?.filename_disk
+          item.files.audio = item.translations[0]?.mp3?.filename_disk
+          item.files.booklet = item.translations[0]?.pdf_booklet?.filename_disk
+          item.files.simple = item.translations[0]?.pdf?.filename_disk
         }
 
         if(type=='conferences'){
           item.id = item.id.toString();
           item.body = he.decode(striptags(item.translations[0]?.text_published)) || ''
           item.private = item.public == 0 ? true : false;
+          item.files = {}
+          item.files.youtube = item.translations[0]?.youtube
+          item.files.video = item.translations[0]?.video?.filename_disk
+          item.files.audio = item.translations[0]?.audio?.filename_disk
+          item.files.booklet = item.translations[0]?.pdf?.filename_disk
+          item.files.simple = item.translations[0]?.pdf_simple?.filename_disk
           delete item.public
         }
-
+        
+        item.slug = item.translations[0]?.slug
         item.place = item.place || null;
         item.city = item.city || null;
         item.state = item.state || null;
@@ -150,11 +180,18 @@ async function generateJson( type, year ) {
 
         item.duration = item.duration ?? 0;
 
-        item.year = dayjs(item.date).year()
+        item.year = dayjs(item.date).year().toString()
-        item.month = dayjs(item.date).month()+1
+        item.month = item.year + " > " + (dayjs(item.date).month()+1).toString().padStart(2,"0")
+
+        item.menu_lvl0 = item.country
+        item.menu_lvl1 = item.country + " > " + item.state
+        item.menu_lvl2 = item.country + " > " + item.state + " > " + item.city
+        item.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place
 
         item.date = dayjs(item.date).unix()
 
+        item.thumbnail = item.thumbnail?.filename_disk
+
         delete item.translations;
         return item;
       })
@@ -163,20 +200,24 @@ async function generateJson( type, year ) {
         fs.mkdirSync(`./${DATA_INPUT_FOLDER}/${type}`);
       }
 
-      const jsonlData = items.map(JSON.stringify).join('\n');
+      jsonlData += items.map(JSON.stringify).join('\n')
 
-      fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}/${year}.json`, jsonlData, (err) => {
+      writeFile( jsonlData , type )
-        if (err) {
-          console.error("Error writing file:", err);
-          return;
-        }
-        console.log("File has been written successfully! : " + year);
-      });
     };
   });
 
 }
 
+function writeFile( jsonlData, type ){
+  fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}.json`, jsonlData, (err) => {
+      if (err) {
+        console.error("Error writing file:", err);
+        return;
+      }
+      console.log("File has been written successfully! : " + type);
+    });
+}
+
 
 //generateJson( 'conferences', 1974);
 //generateJson( 'activities' , 2021 );
@@ -187,10 +228,10 @@ async function generateJson( type, year ) {
 //generateMarkdown( 'activities' );
 
 //generateMarkdown( 'conferences' );
-for( let year = 1974; year < 2019; year++){
+// for( let year = 1974; year < 2019; year++){
-  generateJson( 'conferences', year );
+//   generateJson( 'conferences', year );
-}
+// }
 
-for( let year = 2021; year < 2026; year++){
+for( let year = 2021; year < 2027; year++){
   generateJson( 'activities', year );
 }

typesense/docker-compose.yml

@@ -1,18 +1,17 @@
-version: "3"
 services:
-  typesense:
+  # typesense:
-    image: typesense/typesense:0.25.0.rc36
+  #   image: typesense/typesense:0.29
-    entrypoint: sh -c "/opt/typesense-server --data-dir /data --api-key=abc --enable-cors"
+  #   entrypoint: sh -c "/opt/typesense-server --data-dir /data --api-key=abc --enable-cors"
-    ports:
+  #   ports:
-      - "8108:8108"
+  #     - "8108:8108"
-    volumes:
+  #   volumes:
-      - typesense-data:/data
+  #     - typesense-data:/data
   admin_dashboard:
     container_name: admin_dashboard
     image: ghcr.io/lewynation/typesense-admin-dashboard:latest
     restart: always
     ports:
       - 3005:3000
-volumes:
+# volumes:
-  typesense-data:
+#   typesense-data:
-    driver: local
+#     driver: local

typesense/schema/activities_base.json

@@ -66,13 +66,79 @@
         },
         {
             "name": "year",
-            "type": "int32",
+            "type": "string",
             "facet":true
         },
         {
             "name": "month",
-            "type": "int32",
+            "type": "string",
             "facet":true
+        },
+        {
+            "name": "menu_lvl0",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "menu_lvl1",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "menu_lvl2",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "menu_lvl3",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "thumbnail",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "slug",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "youtube",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "video",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "audio",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "booklet",
+            "type": "string",
+            "facet":true,
+            "optional": true
+        },
+        {
+            "name": "simple",
+            "type": "string",
+            "facet":true,
+            "optional": true
         }
     ],
     "default_sorting_field": "date"