Fixed indexer to produce one file per type.
Added thumbnails, slugs, files into the search index.
This commit is contained in:
Julio Ruiz
parent
5eefb59612
commit
2faede3348
|
|
@ -13,6 +13,8 @@ const { start } = require('repl');
|
|||
const request = require('request');
|
||||
let { render } = require("mustache");
|
||||
|
||||
let jsonlData = '';
|
||||
|
||||
const DATA_INPUT_FOLDER = './input';
|
||||
const DATA_OUTPUT_FOLDER = './output';
|
||||
const TEMPLATES_FOLDER = './templates';
|
||||
|
|
@ -102,6 +104,13 @@ async function generateJson( type, year ) {
|
|||
'duration',
|
||||
'translations.text_published',
|
||||
'public',
|
||||
'thumbnail.filename_disk',
|
||||
'translations.slug',
|
||||
'translations.youtube',
|
||||
'translations.video.filename_disk',
|
||||
'translations.audio.filename_disk',
|
||||
'translations.pdf.filename_disk',
|
||||
'translations.pdf_simple.filename_disk',
|
||||
]
|
||||
|
||||
fields['activities'] = [
|
||||
|
|
@ -116,10 +125,18 @@ async function generateJson( type, year ) {
|
|||
'duration',
|
||||
'translations.interventions.text',
|
||||
'private',
|
||||
'thumbnail.filename_disk',
|
||||
'translations.slug',
|
||||
'translations.youtube',
|
||||
'translations.privateVideo.filename_disk',
|
||||
'translations.mp3.filename_disk',
|
||||
'translations.pdf_booklet.filename_disk',
|
||||
'translations.pdf.filename_disk',
|
||||
]
|
||||
|
||||
let url = `http://directus.carpa.com/items/${type}?fields=${fields[type].join(",")}&deep[translations][_filter][languages_code][_eq]=es&filter[year(date)]=${year}&access_token=dUILDpE5gV224XqOB5xUTzE69sk8VSOL&limit=1000&sort=-date`;
|
||||
|
||||
console.log( url )
|
||||
|
||||
request(url, options, (error, res, body) => {
|
||||
if (error) {
|
||||
|
|
@ -134,15 +151,28 @@ async function generateJson( type, year ) {
|
|||
if(type=='activities'){
|
||||
item.body = he.decode(striptags(item.translations[0]?.interventions[0]?.text)) || ''
|
||||
item.private = item.private == 1 ? true : false;
|
||||
item.files = {}
|
||||
item.files.youtube = item.translations[0]?.youtube
|
||||
item.files.video = item.translations[0]?.privateVideo?.filename_disk
|
||||
item.files.audio = item.translations[0]?.mp3?.filename_disk
|
||||
item.files.booklet = item.translations[0]?.pdf_booklet?.filename_disk
|
||||
item.files.simple = item.translations[0]?.pdf?.filename_disk
|
||||
}
|
||||
|
||||
if(type=='conferences'){
|
||||
item.id = item.id.toString();
|
||||
item.body = he.decode(striptags(item.translations[0]?.text_published)) || ''
|
||||
item.private = item.public == 0 ? true : false;
|
||||
item.files = {}
|
||||
item.files.youtube = item.translations[0]?.youtube
|
||||
item.files.video = item.translations[0]?.video?.filename_disk
|
||||
item.files.audio = item.translations[0]?.audio?.filename_disk
|
||||
item.files.booklet = item.translations[0]?.pdf?.filename_disk
|
||||
item.files.simple = item.translations[0]?.pdf_simple?.filename_disk
|
||||
delete item.public
|
||||
}
|
||||
|
||||
|
||||
item.slug = item.translations[0]?.slug
|
||||
item.place = item.place || null;
|
||||
item.city = item.city || null;
|
||||
item.state = item.state || null;
|
||||
|
|
@ -150,11 +180,18 @@ async function generateJson( type, year ) {
|
|||
|
||||
item.duration = item.duration ?? 0;
|
||||
|
||||
item.year = dayjs(item.date).year()
|
||||
item.month = dayjs(item.date).month()+1
|
||||
item.year = dayjs(item.date).year().toString()
|
||||
item.month = item.year + " > " + (dayjs(item.date).month()+1).toString().padStart(2,"0")
|
||||
|
||||
item.menu_lvl0 = item.country
|
||||
item.menu_lvl1 = item.country + " > " + item.state
|
||||
item.menu_lvl2 = item.country + " > " + item.state + " > " + item.city
|
||||
item.menu_lvl3 = item.country + " > " + item.state + " > " + item.city + " > " + item.place
|
||||
|
||||
item.date = dayjs(item.date).unix()
|
||||
|
||||
item.thumbnail = item.thumbnail?.filename_disk
|
||||
|
||||
delete item.translations;
|
||||
return item;
|
||||
})
|
||||
|
|
@ -163,20 +200,24 @@ async function generateJson( type, year ) {
|
|||
fs.mkdirSync(`./${DATA_INPUT_FOLDER}/${type}`);
|
||||
}
|
||||
|
||||
const jsonlData = items.map(JSON.stringify).join('\n');
|
||||
jsonlData += items.map(JSON.stringify).join('\n')
|
||||
|
||||
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}/${year}.json`, jsonlData, (err) => {
|
||||
if (err) {
|
||||
console.error("Error writing file:", err);
|
||||
return;
|
||||
}
|
||||
console.log("File has been written successfully! : " + year);
|
||||
});
|
||||
writeFile( jsonlData , type )
|
||||
};
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
function writeFile( jsonlData, type ){
|
||||
fs.writeFile(`./${DATA_INPUT_FOLDER}/${type}.json`, jsonlData, (err) => {
|
||||
if (err) {
|
||||
console.error("Error writing file:", err);
|
||||
return;
|
||||
}
|
||||
console.log("File has been written successfully! : " + type);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
//generateJson( 'conferences', 1974);
|
||||
//generateJson( 'activities' , 2021 );
|
||||
|
|
@ -187,10 +228,10 @@ async function generateJson( type, year ) {
|
|||
//generateMarkdown( 'activities' );
|
||||
|
||||
//generateMarkdown( 'conferences' );
|
||||
for( let year = 1974; year < 2019; year++){
|
||||
generateJson( 'conferences', year );
|
||||
}
|
||||
// for( let year = 1974; year < 2019; year++){
|
||||
// generateJson( 'conferences', year );
|
||||
// }
|
||||
|
||||
for( let year = 2021; year < 2026; year++){
|
||||
for( let year = 2021; year < 2027; year++){
|
||||
generateJson( 'activities', year );
|
||||
}
|
||||
|
|
@ -1,18 +1,17 @@
|
|||
version: "3"
|
||||
services:
|
||||
typesense:
|
||||
image: typesense/typesense:0.25.0.rc36
|
||||
entrypoint: sh -c "/opt/typesense-server --data-dir /data --api-key=abc --enable-cors"
|
||||
ports:
|
||||
- "8108:8108"
|
||||
volumes:
|
||||
- typesense-data:/data
|
||||
# typesense:
|
||||
# image: typesense/typesense:0.29
|
||||
# entrypoint: sh -c "/opt/typesense-server --data-dir /data --api-key=abc --enable-cors"
|
||||
# ports:
|
||||
# - "8108:8108"
|
||||
# volumes:
|
||||
# - typesense-data:/data
|
||||
admin_dashboard:
|
||||
container_name: admin_dashboard
|
||||
image: ghcr.io/lewynation/typesense-admin-dashboard:latest
|
||||
restart: always
|
||||
ports:
|
||||
- 3005:3000
|
||||
volumes:
|
||||
typesense-data:
|
||||
driver: local
|
||||
# volumes:
|
||||
# typesense-data:
|
||||
# driver: local
|
||||
|
|
@ -66,13 +66,79 @@
|
|||
},
|
||||
{
|
||||
"name": "year",
|
||||
"type": "int32",
|
||||
"type": "string",
|
||||
"facet":true
|
||||
},
|
||||
{
|
||||
"name": "month",
|
||||
"type": "int32",
|
||||
"type": "string",
|
||||
"facet":true
|
||||
},
|
||||
{
|
||||
"name": "menu_lvl0",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "menu_lvl1",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "menu_lvl2",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "menu_lvl3",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "thumbnail",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "slug",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "youtube",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "video",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "booklet",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "simple",
|
||||
"type": "string",
|
||||
"facet":true,
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"default_sorting_field": "date"
|
||||
|
|
|
|||
Loading…
Reference in New Issue