- Estrazione dei dati da Amazon

- Ciclo di Estrapolazione di tutti i prodotti ed aggiornamento dei campi scraped e scraped_updated
- Creazione file CSV con i campi modificati.
This commit is contained in:
Surya Paolo
2025-05-19 17:33:58 +02:00
parent a374a7d7bc
commit a3c7b92c0c
16 changed files with 5131 additions and 1731 deletions

View File

@@ -241,6 +241,12 @@ const productSchema = new Schema({
scraped: {
type: Boolean,
},
scraped_error: {
type: Boolean,
},
scraped_updated: {
type: Boolean,
},
scraped_date: {
type: Date,
},
@@ -476,7 +482,7 @@ module.exports.findAllIdApp = async function (idapp, code, id, all, isbn) {
if (idapp) {
myfind = {
idapp,
$or: [{ delete: { $exists: false } }, { delete: false }],
$or: [{ deleted: { $exists: false } }, { deleted: { $exists: true, $eq: false } }],
};
}

View File

@@ -183,13 +183,6 @@ const productInfoSchema = new Schema({
sottotitolo: String,
link_macro: String,
scraped: {
type: Boolean,
},
scraped_date: {
type: Date,
},
});
var productInfo = module.exports = mongoose.model('ProductInfo', productInfoSchema);

View File

@@ -47,14 +47,27 @@ class CronMod {
const globalTables = require("../tools/globalTables");
const { Reaction } = require("../models/reaction");
const AmazonBookScraper = require('../modules/Scraping');
let mystr = "";
try {
if (mydata.dbop === "") {
// } else if (mydata.dbop === 'rigeneraTutto') {
// await ListaIngresso.Esegui_CronTab(idapp, mydata);
} else if (mydata.dbop === "ScraperDataAmazon") {
await ScraperDataAmazon(idapp, mydata.options)
} else if (mydata.dbop === "ScraperMultipleDataAmazon") {
mystr = await AmazonBookScraper.ScraperMultipleDataAmazon(idapp, mydata.options);
ris = { mystr };
} else if (mydata.dbop === "ScraperGeneraCSV") {
mystr = await AmazonBookScraper.ScraperGeneraCSV(idapp, mydata.options, res);
ris = { mystr };
} else if (mydata.dbop === "removeDuplicateVariations") {
mystr = await AmazonBookScraper.removeDuplicateVariations(idapp, mydata.options);
ris = { mystr };
} else if (mydata.dbop === "ScraperAzzeraFlagProducts") {
mystr = await AmazonBookScraper.ScraperAzzeraFlagProducts(idapp, mydata.options);
ris = { mystr };
} else if (mydata.dbop === "ReplaceAllCircuits") {
// ++ Replace All Circuitname with 'Circuito RIS %s'
await Circuit.replaceAllCircuitNames(idapp);

View File

@@ -0,0 +1,179 @@
const fs = require('fs'); // 👈 Usa il modulo promises
const axios = require('axios');
const path = require('path');
/**
* Scarica un'immagine da una URL e la salva in una directory locale
* @param {string} url - L'URL dell'immagine da scaricare
* @param {string} filepath - Il percorso dove salvare l'immagine scaricata
*/
class ImageDownloader {
/**
* Scarica un'immagine da una URL e la salva in una directory locale.
* Tenta di scaricare fino a 3 volte in caso di errore, con un ritardo tra i tentativi.
*
* @param {string} url - L'URL dell'immagine da scaricare
* @param {string} filepath - Il percorso dove salvare l'immagine scaricata
* @param {number} maxRetries - Numero massimo di tentativi in caso di fallimento (default: 3)
* @param {number} delay - Ritardo in millisecondi tra i tentativi (default: 1000)
* @returns {Promise<boolean>}
*/
async downloadImage(url, filepath, options = {}) {
const {
maxRetries = 3, // Aumentato il numero di tentativi predefiniti
initialDelay = 1000, // Ritardo iniziale
maxDelay = 10000, // Ritardo massimo
timeout = 30000, // Timeout della richiesta
validateContentType = true, // Verifica del tipo di contenuto
nomefileoriginale = true,
} = options;
// Funzione per il backoff esponenziale
const getDelay = (attempt) => {
return Math.min(initialDelay * Math.pow(2, attempt - 1), maxDelay);
};
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
// Verifica se il filepath esiste già
if (await this.isFileExistsAsync(filepath)) {
fs.unlinkSync(filepath);
}
if (attempt > 1) console.log(`📥 Tentativo ${attempt}/${maxRetries} - Scaricamento: ${url}`);
const response = await axios({
url,
method: 'GET',
responseType: 'stream',
timeout: timeout,
maxRedirects: 5,
headers: {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
Accept: 'image/jpeg,image/png,image/webp,image/gif,image/*', // Specifico per immagini
'Cache-Control': 'no-cache', // Evita problemi di caching
Connection: 'keep-alive',
},
validateStatus: (status) => status === 200, // Per immagini ci aspettiamo 200
maxContentLength: 10 * 1024 * 1024, // Limite di 10MB per immagine
});
// Verifica del content-type se richiesto
if (validateContentType) {
const contentType = response.headers['content-type'];
if (!contentType || !contentType.startsWith('image/')) {
throw new Error(`Content-Type non valido: ${contentType}`);
}
}
// Verifica della dimensione del file
const contentLength = parseInt(response.headers['content-length']);
if (contentLength && contentLength > 100 * 1024 * 1024) {
// 100MB limit
throw new Error('File troppo grande');
}
let downloadedBytes = 0;
response.data.on('data', (chunk) => {
downloadedBytes += chunk.length;
});
let writer = null;
if (nomefileoriginale) {
// Estrai il nome del file dall'URL o da Content-Disposition
//let fileName = this.extractFileNameFromUrl(url) || this.extractFileNameFromHeaders(response.headers);
let fileName = path.basename(response.data.responseUrl);
// Se il nome del file non è specificato, genera un nome predefinito
if (!fileName) {
fileName = `image_${Date.now()}.jpg`;
}
// Genera il percorso completo del file
const fullPath = path.join(path.dirname(filepath), fileName);
filepath = fullPath;
}
// Scrivi il file sul disco
writer = fs.createWriteStream(filepath);
response.data.pipe(writer);
await new Promise((resolve, reject) => {
writer.on('finish', resolve);
writer.on('error', (error) => {
fs.unlink(filepath, () => {}); // Pulizia in caso di errore
reject(error);
});
response.data.on('error', (error) => {
fs.unlink(filepath, () => {});
reject(error);
});
});
console.info(`✅ Immagine scaricata con successo in ${filepath}`);
return { ris: true, filepath };
} catch (error) {
console.error(`❌ Errore nel tentativo ${attempt}/${maxRetries}:`, error.message);
// Pulizia del file in caso di errore
if (await this.isFileExistsAsync(filepath)) {
fs.unlinkSync(filepath);
}
// se in error.message c'è '404' allora esci e ritorna code: 404
if (error.message.includes('404')) {
return { ris: false, code: 404 };
}
if (attempt === maxRetries) {
console.error(`Download fallito dopo ${maxRetries} tentativi: ${error.message}`);
return { ris: false };
}
const delay = getDelay(attempt);
console.info(`🔄 Attendo ${delay}ms prima del prossimo tentativo...`);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
}
// Funzione per estrarre il nome del file dall'URL
extractFileNameFromUrl(url) {
const match = url.match(/\/([^/?#]+)(?:[?#]|$)/);
return match ? decodeURIComponent(match[1]) : null;
}
// Funzione per estrarre il nome del file da Content-Disposition
extractFileNameFromHeaders(headers) {
const contentDisposition = headers['content-disposition'];
if (contentDisposition) {
const match = contentDisposition.match(/filename="([^"]+)"/);
if (match) {
return decodeURIComponent(match[1]);
}
}
return null;
}
async isFileExistsAsync (filename) {
try {
let fileExists = await fs.promises
.stat(filename)
.then(() => true)
.catch(() => false);
// console.log(filename, 'esiste', fileExists)
return fileExists;
} catch (e) {
// console.log(filename, 'esiste', 'FALSE')
return false;
}
}
}
module.exports = ImageDownloader;

View File

@@ -615,6 +615,7 @@ class Macro {
await this.gestisciCollana(productInfo, product);
const risrecInfo = await ProductInfo.findOneAndUpdate(
{ code: productInfo.code },
{ $set: productInfo },

View File

@@ -5,6 +5,9 @@ const Product = require('../models/product');
const ProductInfo = require('../models/productInfo');
const tools = require('../tools/general');
const shared_consts = require('../tools/shared_nodejs');
const fs = require('fs').promises; // 👈 Usa il modulo promises
class AmazonBookScraper {
constructor() {
@@ -23,7 +26,7 @@ class AmazonBookScraper {
// altri header se necessario
},
});
return data;
return { html: data, url };
} catch (err) {
console.error(`Errore fetching ISBN ${isbn10}:`, err.message);
return null;
@@ -57,7 +60,7 @@ class AmazonBookScraper {
}
}
async extractData(myproduct, html) {
async extractData(myproduct, html, url) {
const $ = cheerio.load(html);
const productInfo = await ProductInfo.findOne({ _id: myproduct.idProductInfo }).lean();
@@ -107,24 +110,41 @@ class AmazonBookScraper {
return {
titolo: title,
...(titoloOriginale ? { titoloOriginale } : {}),
...(sottotitolo ? { sottotitolo } : {sottotitolo: ''}),
numpagine,
misure,
edizione,
data_pubblicazione,
editore: publisher,
...(sottotitolo ? { sottotitolo } : { sottotitolo: '' }),
...(numpagine ? { numpagine } : {}),
...(misure ? { misure } : {}),
...(edizione ? { edizione } : {}),
...(data_pubblicazione ? { data_pubblicazione } : {}),
...(publisher ? { editore: publisher } : {}),
url: `<a href="${url}" target="_blank">URL</a>`,
};
}
async scrapeISBN(myproduct, isbn, options) {
try {
const isbn10 = this.isbn13to10(isbn);
const html = await this.fetchPageISBN10(isbn10);
const res = await this.fetchPageISBN10(isbn10);
if (!res) {
await Product.findOneAndUpdate(
{ _id: myproduct._id },
{ $set: { scraped: true, scraped_error: true } },
{ upsert: true, new: true, includeResultMetadata: true }
).lean();
return null;
}
const html = res.html;
if (!html) return null;
const data = await this.extractData(myproduct, html);
let updated = null;
let risupdate = null;
const data = await this.extractData(myproduct, html, res.url);
if (!options?.update) return data;
let recModificato = {};
const arrvariazioni = myproduct.arrvariazioni || [];
let index = -1;
@@ -142,64 +162,206 @@ class AmazonBookScraper {
let aggiornaMisure = false;
let aggiornaEdizione = false;
let aggiornaProductInfo = false;
let aggiornaSottotitolo = false;
if (index !== -1) {
const variante = arrvariazioni[index];
// Determina se aggiornare pagine
aggiornaPages = !options.aggiornasoloSeVuoti || !variante.pagine || variante.pagine === 0;
if (aggiornaPages) variante.pagine = Number(data.numpagine);
aggiornaPages = (!options.aggiornasoloSeVuoti || !variante.pagine || variante.pagine === 0) && data.numpagine;
if (aggiornaPages) {
variante.pagine = Number(data.numpagine);
recModificato['pagine'] = variante.pagine;
}
// Determina se aggiornare misure
aggiornaMisure = !options.aggiornasoloSeVuoti || !variante.misure;
if (aggiornaMisure) variante.misure = data.misure;
aggiornaMisure = (!options.aggiornasoloSeVuoti || !variante.misure) && data.misure;
if (aggiornaMisure) {
variante.misure = data.misure;
recModificato['misure'] = variante.misure;
}
// Determina se aggiornare edizione
aggiornaEdizione = !options.aggiornasoloSeVuoti || !variante.edizione;
if (aggiornaEdizione) variante.edizione = data.edizione;
aggiornaEdizione = (!options.aggiornasoloSeVuoti || !variante.edizione) && data.edizione;
if (aggiornaEdizione) {
variante.edizione = data.edizione;
recModificato['edizione'] = variante.edizione;
}
}
// Determina se aggiornare data pubblicazione
const currentDatePub = productInfo.date_pub;
aggiornaDataPubb = !options.aggiornasoloSeVuoti || !tools.isDateValid(currentDatePub);
const currentDatePub = myproduct.idProductInfo.date_pub;
aggiornaDataPubb =
(!options.aggiornasoloSeVuoti || !tools.isDateValid(currentDatePub)) &&
tools.isDateValid(data.data_pubblicazione);
if (aggiornaDataPubb && data.data_pubblicazione) {
productInfo.date_pub = new Date(data.data_pubblicazione);
}
aggiornaSottotitolo = (!options.aggiornasoloSeVuoti || !myproduct.idProductInfo.sottotitolo) && data.sottotitolo;
if (aggiornaSottotitolo && data.sottotitolo) {
productInfo.sottotitolo = data.sottotitolo;
}
aggiornaSottotitolo = false; // !! PER ORA LO DISATTIVO PERCHE' non esiste sempre il sottotitolo in un libro.
// Aggiorna arrvariazioni se pagine o misure sono cambiati
const aggiornadati = aggiornaPages || aggiornaMisure || aggiornaEdizione;
aggiornaProductInfo = aggiornaDataPubb || aggiornaSottotitolo;
if (aggiornadati) {
await Product.findOneAndUpdate(
updated = await Product.findOneAndUpdate(
{ _id: myproduct._id },
{ $set: { arrvariazioni, scraped: true, scraped_date: new Date() } },
{ $set: { arrvariazioni, scraped: true, scraped_updated: true, scraped_date: new Date() } },
{ upsert: true, new: true, includeResultMetadata: true }
);
} else if (aggiornaProductInfo) {
if (!tools.isObjectEmpty(productInfo)) {
// Aggiorna il flag che ho modificato i dati
updated = await Product.findOneAndUpdate(
{ _id: myproduct._id },
{ $set: { scraped: true, scraped_updated: true, scraped_date: new Date() } },
{ upsert: true, new: true, includeResultMetadata: true }
);
}
}
if (!updated) {
const upd = await Product.findOneAndUpdate(
{ _id: myproduct._id },
{ $set: { scraped: true, scraped_date: new Date() } },
{ upsert: true, new: true, returnDocument: 'after' }
);
console.log('upd', upd);
}
if (aggiornaProductInfo) {
// Aggiorna productInfo se contiene dati
if (!tools.isObjectEmpty(productInfo)) {
const risupdate = await ProductInfo.findOneAndUpdate(
risupdate = await ProductInfo.findOneAndUpdate(
{ _id: myproduct.idProductInfo },
{ $set: productInfo, scraped: true, scraped_date: new Date() },
{ new: true, upsert: true, returnOriginal: false }
{ $set: productInfo },
{ new: true, upsert: true, includeResultMetadata: true }
).lean();
console.log('risupdate', risupdate) ;
}
// console.log('risupdate', risupdate);
}
return data;
const concatenatedProduct = {
...recModificato,
...productInfo,
};
if (updated) {
console.log(' DATI AGGIORNATI:', JSON.stringify(concatenatedProduct));
}
async scrapeMultiple(isbnList, options) {
return { data, updated: this.isRecordAggiornato(updated) || this.isRecordAggiornato(risupdate) };
} catch (error) {
console.error('Errore in scrapeISBN:', error?.message);
return { data: null, updated: false, error: 'Errore in scrapeISBN:' + error?.message };
}
}
isRecordAggiornato(updatedDoc) {
try {
if (updatedDoc) {
const wasUpserted = updatedDoc.lastErrorObject.upserted !== undefined;
return updatedDoc.lastErrorObject.n === 1 && !wasUpserted;
} else {
return false;
}
} catch (e) {
console.log('error isRecordAggiornato', e);
return false;
}
}
numeroValido(num) {
return !isNaN(num) && num !== null && num !== '' && num > 0;
}
datiMancanti(product) {
let datimancanti = false;
if (product.arrvariazioni?.length > 0) {
const arrvar = product.arrvariazioni[0];
if (!this.numeroValido(arrvar.pagine)) {
datimancanti = true;
}
if (!arrvar.misure) {
datimancanti = true;
}
if (!arrvar.edizione) {
datimancanti = true;
}
}
if (product.idProductInfo) {
if (!tools.isDateValid(product.idProductInfo.date_pub)) datimancanti = true;
}
return datimancanti;
}
getRemainingTimeToTheEnd(dataorainizio, index, numrecord) {
// calcola il tempo stimato rimanente (ore e minuti), tenendo conto che sono arrivato a index, e devo raggiongere "numrecord", e sono partito alla data "dataorainizio"
const differenza = ((new Date().getTime() - dataorainizio.getTime()) / (index + 1)) * (numrecord - index);
// Se la differenza è negativa, restituisce 0
if (differenza <= 0) {
return 'COMPLETATO !';
}
// Calcola ore, minuti, secondi rimanenti
const ore = Math.floor(differenza / (1000 * 60 * 60));
const minuti = Math.floor((differenza % (1000 * 60 * 60)) / (1000 * 60));
// Restituisci il tempo rimanente in formato ore:minuti:secondi
return `Stimato: ${ore} ore e ${minuti} min`;
}
includiNelControlloIlRecProduct(product) {
return product.idProductInfo && [1, 4, 34, 45, 46].includes(product.idProductInfo.idStatoProdotto);
}
async scrapeMultiple(products, options) {
const results = [];
for (const isbn of isbnList) {
console.log(`Scraping ISBN: ${isbn}`);
const myproduct = null;
/// myproduct...
const data = await this.scrapeISBN(myproduct, isbn, options);
let quanti = 0;
let mylog = '';
console.log(`scrapeMultiple INIZIATO...`);
let dataorainizio = new Date();
for (let i = 0; i < 100 && i < products.length; i++) {
const product = products[i];
let isbn = product.isbn;
if (this.includiNelControlloIlRecProduct(product)) {
if (this.datiMancanti(product)) {
// console.log(`${quanti} / ${products.length} - Scraping: ${product.idProductInfo.name}`);
const data = await this.scrapeISBN(product, isbn, options);
if (data?.updated) {
results.push({ isbn, ...data });
// Per evitare blocchi, metti una pausa (es. 2 secondi)
await new Promise((r) => setTimeout(r, 2000));
quanti++;
}
if (i % 1 === 0) {
const percentuale = ((quanti / products.length) * 100).toFixed(2);
console.log(
`Scraping: ${product.isbn} - ${product.idProductInfo.name} - ${quanti} su ${i + 1} / ${
products.length
} - [${percentuale}%] - ${this.getRemainingTimeToTheEnd(dataorainizio, i, products.length)}`
);
}
// Per evitare blocchi, metti una pausa (es. 2 secondi)
await new Promise((r) => setTimeout(r, 3000));
}
}
}
mylog += `RECORD AGGIORNATI: ${results.length - 1} su ${quanti}`;
return results;
}
@@ -220,30 +382,238 @@ class AmazonBookScraper {
return html;
}
async ScraperDataAmazon(idapp, options) {
static async ScraperAzzeraFlagProducts(idapp, options) {
// aggiorna tutti i record di Product (con idapp) scraped: false
await Product.updateMany({ idapp, scraped: true }, { $set: { scraped: false } });
await Product.updateMany({ idapp, scraped_updated: true }, { $set: { scraped_updated: false } });
}
static async removeDuplicateVariations(idapp, options) {
let mylog = 'removeDuplicateVariations...\n';
// Fase 1: Troviamo i documenti che hanno almeno due elementi in arrvariazioni,
// uno con `versione` e uno senza.
const result = await Product.aggregate([
{ $match: { idapp } }, // Seleziona il prodotto in base a idapp
{
$unwind: '$arrvariazioni', // Esplodi l'array `arrvariazioni` in documenti separati
},
{
$group: {
_id: '$_id', // Gruppo per _id del prodotto
arrvariazioni: { $push: '$arrvariazioni' }, // Ricostruisci l'array arrvariazioni
// Trova se c'è almeno un elemento con `versione` e uno senza
hasVersione: {
$sum: { $cond: [{ $ifNull: ['$arrvariazioni.versione', false] }, 1, 0] },
},
},
},
{
$match: {
hasVersione: { $gt: 0 }, // Se c'è almeno un record con `versione`
},
},
]);
// Ora possiamo rimuovere i duplicati
for (let doc of result) {
// Filtra gli oggetti dentro `arrvariazioni` per mantenere quelli con versione
const arrvariazioniWithVersione = doc.arrvariazioni.filter((item) => item.versione);
// Rimuovi gli elementi che non hanno versione ma hanno gli stessi altri campi
const cleanedArr = arrvariazioniWithVersione.filter(
(item, index, self) =>
index ===
self.findIndex(
(t) =>
t.active === item.active &&
t.status === item.status &&
t.price === item.price &&
t.sale_price === item.sale_price &&
t.quantita === item.quantita &&
t.pagine === item.pagine &&
t.misure === item.misure &&
t.edizione === item.edizione &&
t.ristampa === item.ristampa &&
t.formato === item.formato &&
t.tipologia === item.tipologia &&
t.idTipologia === item.idTipologia &&
t.idTipoFormato === item.idTipoFormato &&
t.preOrderDate === item.preOrderDate &&
t.addtocart_link === item.addtocart_link &&
t.eta === item.eta
)
);
if (doc.arrvariazioni.length - cleanedArr.length > 0) {
const logtemp = `Elaborato ${doc._id} con ${arrvariazioniWithVersione.length} elementi\n`;
logtemp += `Rimossi ${doc.arrvariazioni.length - cleanedArr.length} duplicati\n`;
console.log(logtemp);
mylog += logtemp;
// Aggiorna il documento eliminando i duplicati
await Product.updateOne({ _id: doc._id }, { $set: { arrvariazioni: cleanedArr } });
}
}
return { mylog };
}
static async queryArrVariazioni(idapp) {
const result = await Product.aggregate([
{ $match: { idapp, 'arrvariazioni.0': { $exists: true }, 'arrvariazioni.1': { $exists: true } } },
{ $project: { arrvariazioni: 1, _id: 0 } },
]);
console.log('result', result);
}
static async ScraperGeneraCSV(idapp, options, res) {
// Dichiara le intestazioni del CSV
const headers = ['isbn', 'titolo', 'pagine', 'misure', 'edizione', 'date_pub' /*'sottotitolo'*/];
try {
// Trova i prodotti e popula 'idProductInfo'
const products = await Product.find({ idapp, scraped_updated: true })
.populate({ path: 'idProductInfo', select: 'date_pub name sottotitolo' })
.lean();
// Funzione per "appiattire" i dati
const flattenData = (data) => {
return data.map((item) => {
const flattened = { ...item };
// Se arrvariazioni esiste, prendi solo il primo elemento o elabora come richiesto
if (item.arrvariazioni && item.arrvariazioni.length > 0) {
const variation = item.arrvariazioni[0]; // Usa il primo elemento o modifica se necessario
flattened.pagine = variation.pagine || ''; // Usa '' se pagine non esiste
flattened.misure = variation.misure || '';
flattened.edizione = variation.edizione || '';
flattened.ristampa = variation.ristampa || '';
}
// Assicurati che 'idProductInfo' esista prima di usarlo
flattened.date_pub = item.idProductInfo ? item.idProductInfo.date_pub : '';
flattened.titolo = item.idProductInfo ? item.idProductInfo.name : '';
// flattened.sottotitolo = item.idProductInfo ? item.idProductInfo.sottotitolo : '';
return flattened;
});
};
// Appiattisci i dati
const records = flattenData(products);
// Prepara le righe del CSV
const rows = records.map((item) => {
// Se 'date_pub' è valido, convertilo in formato data, altrimenti metti una stringa vuota
const formattedDate = item.date_pub
? item.date_pub.toLocaleDateString('it-IT', { year: 'numeric', month: '2-digit', day: '2-digit' })
: '';
return [
item.isbn || '', // Assicurati che ISBN sia sempre una stringa, anche vuota
item.titolo || '',
//item.sottotitolo || '',
item.pagine || '', // Gestisci il caso in cui 'pagine' non esiste
item.misure || '',
item.edizione || '',
formattedDate, // La data formattata
];
});
// Aggiungi l'intestazione al CSV
rows.unshift(headers);
// Unisci tutte le righe con il delimitatore "|"
const csvData = rows.map((row) => row.join('|')).join('\n');
// Scrivi il file CSV in modo asincrono
// Ritorna la stringa CSV come oggetto per eventuali elaborazioni future
return { data: csvData };
} catch (e) {
console.error('Error in ScraperGeneraCSV:', e);
return { error: e.message };
}
}
static async ScraperDataAmazon(idapp, options) {
const scraper = new AmazonBookScraper();
const isbn = options.isbn;
try {
const myproduct = await Product.getProductByIsbn(isbn);
const data = await scraper.scrapeISBN(myproduct, isbn, options);
const ris = await scraper.scrapeISBN(myproduct, isbn, options);
// let html = this.generateHtmlTableFromObject(data);
console.log(data);
return res.status(200).send({ code: server_constants.RIS_CODE_OK, data, html });
console.log(ris?.data);
return res.status(200).send({ code: server_constants.RIS_CODE_OK, data: ris?.data, html });
} catch (e) {
console.error(e);
return res.status(400).send({ code: server_constants.RIS_CODE_ERR, msg: '' });
}
}
async ScraperMultipleDataAmazon(idapp, options) {
static async ScraperMultipleDataAmazon(idapp, options) {
const scraper = new AmazonBookScraper();
const isbnList = ['8850224248']; // metti i tuoi ISBN qui
try {
const books = await scraper.scrapeMultiple(isbnList);
// Prendi solo quelli che non sono ancora stati scraped !
const products = await Product.aggregate([
// Filtro di base sui campi idapp, isbn, scraped, e scraped_error
{
$match: {
idapp,
isbn: { $exists: true, $ne: '' },
scraped: { $ne: true }, // Escludi direttamente i record con scraped = true
$or: [{ deleted: { $exists: false } }, { deleted: { $exists: true, $eq: false } }],
$or: [{ scraped_error: { $exists: false } }, { scraped_error: { $exists: true, $eq: false } }],
},
},
// Popoliamo il campo idProductInfo
{
$lookup: {
from: 'productinfos', // Nome della collezione per 'idProductInfo'
localField: 'idProductInfo', // Campo del documento corrente (Product)
foreignField: '_id', // Campo di riferimento in ProductInfo
as: 'idProductInfo', // Campo in cui verranno inseriti i dati popolati
},
},
// De-strutturiamo l'array idProductInfo, se è un array
{
$unwind: {
path: '$idProductInfo',
preserveNullAndEmptyArrays: true, // Mantieni i documenti anche se idProductInfo è null o vuoto
},
},
{
$match: {
'idProductInfo.idStatoProdotto': { $in: [1, 4, 34, 45, 46] }, // Condizione su idStatoProdotto
},
},
// Proiettiamo solo i campi necessari
{
$project: {
scraped: 1,
scraped_updated: 1,
isbn: 1,
title: 1,
sottotitolo: 1,
arrvariazioni: 1,
'idProductInfo._id': 1,
'idProductInfo.date_pub': 1,
'idProductInfo.name': 1,
'idProductInfo.sottotitolo': 1,
'idProductInfo.idStatoProdotto': 1,
'idProductInfo.link_macro': 1,
'idProductInfo.imagefile': 1,
},
},
// A questo punto, puoi aggiungere altre operazioni di aggregazione se necessario (e.g., ordinamento)
]);
// console.log(products);
const books = await scraper.scrapeMultiple(products, options);
console.log(books);
} catch (e) {
console.error(e);

View File

@@ -0,0 +1,885 @@
"use strict";
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
function _defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } }
function _createClass(Constructor, protoProps, staticProps) { if (protoProps) _defineProperties(Constructor.prototype, protoProps); if (staticProps) _defineProperties(Constructor, staticProps); return Constructor; }
var axios = require('axios');
var apiUrl = 'https://api.cloudflare.com/client/v4'; // Endpoint
var MailinaboxClass = require('./Mailinabox.js');
var CloudFlare =
/*#__PURE__*/
function () {
function CloudFlare(config) {
_classCallCheck(this, CloudFlare);
this.config = config ? config : {};
if (!this.config.arrTokens) {
this.config.arrTokens = process.env.CLOUDFLARE_TOKENS;
}
}
_createClass(CloudFlare, [{
key: "init",
value: function init() {
if (this.config.arrTokens) {
// this.zones = this.fetchCloudflareZones(this.config.apiToken);
this.zones = [];
this.dnsRecords = null;
}
}
}, {
key: "fetchCloudflareZones",
value: function fetchCloudflareZones(apiToken) {
var response, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, zone, domain;
return regeneratorRuntime.async(function fetchCloudflareZones$(_context) {
while (1) {
switch (_context.prev = _context.next) {
case 0:
_context.prev = 0;
_context.next = 3;
return regeneratorRuntime.awrap(axios.get(apiUrl + '/zones', {
headers: {
'Authorization': "Bearer ".concat(apiToken),
// Autenticazione con token
'Content-Type': 'application/json' // Tipo di contenuto
}
}));
case 3:
response = _context.sent;
// Estrai i dati dalla risposta
this.zones = response.data.result;
if (!(this.zones && this.zones.length > 0)) {
_context.next = 33;
break;
}
_iteratorNormalCompletion = true;
_didIteratorError = false;
_iteratorError = undefined;
_context.prev = 9;
_iterator = this.zones[Symbol.iterator]();
case 11:
if (_iteratorNormalCompletion = (_step = _iterator.next()).done) {
_context.next = 19;
break;
}
zone = _step.value;
domain = zone.name;
_context.next = 16;
return regeneratorRuntime.awrap(this.findAndUpdateOnSite(domain, zone, apiToken));
case 16:
_iteratorNormalCompletion = true;
_context.next = 11;
break;
case 19:
_context.next = 25;
break;
case 21:
_context.prev = 21;
_context.t0 = _context["catch"](9);
_didIteratorError = true;
_iteratorError = _context.t0;
case 25:
_context.prev = 25;
_context.prev = 26;
if (!_iteratorNormalCompletion && _iterator["return"] != null) {
_iterator["return"]();
}
case 28:
_context.prev = 28;
if (!_didIteratorError) {
_context.next = 31;
break;
}
throw _iteratorError;
case 31:
return _context.finish(28);
case 32:
return _context.finish(25);
case 33:
return _context.abrupt("return", this.zones);
case 36:
_context.prev = 36;
_context.t1 = _context["catch"](0);
console.error('Errore durante il recupero delle zone di Cloudflare:', _context.t1.message);
case 39:
case "end":
return _context.stop();
}
}
}, null, this, [[0, 36], [9, 21, 25, 33], [26,, 28, 32]]);
}
}, {
key: "findAndUpdateOnSite",
value: function findAndUpdateOnSite(domain, zone, apiToken) {
var Site, normalizedDomain, recsite, modif, miab, dkim;
return regeneratorRuntime.async(function findAndUpdateOnSite$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
_context2.prev = 0;
Site = require('../models/site');
domain = domain.replace(/^https?:\/\//, '');
normalizedDomain = domain.startsWith("http://") || domain.startsWith("https://") ? domain : "https://" + domain; // Trova il sito usando il dominio normalizzato
_context2.next = 6;
return regeneratorRuntime.awrap(Site.findOne({
host: normalizedDomain
}));
case 6:
recsite = _context2.sent;
if (!recsite) {
_context2.next = 21;
break;
}
// Aggiorna i parametri token e zoneId
modif = recsite.cf_token !== apiToken || recsite.cf_zoneId !== zone.id;
if (!(modif && apiToken)) {
_context2.next = 12;
break;
}
_context2.next = 12;
return regeneratorRuntime.awrap(Site.findOneAndUpdate({
_id: recsite._id
}, {
$set: {
cf_token: apiToken,
cf_zoneId: zone.id
}
}, {
"new": true
}).then(function (updatedSite) {
console.log('Site aggiornato:', updatedSite);
})["catch"](function (error) {
console.error('Errore durante l\'aggiornamento del Site:', error);
}));
case 12:
if (!(recsite.enable_servermail && !recsite.dkim)) {
_context2.next = 21;
break;
}
miab = new MailinaboxClass(null);
miab.init();
_context2.next = 17;
return regeneratorRuntime.awrap(miab.getDKIMRecord(recsite.host.replace(/^https?:\/\//, '')));
case 17:
dkim = _context2.sent;
if (!dkim) {
_context2.next = 21;
break;
}
_context2.next = 21;
return regeneratorRuntime.awrap(Site.findOneAndUpdate({
_id: recsite._id
}, {
$set: {
dkim: dkim
}
}));
case 21:
_context2.next = 26;
break;
case 23:
_context2.prev = 23;
_context2.t0 = _context2["catch"](0);
console.error('Errore durante il recupero dei record DNS di Cloudflare:', _context2.t0.message);
case 26:
case "end":
return _context2.stop();
}
}
}, null, null, [[0, 23]]);
} // Funzione per estrarre i record DNS
}, {
key: "fetchDNSRecords",
value: function fetchDNSRecords(apiToken, zoneId) {
var apiUrlDNS, response;
return regeneratorRuntime.async(function fetchDNSRecords$(_context3) {
while (1) {
switch (_context3.prev = _context3.next) {
case 0:
apiUrlDNS = apiUrl + "/zones/".concat(zoneId, "/dns_records");
_context3.prev = 1;
_context3.next = 4;
return regeneratorRuntime.awrap(axios.get(apiUrlDNS, {
headers: {
'Authorization': "Bearer ".concat(apiToken),
// Autenticazione con token
'Content-Type': 'application/json' // Tipo di contenuto
}
}));
case 4:
response = _context3.sent;
this.dnsRecords = response.data.result;
return _context3.abrupt("return", this.dnsRecords);
case 9:
_context3.prev = 9;
_context3.t0 = _context3["catch"](1);
console.error('Errore durante il recupero dei record DNS di Cloudflare:', _context3.t0.message);
case 12:
case "end":
return _context3.stop();
}
}
}, null, this, [[1, 9]]);
} // Funzione per aggiornare un record DNS
}, {
key: "updateDNSRecord",
value: function updateDNSRecord(apiToken, zoneId, dnsRecordId, newDnsRecordData) {
var apiUrlDNS, response, updatedRecord;
return regeneratorRuntime.async(function updateDNSRecord$(_context4) {
while (1) {
switch (_context4.prev = _context4.next) {
case 0:
apiUrlDNS = apiUrl + "/zones/".concat(zoneId, "/dns_records/").concat(dnsRecordId);
_context4.prev = 1;
_context4.next = 4;
return regeneratorRuntime.awrap(axios.put(apiUrlDNS, newDnsRecordData, {
headers: {
'Authorization': "Bearer ".concat(apiToken),
// Autenticazione con token
'Content-Type': 'application/json' // Tipo di contenuto
}
}));
case 4:
response = _context4.sent;
updatedRecord = response.data.result; // Stampa il record DNS aggiornato
console.log('Record DNS aggiornato:', updatedRecord);
return _context4.abrupt("return", updatedRecord);
case 10:
_context4.prev = 10;
_context4.t0 = _context4["catch"](1);
console.error('Errore durante l\'aggiornamento del record DNS:', _context4.t0.message);
case 13:
case "end":
return _context4.stop();
}
}
}, null, null, [[1, 10]]);
} // Funzione per creare un record DNS di CloudFlare
}, {
key: "createDNSRecord",
value: function createDNSRecord(apiToken, zoneId, dnsRecordData) {
var apiUrlDNS, response, createdRecord;
return regeneratorRuntime.async(function createDNSRecord$(_context5) {
while (1) {
switch (_context5.prev = _context5.next) {
case 0:
apiUrlDNS = apiUrl + "/zones/".concat(zoneId, "/dns_records");
_context5.prev = 1;
_context5.next = 4;
return regeneratorRuntime.awrap(axios.post(apiUrlDNS, dnsRecordData, {
headers: {
'Authorization': "Bearer ".concat(apiToken),
// Autenticazione con token
'Content-Type': 'application/json' // Tipo di contenuto
}
}));
case 4:
response = _context5.sent;
createdRecord = response.data.result;
console.log('Record DNS creato:', createdRecord);
return _context5.abrupt("return", createdRecord);
case 10:
_context5.prev = 10;
_context5.t0 = _context5["catch"](1);
console.error('Errore durante la creazione del record DNS:', _context5.t0.message);
case 13:
case "end":
return _context5.stop();
}
}
}, null, null, [[1, 10]]);
} // Funzione per cancellare un record DNS di CloudFlare
}, {
key: "deleteDNSRecord",
value: function deleteDNSRecord(apiToken, zoneId, dnsRecordId) {
var apiUrlDNS, response, deletedRecord;
return regeneratorRuntime.async(function deleteDNSRecord$(_context6) {
while (1) {
switch (_context6.prev = _context6.next) {
case 0:
apiUrlDNS = apiUrl + "/zones/".concat(zoneId, "/dns_records/").concat(dnsRecordId);
_context6.prev = 1;
_context6.next = 4;
return regeneratorRuntime.awrap(axios["delete"](apiUrlDNS, {
headers: {
'Authorization': "Bearer ".concat(apiToken),
// Autenticazione con token
'Content-Type': 'application/json' // Tipo di contenuto
}
}));
case 4:
response = _context6.sent;
deletedRecord = response.data.result;
console.log('Record DNS cancellato:', deletedRecord);
return _context6.abrupt("return", deletedRecord);
case 10:
_context6.prev = 10;
_context6.t0 = _context6["catch"](1);
console.error('Errore durante la cancellazione del record DNS:', _context6.t0.message);
case 13:
case "end":
return _context6.stop();
}
}
}, null, null, [[1, 10]]);
}
}, {
key: "setCorrectIpsOnDNS",
value: function setCorrectIpsOnDNS(domainrec) {
var arrparams, i;
return regeneratorRuntime.async(function setCorrectIpsOnDNS$(_context7) {
while (1) {
switch (_context7.prev = _context7.next) {
case 0:
arrparams = [{
urladd: '',
paramsite: 'host_ip',
type: 'A'
}, {
urladd: 'test.',
paramsite: 'host_test_ip',
type: 'A'
}, {
urladd: 'api.',
paramsite: 'host_api_ip',
type: 'A'
}, {
urladd: 'testapi.',
paramsite: 'host_testapi_ip',
type: 'A'
}, {
urladd: 'www.',
paramsite: 'host',
type: 'CNAME'
}];
i = 0;
case 2:
if (!(i < arrparams.length)) {
_context7.next = 8;
break;
}
_context7.next = 5;
return regeneratorRuntime.awrap(this.setSingleIpsOnDNS(domainrec.name, arrparams[i]));
case 5:
i++;
_context7.next = 2;
break;
case 8:
_context7.next = 10;
return regeneratorRuntime.awrap(this.setServerMail(domainrec));
case 10:
_context7.next = 12;
return regeneratorRuntime.awrap(new Promise(function (resolve) {
return setTimeout(resolve, 3000);
}));
case 12:
case "end":
return _context7.stop();
}
}
}, null, this);
}
}, {
key: "setServerMail",
value: function setServerMail(domainrec) {
var _this = this;
var Site, domain, normalizedDomain;
return regeneratorRuntime.async(function setServerMail$(_context9) {
while (1) {
switch (_context9.prev = _context9.next) {
case 0:
_context9.prev = 0;
// get the parameters (Token and zoneId on Database)
Site = require('../models/site');
domain = domainrec.name.replace(/^https?:\/\//, '');
normalizedDomain = domain.startsWith("http://") || domain.startsWith("https://") ? domain : "https://" + domain; // Trova il sito usando il dominio normalizzato
_context9.next = 6;
return regeneratorRuntime.awrap(Site.findOne({
host: normalizedDomain
}).lean().then(function _callee(site) {
var myarrrecdns, recTypeMX, newRecord, recTypeTXTspf1, contentTXTspf1, _newRecord, nameDkimtoFind, recTypeTXTDKIM, contentTXTDKIM, _newRecord2, nameDmarctoFind, recTypeTXTdmarc, contentTXTdmarc, _newRecord3, myType, strfind, contentExpected, rectofind, _newRecord4, _newRecord5;
return regeneratorRuntime.async(function _callee$(_context8) {
while (1) {
switch (_context8.prev = _context8.next) {
case 0:
if (!site) {
_context8.next = 91;
break;
}
if (!(site.cf_token && site.cf_zoneId)) {
_context8.next = 91;
break;
}
_context8.next = 4;
return regeneratorRuntime.awrap(_this.fetchDNSRecords(site.cf_token, site.cf_zoneId));
case 4:
myarrrecdns = _context8.sent;
_context8.next = 7;
return regeneratorRuntime.awrap(myarrrecdns.find(function (rec) {
return rec.type === 'MX' && rec.name === domain;
}));
case 7:
recTypeMX = _context8.sent;
if (!recTypeMX) {
_context8.next = 14;
break;
}
if (!(recTypeMX.content !== site.servermail)) {
_context8.next = 12;
break;
}
recTypeMX.content = site.servermail;
return _context8.abrupt("return", _this.updateDNSRecord(site.cf_token, site.cf_zoneId, recTypeMX.id, recTypeMX));
case 12:
_context8.next = 17;
break;
case 14:
// create a new record
newRecord = {
type: 'MX',
name: domain,
content: site.servermail,
ttl: 1,
proxied: false,
priority: 10
};
_context8.next = 17;
return regeneratorRuntime.awrap(_this.createDNSRecord(site.cf_token, site.cf_zoneId, newRecord));
case 17:
_context8.next = 19;
return regeneratorRuntime.awrap(myarrrecdns.find(function (rec) {
return rec.type === 'TXT' && rec.name === domain && rec.content.indexOf('v=spf1') > -1;
}));
case 19:
recTypeTXTspf1 = _context8.sent;
if (!site.servermailip) {
_context8.next = 31;
break;
}
contentTXTspf1 = "v=spf1 a mx:".concat(site.servermail, " ip4:").concat(site.servermailip, " ~all");
if (!recTypeTXTspf1) {
_context8.next = 28;
break;
}
if (!(recTypeTXTspf1.content !== contentTXTspf1)) {
_context8.next = 26;
break;
}
recTypeTXTspf1.content = contentTXTspf1;
return _context8.abrupt("return", _this.updateDNSRecord(site.cf_token, site.cf_zoneId, recTypeTXTspf1.id, recTypeTXTspf1));
case 26:
_context8.next = 31;
break;
case 28:
// create a new record
_newRecord = {
type: 'TXT',
name: domain,
content: contentTXTspf1,
ttl: 3600
};
_context8.next = 31;
return regeneratorRuntime.awrap(_this.createDNSRecord(site.cf_token, site.cf_zoneId, _newRecord));
case 31:
// mail._domainkey.nomedominio v=DKIM1; h=sha256; k=rsa; s=email; p=<dkim>
nameDkimtoFind = "mail._domainkey.".concat(domain);
_context8.next = 34;
return regeneratorRuntime.awrap(myarrrecdns.find(function (rec) {
return rec.type === 'TXT' && rec.name === nameDkimtoFind;
}));
case 34:
recTypeTXTDKIM = _context8.sent;
if (!site.dkim) {
_context8.next = 46;
break;
}
contentTXTDKIM = "v=DKIM1; h=sha256; k=rsa; s=email; p=".concat(site.dkim);
if (!recTypeTXTDKIM) {
_context8.next = 43;
break;
}
if (!(recTypeTXTDKIM.content !== contentTXTDKIM)) {
_context8.next = 41;
break;
}
recTypeTXTDKIM.content = contentTXTDKIM;
return _context8.abrupt("return", _this.updateDNSRecord(site.cf_token, site.cf_zoneId, recTypeTXTDKIM.id, recTypeTXTDKIM));
case 41:
_context8.next = 46;
break;
case 43:
// create a new record
_newRecord2 = {
type: 'TXT',
name: nameDkimtoFind,
content: contentTXTDKIM,
ttl: 1
};
_context8.next = 46;
return regeneratorRuntime.awrap(_this.createDNSRecord(site.cf_token, site.cf_zoneId, _newRecord2));
case 46:
// DMARC:
nameDmarctoFind = "_dmarc.".concat(domain);
_context8.next = 49;
return regeneratorRuntime.awrap(myarrrecdns.find(function (rec) {
return rec.type === 'TXT' && rec.name === nameDmarctoFind && rec.content.indexOf('v=DMARC1') > -1;
}));
case 49:
recTypeTXTdmarc = _context8.sent;
if (!site.servermailip) {
_context8.next = 61;
break;
}
contentTXTdmarc = "v=DMARC1; p=quarantine; ruf=mailto:dmarc@".concat(domain, ";");
if (!recTypeTXTdmarc) {
_context8.next = 58;
break;
}
if (!(recTypeTXTdmarc.content !== contentTXTdmarc)) {
_context8.next = 56;
break;
}
recTypeTXTdmarc.content = contentTXTdmarc;
return _context8.abrupt("return", _this.updateDNSRecord(site.cf_token, site.cf_zoneId, recTypeTXTspf1.id, recTypeTXTspf1));
case 56:
_context8.next = 61;
break;
case 58:
// create a new record
_newRecord3 = {
type: 'TXT',
name: nameDmarctoFind,
content: contentTXTdmarc,
ttl: 1
};
_context8.next = 61;
return regeneratorRuntime.awrap(_this.createDNSRecord(site.cf_token, site.cf_zoneId, _newRecord3));
case 61:
// AutoConfig:
myType = 'CNAME';
strfind = "autoconfig.".concat(domain);
contentExpected = site.servermail;
_context8.next = 66;
return regeneratorRuntime.awrap(myarrrecdns.find(function (rec) {
return rec.type === 'CNAME' && rec.name === strfind;
}));
case 66:
rectofind = _context8.sent;
if (!rectofind) {
_context8.next = 73;
break;
}
if (!(rectofind.content !== contentExpected)) {
_context8.next = 71;
break;
}
rectofind.content = contentExpected;
return _context8.abrupt("return", _this.updateDNSRecord(site.cf_token, site.cf_zoneId, rectofind.id, rectofind));
case 71:
_context8.next = 76;
break;
case 73:
// create a new record
_newRecord4 = {
type: myType,
name: strfind,
content: contentExpected,
ttl: 1
};
_context8.next = 76;
return regeneratorRuntime.awrap(_this.createDNSRecord(site.cf_token, site.cf_zoneId, _newRecord4));
case 76:
// AutoDiscover:
myType = 'CNAME';
strfind = "autodiscover.".concat(domain);
contentExpected = site.servermail;
_context8.next = 81;
return regeneratorRuntime.awrap(myarrrecdns.find(function (rec) {
return rec.type === 'CNAME' && rec.name === strfind;
}));
case 81:
rectofind = _context8.sent;
if (!rectofind) {
_context8.next = 88;
break;
}
if (!(rectofind.content !== contentExpected)) {
_context8.next = 86;
break;
}
rectofind.content = contentExpected;
return _context8.abrupt("return", _this.updateDNSRecord(site.cf_token, site.cf_zoneId, rectofind.id, rectofind));
case 86:
_context8.next = 91;
break;
case 88:
// create a new record
_newRecord5 = {
type: myType,
name: strfind,
content: contentExpected,
ttl: 1
};
_context8.next = 91;
return regeneratorRuntime.awrap(_this.createDNSRecord(site.cf_token, site.cf_zoneId, _newRecord5));
case 91:
case "end":
return _context8.stop();
}
}
});
}));
case 6:
_context9.next = 11;
break;
case 8:
_context9.prev = 8;
_context9.t0 = _context9["catch"](0);
console.error('e', _context9.t0);
case 11:
case "end":
return _context9.stop();
}
}
}, null, null, [[0, 8]]);
}
}, {
key: "setSingleIpsOnDNS",
value: function setSingleIpsOnDNS(domain, paramobj) {
var _this2 = this;
var Site, normalizedDomain;
return regeneratorRuntime.async(function setSingleIpsOnDNS$(_context11) {
while (1) {
switch (_context11.prev = _context11.next) {
case 0:
_context11.prev = 0;
// get the parameters (Token and zoneId on Database)
Site = require('../models/site');
domain = domain.replace(/^https?:\/\//, '');
normalizedDomain = domain.startsWith("http://") || domain.startsWith("https://") ? domain : "https://" + domain; // Trova il sito usando il dominio normalizzato
_context11.next = 6;
return regeneratorRuntime.awrap(Site.findOne({
host: normalizedDomain
}).lean().then(function _callee2(site) {
var myarrrecdns, nametofind, recType, paramexpected, newRecord;
return regeneratorRuntime.async(function _callee2$(_context10) {
while (1) {
switch (_context10.prev = _context10.next) {
case 0:
if (!site) {
_context10.next = 20;
break;
}
if (!(site.cf_token && site.cf_zoneId)) {
_context10.next = 20;
break;
}
_context10.next = 4;
return regeneratorRuntime.awrap(_this2.fetchDNSRecords(site.cf_token, site.cf_zoneId));
case 4:
myarrrecdns = _context10.sent;
nametofind = paramobj.urladd + domain; // find
_context10.next = 8;
return regeneratorRuntime.awrap(myarrrecdns.find(function (rec) {
return rec.type === paramobj.type && rec.name === nametofind;
}));
case 8:
recType = _context10.sent;
paramexpected = '';
try {
paramexpected = site[paramobj.paramsite].replace(/^https?:\/\//, '');
;
} catch (e) {}
if (!(recType && paramexpected)) {
_context10.next = 17;
break;
}
if (!(recType.content !== paramexpected)) {
_context10.next = 15;
break;
}
recType.content = paramexpected;
return _context10.abrupt("return", _this2.updateDNSRecord(site.cf_token, site.cf_zoneId, recType.id, recType));
case 15:
_context10.next = 20;
break;
case 17:
// create a new record
newRecord = {
type: paramobj.type,
name: nametofind,
content: paramexpected,
ttl: 1,
proxied: true
};
_context10.next = 20;
return regeneratorRuntime.awrap(_this2.createDNSRecord(site.cf_token, site.cf_zoneId, newRecord));
case 20:
case "end":
return _context10.stop();
}
}
});
}));
case 6:
_context11.next = 11;
break;
case 8:
_context11.prev = 8;
_context11.t0 = _context11["catch"](0);
console.error('e', _context11.t0);
case 11:
case "end":
return _context11.stop();
}
}
}, null, null, [[0, 8]]);
}
}]);
return CloudFlare;
}();
module.exports = CloudFlare;

1745
src/server/modules/dist/CronMod.dev.js vendored Normal file

File diff suppressed because it is too large Load Diff

45
src/server/modules/dist/ErrorHandler.dev.js vendored Executable file
View File

@@ -0,0 +1,45 @@
"use strict";
function _typeof(obj) { if (typeof Symbol === "function" && typeof Symbol.iterator === "symbol") { _typeof = function _typeof(obj) { return typeof obj; }; } else { _typeof = function _typeof(obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; } return _typeof(obj); }
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
function _possibleConstructorReturn(self, call) { if (call && (_typeof(call) === "object" || typeof call === "function")) { return call; } return _assertThisInitialized(self); }
function _assertThisInitialized(self) { if (self === void 0) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return self; }
function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function"); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, writable: true, configurable: true } }); if (superClass) _setPrototypeOf(subClass, superClass); }
function _wrapNativeSuper(Class) { var _cache = typeof Map === "function" ? new Map() : undefined; _wrapNativeSuper = function _wrapNativeSuper(Class) { if (Class === null || !_isNativeFunction(Class)) return Class; if (typeof Class !== "function") { throw new TypeError("Super expression must either be null or a function"); } if (typeof _cache !== "undefined") { if (_cache.has(Class)) return _cache.get(Class); _cache.set(Class, Wrapper); } function Wrapper() { return _construct(Class, arguments, _getPrototypeOf(this).constructor); } Wrapper.prototype = Object.create(Class.prototype, { constructor: { value: Wrapper, enumerable: false, writable: true, configurable: true } }); return _setPrototypeOf(Wrapper, Class); }; return _wrapNativeSuper(Class); }
function isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Date.prototype.toString.call(Reflect.construct(Date, [], function () {})); return true; } catch (e) { return false; } }
function _construct(Parent, args, Class) { if (isNativeReflectConstruct()) { _construct = Reflect.construct; } else { _construct = function _construct(Parent, args, Class) { var a = [null]; a.push.apply(a, args); var Constructor = Function.bind.apply(Parent, a); var instance = new Constructor(); if (Class) _setPrototypeOf(instance, Class.prototype); return instance; }; } return _construct.apply(null, arguments); }
function _isNativeFunction(fn) { return Function.toString.call(fn).indexOf("[native code]") !== -1; }
function _setPrototypeOf(o, p) { _setPrototypeOf = Object.setPrototypeOf || function _setPrototypeOf(o, p) { o.__proto__ = p; return o; }; return _setPrototypeOf(o, p); }
function _getPrototypeOf(o) { _getPrototypeOf = Object.setPrototypeOf ? Object.getPrototypeOf : function _getPrototypeOf(o) { return o.__proto__ || Object.getPrototypeOf(o); }; return _getPrototypeOf(o); }
var TypedError =
/*#__PURE__*/
function (_Error) {
_inherits(TypedError, _Error);
function TypedError(args, status, type, error) {
var _this;
_classCallCheck(this, TypedError);
_this = _possibleConstructorReturn(this, _getPrototypeOf(TypedError).call(this, args));
_this.status = status;
_this.type = type;
_this.error = error;
return _this;
}
return TypedError;
}(_wrapNativeSuper(Error));
module.exports = TypedError;

View File

@@ -0,0 +1,348 @@
"use strict";
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
function _defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } }
function _createClass(Constructor, protoProps, staticProps) { if (protoProps) _defineProperties(Constructor.prototype, protoProps); if (staticProps) _defineProperties(Constructor, staticProps); return Constructor; }
var axios = require('axios');
var apiUrl = 'https://api.cloudflare.com/client/v4'; // Endpoint
var Mailinabox =
/*#__PURE__*/
function () {
function Mailinabox(config) {
_classCallCheck(this, Mailinabox);
this.config = config ? config : {};
if (!this.config.miabHost) {
this.config.miabHost = process.env.MIAB_HOST;
this.config.adminEmail = process.env.MIAB_ADMIN_EMAIL;
this.config.adminPassword = process.env.MIAB_ADMIN_PASSWORD;
}
}
_createClass(Mailinabox, [{
key: "init",
value: function init() {
if (this.config.arrTokens) {
this.zones = [];
}
}
}, {
key: "checkIfParamOk",
value: function checkIfParamOk() {
if (!this.config.miabHost || !this.config.adminEmail || !this.config.adminPassword) {
console.error('Configurazione mancante per il recupero del record DKIM.');
return false;
}
return true;
} // Funzione per ottenere il record DKIM
}, {
key: "getDKIMRecord",
value: function getDKIMRecord(domain) {
var url, auth, response, records, dkimRecord, pattern, match;
return regeneratorRuntime.async(function getDKIMRecord$(_context) {
while (1) {
switch (_context.prev = _context.next) {
case 0:
if (this.checkIfParamOk()) {
_context.next = 2;
break;
}
return _context.abrupt("return", '');
case 2:
url = "https://".concat(this.config.miabHost, "/admin/dns/zonefile/").concat(domain);
auth = Buffer.from("".concat(this.config.adminEmail, ":").concat(this.config.adminPassword)).toString('base64');
_context.prev = 4;
_context.next = 7;
return regeneratorRuntime.awrap(axios.get(url, {
headers: {
'Authorization': "Basic ".concat(auth)
}
}));
case 7:
response = _context.sent;
// console.log(`Record DNS esterni per ${config.domain}:`);
// Analizza la risposta per estrarre i record DNS
records = response.data.split('\n').filter(function (line) {
return line.trim() !== '' && !line.startsWith(';');
}).map(function (line) {
return line.trim();
}); // Trova e stampa il record DKIM
dkimRecord = records.find(function (record) {
return record.includes('mail._domainkey');
});
if (!dkimRecord) {
_context.next = 18;
break;
}
pattern = /p=([A-Za-z0-9+/=]+)(?:"\s*"([A-Za-z0-9+/=]+))?/; // Esegui la ricerca
match = dkimRecord.match(pattern);
if (!match) {
_context.next = 17;
break;
}
return _context.abrupt("return", match[1] + (match[2] || ''));
case 17:
console.log('Record DKIM non trovato.');
case 18:
return _context.abrupt("return", '');
case 21:
_context.prev = 21;
_context.t0 = _context["catch"](4);
console.error('Errore nel recupero del record DKIM:', _context.t0.message);
if (_context.t0.response) {
console.error('Dettagli errore:', _context.t0.response.data);
}
case 25:
return _context.abrupt("return", '');
case 26:
case "end":
return _context.stop();
}
}
}, null, this, [[4, 21]]);
}
}, {
key: "MIAB_getEmails",
value: function MIAB_getEmails(myrec) {
var url, auth, response, records;
return regeneratorRuntime.async(function MIAB_getEmails$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
if (!(!this.checkIfParamOk() || !myrec.domain)) {
_context2.next = 2;
break;
}
return _context2.abrupt("return");
case 2:
url = "https://".concat(this.config.miabHost, "/admin/mail/users?format=text");
auth = Buffer.from("".concat(this.config.adminEmail, ":").concat(this.config.adminPassword)).toString('base64');
_context2.prev = 4;
_context2.next = 7;
return regeneratorRuntime.awrap(axios.get(url, {
headers: {
'Authorization': "Basic ".concat(auth)
}
}));
case 7:
response = _context2.sent;
records = response.data.split('\n').filter(function (line) {
return line.trim() !== '' && line.indexOf(myrec.domain) > -1;
}).map(function (line) {
return line.trim();
});
return _context2.abrupt("return", records);
case 12:
_context2.prev = 12;
_context2.t0 = _context2["catch"](4);
console.error('Errore nel recupero delle Email ', _context2.t0.message);
if (_context2.t0.response) {
console.error('Dettagli errore:', _context2.t0.response.data);
}
case 16:
return _context2.abrupt("return", '');
case 17:
case "end":
return _context2.stop();
}
}
}, null, this, [[4, 12]]);
}
}, {
key: "removeEmail",
value: function removeEmail(myrec) {
var url, auth, myrecout, response, ris;
return regeneratorRuntime.async(function removeEmail$(_context3) {
while (1) {
switch (_context3.prev = _context3.next) {
case 0:
if (!(!this.checkIfParamOk() || !myrec.email)) {
_context3.next = 2;
break;
}
return _context3.abrupt("return");
case 2:
url = "https://".concat(this.config.miabHost, "/admin/mail/users/remove");
auth = Buffer.from("".concat(this.config.adminEmail, ":").concat(this.config.adminPassword)).toString('base64');
_context3.prev = 4;
myrecout = "email=".concat(myrec.email);
_context3.next = 8;
return regeneratorRuntime.awrap(axios.post(url, myrecout, {
headers: {
'Authorization': "Basic ".concat(auth)
}
}));
case 8:
response = _context3.sent;
ris = response.data;
return _context3.abrupt("return", ris);
case 13:
_context3.prev = 13;
_context3.t0 = _context3["catch"](4);
console.error('Errore nella cancellazione della Email ' + record.email, _context3.t0.message);
if (_context3.t0.response) {
console.error('Dettagli errore:', _context3.t0.response.data);
}
case 17:
return _context3.abrupt("return", '');
case 18:
case "end":
return _context3.stop();
}
}
}, null, this, [[4, 13]]);
}
}, {
key: "addEmail",
value: function addEmail(myrec) {
var url, auth, privileges, myrecout, response, ris;
return regeneratorRuntime.async(function addEmail$(_context4) {
while (1) {
switch (_context4.prev = _context4.next) {
case 0:
if (!(!this.checkIfParamOk() || !myrec.email)) {
_context4.next = 2;
break;
}
return _context4.abrupt("return");
case 2:
url = "https://".concat(this.config.miabHost, "/admin/mail/users/add");
auth = Buffer.from("".concat(this.config.adminEmail, ":").concat(this.config.adminPassword)).toString('base64');
_context4.prev = 4;
privileges = myrec.privileges ? 'admin' : '';
myrecout = "email=".concat(myrec.email, "&password=").concat(myrec.pwd, "&privileges=\"").concat(privileges, "\"");
_context4.next = 9;
return regeneratorRuntime.awrap(axios.post(url, myrecout, {
headers: {
'Authorization': "Basic ".concat(auth)
}
}));
case 9:
response = _context4.sent;
ris = response.data;
return _context4.abrupt("return", ris);
case 14:
_context4.prev = 14;
_context4.t0 = _context4["catch"](4);
console.error('Errore nella creazione della Email ' + record.email, _context4.t0.message);
if (_context4.t0.response) {
console.error('Dettagli errore:', _context4.t0.response.data);
}
case 18:
return _context4.abrupt("return", '');
case 19:
case "end":
return _context4.stop();
}
}
}, null, this, [[4, 14]]);
}
}, {
key: "setMailUserPassword",
value: function setMailUserPassword(myrec) {
var url, auth, data, response, ris;
return regeneratorRuntime.async(function setMailUserPassword$(_context5) {
while (1) {
switch (_context5.prev = _context5.next) {
case 0:
if (!(!this.checkIfParamOk() || !myrec.email)) {
_context5.next = 2;
break;
}
return _context5.abrupt("return");
case 2:
url = "https://".concat(this.config.miabHost, "/admin/mail/users/password");
auth = Buffer.from("".concat(this.config.adminEmail, ":").concat(this.config.adminPassword)).toString('base64');
_context5.prev = 4;
data = "email=".concat(myrec.email, "&password=").concat(myrec.pwd);
_context5.next = 8;
return regeneratorRuntime.awrap(axios.post(url, data, {
headers: {
'Authorization': "Basic ".concat(auth),
'Content-Type': 'application/x-www-form-urlencoded'
}
}));
case 8:
response = _context5.sent;
ris = '';
if (response.status === 200) {
ris = "Password cambiata con successo per ".concat(myrec.email);
} else {
ris = "Errore nel cambio password per ".concat(myrec.email);
}
return _context5.abrupt("return", ris);
case 14:
_context5.prev = 14;
_context5.t0 = _context5["catch"](4);
console.error('Errore nella creazione della Email ' + record.email, _context5.t0.message);
if (_context5.t0.response) {
console.error('Dettagli errore:', _context5.t0.response.data);
}
case 18:
return _context5.abrupt("return", '');
case 19:
case "end":
return _context5.stop();
}
}
}, null, this, [[4, 14]]);
}
}]);
return Mailinabox;
}();
module.exports = Mailinabox;

View File

@@ -13,7 +13,7 @@ module.exports = {
try {
const pathfile = Path.join(__dirname, tablename + '.js');
if (tools.isFileExists(pathfile)) {
if (await tools.isFileExistsAsync(pathfile)) {
const mydbfile = require(pathfile);
if (mydbfile && mydbfile.list) {
@@ -39,7 +39,7 @@ module.exports = {
let primavolta = false;
const pathfile = Path.join(__dirname, tablename + '.js');
if (tools.isFileExists(pathfile)) {
if (await tools.isFileExistsAsync(pathfile)) {
const mydbfile = require(pathfile);
if (mydbfile && mydbfile.list) {

View File

@@ -139,7 +139,7 @@ function updateProductInfoCatProds(productInfo, reccatprod) {
async function compressPdf(inputFile, outputFile, compressione) {
try {
const tempFolder = path.join(cwd, 'temp');
const hasTempFolder = tools.isFileExists(tempFolder);
const hasTempFolder = await tools.isFileExistsAsync(tempFolder);
if (!hasTempFolder) {
console.log('creo directory', tempFolder);
@@ -176,7 +176,7 @@ async function compressPdf(inputFile, outputFile, compressione) {
async function convertPDF_GS(inputFile, outputFile, width, height) {
// Verifica che il file di input esista
if (!tools.isFileExists(inputFile)) {
if (!await tools.isFileExistsAsync(inputFile)) {
throw new Error(`Il file di input non esiste: ${inputFile}`);
}
@@ -210,7 +210,7 @@ async function convertPDF_GS(inputFile, outputFile, width, height) {
console.log('Conversione completata con successo!');
// Verifica che il file di output sia stato generato
if (tools.isFileExists(outputFile)) {
if (await tools.isFileExistsAsync(outputFile)) {
console.log('File di output generato:', outputFile);
} else {
console.error('Il File di output NON è stato generato! :', outputFile);
@@ -256,7 +256,7 @@ async function extractPdfInfo(inputFile) {
}
async function convertPDF_PdfLib(idapp, inputFile, outputFile, options) {
if (!tools.isFileExists(inputFile)) {
if (!await tools.isFileExistsAsync(inputFile)) {
throw new Error(`Il file di input non esiste: ${inputFile}`);
}

View File

@@ -2425,11 +2425,11 @@ function uploadFile(req, res, version) {
//fit: sharp.fit.cover,
fit: sharp.fit.contain,
// position: sharp.strategy.entropy,
}).withMetadata().toFile(resized_img, function (err) {
}).withMetadata().toFile(resized_img, async function (err) {
// console.log('3) Ridimensionata Immagine ' + newname, 'in', resized_img);
if (tools.isFileExists(resized_img)) {
if (await tools.isFileExistsAsync(resized_img)) {
// console.log('4) Cancella l \'immagine grande originale:', newname);
// DELETE THE ORIGINAL BIG
tools.delete(newname, false, () => { });

View File

@@ -534,14 +534,14 @@ connectToDatabase(connectionUrl, options)
try {
// console.log('checkdir', folderprof);
if (!tools.isFileExists(folderprof)) {
if (!await tools.isFileExistsAsync(folderprof)) {
console.log('*** Creadir', folderprof);
await fs.mkdirSync(folderprof);
}
folderprof = dir + 'profile/' + myuser.username + '/' + table;
// console.log('checkdir', folderprof);
if (!tools.isFileExists(folderprof)) {
if (!await tools.isFileExistsAsync(folderprof)) {
console.log('creadir', folderprof);
await fs.mkdirSync(folderprof);
}
@@ -557,18 +557,18 @@ connectToDatabase(connectionUrl, options)
// console.log('file', file);
// console.log('filefrom', filefrom);
if (!tools.isFileExists(file)) {
if (!await tools.isFileExistsAsync(file)) {
// non esiste
console.log('non esiste', file);
console.log(' filefrom', filefrom);
console.log(' filefrom2', filefrom2);
}
if (!tools.isFileExists(file) && tools.isFileExists(filefrom)) {
if (!await tools.isFileExistsAsync(file) && await tools.isFileExistsAsync(filefrom)) {
console.log('@@@@@@ copia file:', filefrom, 'a', file);
tools.copy(filefrom, file);
}
if (!tools.isFileExists(file) && tools.isFileExists(filefrom2)) {
if (!await tools.isFileExistsAsync(file) && await tools.isFileExistsAsync(filefrom2)) {
console.log('@@@@@@ copia file 2:', filefrom2, 'a', file);
tools.copy(filefrom2, file);
}
@@ -675,7 +675,7 @@ connectToDatabase(connectionUrl, options)
}
// Funzione migliorata per ottenere chiave e certificato
function getCredentials(hostname) {
async function getCredentials(hostname) {
try {
let keyPath, certPath;
@@ -691,10 +691,10 @@ connectToDatabase(connectionUrl, options)
}
// Verifica esistenza file
if (!tools.isFileExists(keyPath)) {
if (!await tools.isFileExistsAsync(keyPath)) {
throw new Error(`Chiave privata non trovata: ${keyPath}`);
}
if (!tools.isFileExists(certPath)) {
if (!await tools.isFileExistsAsync(certPath)) {
throw new Error(`Certificato non trovato: ${certPath}`);
}
@@ -875,11 +875,11 @@ connectToDatabase(connectionUrl, options)
});
}
function createHttpOrHttpsServer(app, port, isProduction, domains) {
async function createHttpOrHttpsServer(app, port, isProduction, domains) {
if (isProduction) {
domains.forEach((domain) => {
const credentials = getCredentials(domain.hostname);
createHttpsServer({
const promises = domains.map(async (domain) => {
const credentials = await getCredentials(domain.hostname);
return createHttpsServer({
hostname: domain.hostname,
port: domain.port,
website: domain.website,
@@ -888,6 +888,7 @@ connectToDatabase(connectionUrl, options)
timeoutMinutes: 6,
});
});
await Promise.all(promises);
return null;
}
@@ -936,7 +937,7 @@ connectToDatabase(connectionUrl, options)
const pty = require('node-pty');
let scriptProcess = null;
ws.on('message', (message) => {
ws.on('message', async (message) => {
try {
const parsed = JSON.parse(message);
@@ -944,7 +945,7 @@ connectToDatabase(connectionUrl, options)
if (scriptProcess) scriptProcess.kill();
const scriptPath = path.join(__dirname, '..', '..', parsed.scriptName);
if (!tools.isFileExists(scriptPath)) {
if (!await tools.isFileExistsAsync(scriptPath)) {
return ws.send(JSON.stringify({ type: 'error', data: 'Script non trovato o non autorizzato' }));
}
@@ -999,7 +1000,7 @@ connectToDatabase(connectionUrl, options)
return wss;
}
function startServer(app, port) {
async function startServer(app, port) {
try {
const isProduction = ['production', 'test'].includes(process.env.NODE_ENV);
const ISDEBUG = false;
@@ -1014,7 +1015,7 @@ connectToDatabase(connectionUrl, options)
setupMiddleware(app, corsOptions, ISDEBUG);
const server = createHttpOrHttpsServer(app, port, isProduction, domains);
const server = await createHttpOrHttpsServer(app, port, isProduction, domains);
const wss = setupWebSocketServer(server);

File diff suppressed because it is too large Load Diff

View File

@@ -2552,6 +2552,11 @@ csurf@^1.11.0:
csrf "3.1.0"
http-errors "~1.7.3"
csv-writer@^1.6.0:
version "1.6.0"
resolved "https://registry.yarnpkg.com/csv-writer/-/csv-writer-1.6.0.tgz#d0cea44b6b4d7d3baa2ecc6f3f7209233514bcf9"
integrity sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==
dashdash@^1.12.0:
version "1.14.1"
resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"