- Estrazione dei dati da Amazon

- Ciclo di Estrapolazione di tutti i prodotti ed aggiornamento dei campi scraped e scraped_updated
- Creazione file CSV con i campi modificati.
This commit is contained in:
Surya Paolo
2025-05-19 17:33:58 +02:00
parent a374a7d7bc
commit a3c7b92c0c
16 changed files with 5131 additions and 1731 deletions

View File

@@ -534,14 +534,14 @@ connectToDatabase(connectionUrl, options)
try {
// console.log('checkdir', folderprof);
if (!tools.isFileExists(folderprof)) {
if (!await tools.isFileExistsAsync(folderprof)) {
console.log('*** Creadir', folderprof);
await fs.mkdirSync(folderprof);
}
folderprof = dir + 'profile/' + myuser.username + '/' + table;
// console.log('checkdir', folderprof);
if (!tools.isFileExists(folderprof)) {
if (!await tools.isFileExistsAsync(folderprof)) {
console.log('creadir', folderprof);
await fs.mkdirSync(folderprof);
}
@@ -557,18 +557,18 @@ connectToDatabase(connectionUrl, options)
// console.log('file', file);
// console.log('filefrom', filefrom);
if (!tools.isFileExists(file)) {
if (!await tools.isFileExistsAsync(file)) {
// non esiste
console.log('non esiste', file);
console.log(' filefrom', filefrom);
console.log(' filefrom2', filefrom2);
}
if (!tools.isFileExists(file) && tools.isFileExists(filefrom)) {
if (!await tools.isFileExistsAsync(file) && await tools.isFileExistsAsync(filefrom)) {
console.log('@@@@@@ copia file:', filefrom, 'a', file);
tools.copy(filefrom, file);
}
if (!tools.isFileExists(file) && tools.isFileExists(filefrom2)) {
if (!await tools.isFileExistsAsync(file) && await tools.isFileExistsAsync(filefrom2)) {
console.log('@@@@@@ copia file 2:', filefrom2, 'a', file);
tools.copy(filefrom2, file);
}
@@ -675,7 +675,7 @@ connectToDatabase(connectionUrl, options)
}
// Funzione migliorata per ottenere chiave e certificato
function getCredentials(hostname) {
async function getCredentials(hostname) {
try {
let keyPath, certPath;
@@ -691,10 +691,10 @@ connectToDatabase(connectionUrl, options)
}
// Verifica esistenza file
if (!tools.isFileExists(keyPath)) {
if (!await tools.isFileExistsAsync(keyPath)) {
throw new Error(`Chiave privata non trovata: ${keyPath}`);
}
if (!tools.isFileExists(certPath)) {
if (!await tools.isFileExistsAsync(certPath)) {
throw new Error(`Certificato non trovato: ${certPath}`);
}
@@ -875,11 +875,11 @@ connectToDatabase(connectionUrl, options)
});
}
function createHttpOrHttpsServer(app, port, isProduction, domains) {
async function createHttpOrHttpsServer(app, port, isProduction, domains) {
if (isProduction) {
domains.forEach((domain) => {
const credentials = getCredentials(domain.hostname);
createHttpsServer({
const promises = domains.map(async (domain) => {
const credentials = await getCredentials(domain.hostname);
return createHttpsServer({
hostname: domain.hostname,
port: domain.port,
website: domain.website,
@@ -888,6 +888,7 @@ connectToDatabase(connectionUrl, options)
timeoutMinutes: 6,
});
});
await Promise.all(promises);
return null;
}
@@ -936,7 +937,7 @@ connectToDatabase(connectionUrl, options)
const pty = require('node-pty');
let scriptProcess = null;
ws.on('message', (message) => {
ws.on('message', async (message) => {
try {
const parsed = JSON.parse(message);
@@ -944,7 +945,7 @@ connectToDatabase(connectionUrl, options)
if (scriptProcess) scriptProcess.kill();
const scriptPath = path.join(__dirname, '..', '..', parsed.scriptName);
if (!tools.isFileExists(scriptPath)) {
if (!await tools.isFileExistsAsync(scriptPath)) {
return ws.send(JSON.stringify({ type: 'error', data: 'Script non trovato o non autorizzato' }));
}
@@ -999,7 +1000,7 @@ connectToDatabase(connectionUrl, options)
return wss;
}
function startServer(app, port) {
async function startServer(app, port) {
try {
const isProduction = ['production', 'test'].includes(process.env.NODE_ENV);
const ISDEBUG = false;
@@ -1014,7 +1015,7 @@ connectToDatabase(connectionUrl, options)
setupMiddleware(app, corsOptions, ISDEBUG);
const server = createHttpOrHttpsServer(app, port, isProduction, domains);
const server = await createHttpOrHttpsServer(app, port, isProduction, domains);
const wss = setupWebSocketServer(server);