diff --git a/app/controllers/calculation.controller.js b/app/controllers/calculation.controller.js index 6ae96fe..466dd81 100644 --- a/app/controllers/calculation.controller.js +++ b/app/controllers/calculation.controller.js @@ -1,25 +1,23 @@ const db = require("../models"); -const Submission = db.Submission; -const SubmissionProduct = db.SubmissionProduct; -const SubmissionHistory = db.SubmissionHistory; -const Establishment = db.Establishment; -const EstablishmentUser = db.EstablishmentUser; -const UnitMaster = db.UnitMaster; -const VariationReasonMaster = db.VariationReasonMaster; -const ZeroTargetReasonMaster = db.ZeroTargetReasonMaster; -const Product = db.Product; -const CityTown = db.CityTown; -const Emirate = db.Emirate; -const QuarterlyWindowsConfiguration = db.QuarterlyWindowsConfiguration; -const User = db.user; -const ExcelJS = require("exceljs"); -const { Op } = require("sequelize"); -const { Sequelize } = require("sequelize"); -const { sendEmail } = require("../services/email.service"); -const { sendEmailService } = require("../services/email.service"); -const { getQuarterPeriods } = require("../services/quarterService"); -const { sanitizeForLog } = require("../utils/sanitize"); const logger = require("../services/logger"); +const IPICalculationService = require("../services/ipi_calculation_service"); +const SubmissionAutoFillService = require("../services/auto_fill_missing_quarterly_submissions_service"); + + +// Database configuration +const dbConfig = { + host: process.env.DB_HOST, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + database: process.env.DB_NAME, + dialect: process.env.DB_DIALECT || 'mysql', + pool: { + max: 5, + min: 0, + acquire: 30000, + idle: 10000 + } +}; // SELECT @@ -80,3 +78,217 @@ const logger = require("../services/logger"); // p.id; + + + + + +// 1. Calculate Base Year Production (Run once for base year 2022) +exports.calculate_base_year = async (req, res) => { + try { + const { baseYear = 2022, forceRecalculate = false } = req.body; + + const service = new IPICalculationService(dbConfig); + const result = await service.calculateBaseYearProduction(baseYear , forceRecalculate); + await service.close(); + + res.json({ + success: true, + message: `Base year ${baseYear} production calculated successfully`, + data: result + }); + + } catch (error) { + console.error('Error:', error); + res.status(500).json({ + success: false, + message: 'Error calculating base year production', + error: error.message + }); + } +}; + + +// 2. Calculate IPI for Specific Month +exports.calculate_month = async (req, res) => { + try { + // const { year, month } = req.body; + const year = 2026; + const month = 3; + + if (!year || !month) { + return res.status(400).json({ + success: false, + message: 'Year and month are required' + }); + } + + if (month < 1 || month > 12) { + return res.status(400).json({ + success: false, + message: 'Month must be between 1 and 12' + }); + } + + const service = new IPICalculationService(dbConfig); + const result = await service.runCompleteCalculation(year, month); + await service.close(); + + res.json({ + success: true, + message: `IPI calculated for ${year}-${String(month).padStart(2, '0')}`, + data: result + }); + + } catch (error) { + console.error('Error:', error); + res.status(500).json({ + success: false, + message: 'Error calculating IPI', + error: error.message + }); + } +}; + + +// 3. Calculate IPI for Multiple Months +exports.calculate_quarter = async (req, res) => { + try { + + const { year, quarter } = req.body; + + if (!year || !quarter) { + return res.status(400).json({ + success: false, + message: 'Year and quarter are required' + }); + } + + // Quarter to month mapping + const quarterMap = { + Q1: { startMonth: 1, endMonth: 3 }, + Q2: { startMonth: 4, endMonth: 6 }, + Q3: { startMonth: 7, endMonth: 9 }, + Q4: { startMonth: 10, endMonth: 12 }, + }; + + const selectedQuarter = quarterMap[quarter]; + + if (!selectedQuarter) { + return res.status(400).json({ + success: false, + message: 'Invalid quarter. Accepted values: Q1, Q2, Q3, Q4' + }); + } + + const { startMonth, endMonth } = selectedQuarter; + + const service = new IPICalculationService(dbConfig); + const results = []; + + for (let month = startMonth; month <= endMonth; month++) { + const result = await service.runCompleteCalculation(year, month); + results.push({ + month, + ...result + }); + } + + await service.close(); + + res.json({ success: true, message: `IPI calculated for ${year} months ${startMonth}-${endMonth}`, data: results }); + + } catch (error) { + console.error('Error:', error); + res.status(500).json({ + success: false, + message: 'Error calculating IPI range', + error: error.message + }); + } +}; + + +// 4. Get Calculation Status/Log +exports.calculation_log = async (req, res) => { + try { + const { year, month, type } = req.query; + + const service = new IPICalculationService(dbConfig); + const connection = await service.pool.getConnection(); + + let query = ` + SELECT * + FROM calculation_log + WHERE 1=1 + `; + + const params = []; + + if (year) { + query += ` AND reference_year = ?`; + params.push(year); + } + + if (month) { + query += ` AND reference_month = ?`; + params.push(month); + } + + if (type) { + query += ` AND calculation_type = ?`; + params.push(type); + } + + query += ` ORDER BY started_at DESC LIMIT 100`; + + const [results] = await connection.query(query, params); + connection.release(); + await service.close(); + + res.json({ success: true, data: results}); + + } catch (error) { + console.error('Error:', error); + res.status(500).json({ success: false, message: 'Error fetching calculation log', error: error.message }); + } +}; + + +// 5. Survey Auto Submit +exports.survey_auto_submit = async (req, res) => { + try { + const { year, quarter } = req.body; + + const autoSubmissionService = new SubmissionAutoFillService(dbConfig); + + // Auto-fill missing submissions for specific year and quarter + const result = await autoSubmissionService.autoFillMissingSubmissions(year, quarter); + + // Get report for specific quarter + // const result2 = await autoSubmissionService.getAutoFillReport(year, quarter); + + // Verify completeness for specific quarter + const result3 = await autoSubmissionService.verifyCompleteness(year, quarter); + + + // Or verify entire year + // await autoSubmissionService.verifyCompleteness(year); + + res.json({ success: true, autoFillMissingSubmissions: result , verifyCompleteness: result3 }); + + + + } catch (error) { + console.error('Error:', error); + res.status(500).json({ + success: false, + message: '', + error: error.message + }); + } +}; + + + + diff --git a/app/controllers/logController.js b/app/controllers/logController.js new file mode 100644 index 0000000..91c88b8 --- /dev/null +++ b/app/controllers/logController.js @@ -0,0 +1,67 @@ +// controllers/logController.js +const fs = require('fs').promises; +const path = require('path'); + +const LOG_DIR = path.join(__dirname, '../writable/logs'); + +exports.viewLogs = async (req, res) => { + try { + const files = await fs.readdir(LOG_DIR); + const logFiles = files.filter(file => file.endsWith('.log')).sort().reverse(); + + res.render('logs/viewer', { + title: 'Log Viewer', + logFiles + }); + } catch (error) { + console.error('Error reading log directory:', error); + res.status(500).render('error', { + message: 'Unable to load log files', + error + }); + } +}; + +exports.getLogContent = async (req, res) => { + try { + const { filename } = req.params; + const filePath = path.join(LOG_DIR, filename); + + // Security check: ensure the file is within LOG_DIR + const realPath = await fs.realpath(filePath); + if (!realPath.startsWith(await fs.realpath(LOG_DIR))) { + return res.status(403).json({ error: 'Access denied' }); + } + + const content = await fs.readFile(filePath, 'utf-8'); + const lines = content.split('\n').filter(line => line.trim()); + + res.json({ + filename, + content, + lines: lines.reverse(), // Most recent first + lineCount: lines.length + }); + } catch (error) { + console.error('Error reading log file:', error); + res.status(500).json({ error: 'Unable to read log file' }); + } +}; + +exports.downloadLog = async (req, res) => { + try { + const { filename } = req.params; + const filePath = path.join(LOG_DIR, filename); + + // Security check + const realPath = await fs.realpath(filePath); + if (!realPath.startsWith(await fs.realpath(LOG_DIR))) { + return res.status(403).send('Access denied'); + } + + res.download(filePath); + } catch (error) { + console.error('Error downloading log file:', error); + res.status(500).send('Unable to download log file'); + } +}; \ No newline at end of file diff --git a/app/routes/logRoutes.js b/app/routes/logRoutes.js new file mode 100644 index 0000000..4376a5f --- /dev/null +++ b/app/routes/logRoutes.js @@ -0,0 +1,19 @@ +// routes/logRoutes.js +const express = require('express'); +const router = express.Router(); +const logController = require('../controllers/logController'); + +// Main log viewer page +router.get('/', logController.viewLogs); + +// Get log file content (AJAX) +router.get('/content/:filename', logController.getLogContent); + +// Download log file +router.get('/download/:filename', logController.downloadLog); + +module.exports = router; + +// Add this route to your main app.js: +// const logRoutes = require('./routes/logRoutes'); +// app.use('/logs', logRoutes); \ No newline at end of file diff --git a/app/routes/routes.js b/app/routes/routes.js index 335299b..8487ed3 100644 --- a/app/routes/routes.js +++ b/app/routes/routes.js @@ -17,6 +17,7 @@ const ConfigController = require("../controllers/config.controller"); const dashboardController = require("../controllers/dashboard.controller"); const notificationTemplateController = require("../controllers/notificationTemplate.controller"); const quarterlyWindowsController = require("../controllers/quarterlyWindowsConfiguration.controller"); +const calculationController = require("../controllers/calculation.controller"); const fs = require("fs"); const path = require("path"); const multer = require("multer"); @@ -24,7 +25,115 @@ const { UPLOAD_DIR } = require("../config/upload.config"); const upload = multer({ dest: UPLOAD_DIR }); - + + +/** + * @swagger + * /api/calculate_base_year: + * post: + * summary: Base year calculation 2022 + * tags: [IIP Calculation] + * security: + * - appSignature: [] + * - CSRF: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: + * - baseYear + * - forceRecalculate + * properties: + * baseYear: + * type: integer + * example: 2022 + * forceRecalculate: + * type: string + * example: false + * responses: + * 200: + * description: Base year production calculated successfully + */ + +router.post("/calculate_base_year",[] , calculationController.calculate_base_year); + +/** + * @swagger + * /api/survey_auto_submit: + * post: + * summary: Auto survey submission for Quarter + * tags: [IIP Calculation] + * security: + * - appSignature: [] + * - CSRF: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: + * - year + * - quarter + * properties: + * year: + * type: integer + * example: 2025 + * quarter: + * type: string + * enum: [Q1, Q2, Q3, Q4] + * example: Q2 + * responses: + * 200: + * description: Auto survey completed successfully + */ +router.post("/survey_auto_submit",[] , calculationController.survey_auto_submit); + +/** + * @swagger + * /api/calculate_quarter: + * post: + * summary: Calculate IPI for a selected year and quarter (Quarter → Month mapping) + * tags: [IIP Calculation] + * security: + * - appSignature: [] + * - CSRF: [] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: + * - year + * - quarter + * properties: + * year: + * type: integer + * example: 2025 + * quarter: + * type: string + * enum: [Q1, Q2, Q3, Q4] + * example: Q2 + * description: | + * Quarter to month mapping: + * - Q1 → Jan to Mar (1 - 3) + * - Q2 → Apr to Jun (4 - 6) + * - Q3 → Jul to Sep (7 - 9) + * - Q4 → Oct to Dec (10 - 12) + * responses: + * 200: + * description: Quarterly IPI calculation completed successfully + */ + +router.post("/calculate_quarter",[] , calculationController.calculate_quarter); + + +router.get("/calculate_month",[] , calculationController.calculate_month); + + /** * @swagger diff --git a/app/services/auto_fill_missing_quarterly_submissions_service.js b/app/services/auto_fill_missing_quarterly_submissions_service.js new file mode 100644 index 0000000..260e895 --- /dev/null +++ b/app/services/auto_fill_missing_quarterly_submissions_service.js @@ -0,0 +1,521 @@ + +// Auto-Fill Missing Quarterly Submissions Service +const mysql = require('mysql2/promise'); + +class AutoFillSubmissionService { + + constructor(dbConfig) { + this.pool = mysql.createPool(dbConfig); + } + + + // MAIN FUNCTION: Auto-fill missing submissions for a specific year and quarter + async autoFillMissingSubmissions(year, quarter) { + const connection = await this.pool.getConnection(); + + try { + await connection.beginTransaction(); + + console.log(`\n========================================`); + console.log(`Auto-filling Missing Submissions for ${year} - ${quarter}`); + console.log(`========================================\n`); + + // Validate quarter + const validQuarters = ['Q1', 'Q2', 'Q3', 'Q4']; + if (!validQuarters.includes(quarter)) { + throw new Error(`Invalid quarter: ${quarter}. Must be Q1, Q2, Q3, or Q4`); + } + + // Step 1: Find establishments that don't have submission for this specific quarter + console.log(`Step 1: Finding establishments missing ${quarter} submissions...`); + const missingEstablishments = await this.findMissingEstablishments(connection, year, quarter); + + if (missingEstablishments.length === 0) { + console.log(`✓ All establishments have submitted ${quarter} for ${year}!`); + await connection.commit(); + return { success: true, establishmentsProcessed: 0, submissionsCreated: 0 }; + } + + console.log(`Found ${missingEstablishments.length} establishments missing ${quarter} submission\n`); + + let submissionsCreated = 0; + + // Step 2: Process each establishment + for (const establishment of missingEstablishments) { + console.log(`Processing Establishment ID: ${establishment.establishment_id} (${establishment.factory_name})`); + + const created = await this.createSubmissionForQuarter( + connection, + establishment.establishment_id, + year, + quarter + ); + + if (created) { + submissionsCreated++; + console.log(` ✓ Created submission for ${quarter}\n`); + } else { + console.log(` ⚠ Could not create submission (no source found)\n`); + } + } + + await connection.commit(); + + console.log(`========================================`); + console.log(`✓ Auto-fill Completed Successfully`); + console.log(` Establishments Processed: ${missingEstablishments.length}`); + console.log(` Submissions Created: ${submissionsCreated}`); + console.log(`========================================\n`); + + return { + success: true, + establishmentsProcessed: missingEstablishments.length, + submissionsCreated: submissionsCreated + }; + + } catch (error) { + await connection.rollback(); + console.error('Error in auto-fill process:', error); + throw error; + } finally { + connection.release(); + } + } + + + // Step 1: Find establishments missing submission for specific year/quarter + async findMissingEstablishments(connection, year, quarter) { + const query = ` + SELECT + e.id as establishment_id, + e.establishment_code, + e.factory_name + FROM establishments e + WHERE NOT EXISTS ( + SELECT 1 + FROM submission s + WHERE s.establishment_id = e.id + AND s.year = ? + AND s.quarter = ? + ) + ORDER BY e.id + `; + + const [results] = await connection.query(query, [year, quarter]); + return results; + } + + + // Step 2: Create submission for specific quarter using previous submission + async createSubmissionForQuarter(connection, establishmentId, year, quarter) { + // Find the source submission (previous quarter or previous year) + const sourceSubmissionId = await this.findPreviousSubmission( + connection, + establishmentId, + year, + quarter + ); + + if (!sourceSubmissionId) { + console.log(` ⚠ No previous submission found for establishment ${establishmentId}`); + return false; + } + + // Create new submission based on forecast values from source + await this.createSubmissionFromForecast( + connection, + establishmentId, + year, + quarter, + sourceSubmissionId + ); + + return true; + } + + + // Find the previous submission (previous quarter or previous year same quarter) + async findPreviousSubmission(connection, establishmentId, year, quarter) { + const quarterOrder = ['Q1', 'Q2', 'Q3', 'Q4']; + const currentQuarterIndex = quarterOrder.indexOf(quarter); + + // Strategy 1: Try to find previous quarter in the same year + if (currentQuarterIndex > 0) { + const previousQuarter = quarterOrder[currentQuarterIndex - 1]; + + const [result] = await connection.query( + `SELECT id FROM submission + WHERE establishment_id = ? AND year = ? AND quarter = ? + LIMIT 1`, + [establishmentId, year, previousQuarter] + ); + + if (result.length > 0) { + console.log(` Using source: ${year} ${previousQuarter} (submission ID: ${result[0].id})`); + return result[0].id; + } + } + + // Strategy 2: If Q1, try to find Q4 of previous year + if (quarter === 'Q1') { + const [result] = await connection.query( + `SELECT id FROM submission + WHERE establishment_id = ? AND year = ? AND quarter = 'Q4' + ORDER BY id DESC + LIMIT 1`, + [establishmentId, year - 1] + ); + + if (result.length > 0) { + console.log(` Using source: ${year - 1} Q4 (submission ID: ${result[0].id})`); + return result[0].id; + } + } + + // Strategy 3: Try to find same quarter from previous year + const [result] = await connection.query( + `SELECT id FROM submission + WHERE establishment_id = ? AND year = ? AND quarter = ? + ORDER BY id DESC + LIMIT 1`, + [establishmentId, year - 1, quarter] + ); + + if (result.length > 0) { + console.log(` Using source: ${year - 1} ${quarter} (submission ID: ${result[0].id})`); + return result[0].id; + } + + // Strategy 4: Find any last submission for this establishment + const [lastResult] = await connection.query( + `SELECT id, year, quarter FROM submission + WHERE establishment_id = ? + ORDER BY year DESC, FIELD(quarter, 'Q4', 'Q3', 'Q2', 'Q1') DESC + LIMIT 1`, + [establishmentId] + ); + + if (lastResult.length > 0) { + console.log(` Using source: ${lastResult[0].year} ${lastResult[0].quarter} (submission ID: ${lastResult[0].id})`); + return lastResult[0].id; + } + + return null; + } + + + // Create new submission using forecast values from source submission + async createSubmissionFromForecast(connection, establishmentId, year, quarter, sourceSubmissionId) { + // Get the source submission details + const [sourceSubmission] = await connection.query( + `SELECT * FROM submission WHERE id = ?`, + [sourceSubmissionId] + ); + + if (sourceSubmission.length === 0) { + throw new Error(`Source submission ${sourceSubmissionId} not found`); + } + + const source = sourceSubmission[0]; + + // Create new submission record + const [submissionResult] = await connection.query( + `INSERT INTO submission + (establishment_id, quarter, year, edit_request, edit_access, + status, created_by, created_at, approve_reject_status) + VALUES (?, ?, ?, 0, 0, 'Approved', ?, NOW(), 1)`, + [establishmentId, quarter, year, source.created_by || 1] + ); + + const newSubmissionId = submissionResult.insertId; + + // Get all products from source submission + const [sourceProducts] = await connection.query( + `SELECT * FROM submission_products WHERE submission_id = ? AND is_active = 1`, + [sourceSubmissionId] + ); + + // Copy products using forecast values as current values + for (const product of sourceProducts) { + await connection.query( + `INSERT INTO submission_products + (submission_id, product_id, unit_id, annual_installed_capacity, + previous_quantity_period_one, previous_quantity_period_two, previous_quantity_period_three, + previous_cost_period_one, previous_cost_period_two, previous_cost_period_three, + current_quantity_period_one, current_quantity_period_two, current_quantity_period_three, + current_cost_period_one, current_cost_period_two, current_cost_period_three, + forecast_quantity_period_one, forecast_quantity_period_two, forecast_quantity_period_three, + forecast_cost_period_one, forecast_cost_period_two, forecast_cost_period_three, + previous_quantity, previous_cost, current_quantity, current_cost, + forecast_quantity, forecast_cost, + variation_reason_master_id, other_variation_reason, + zero_target_reason_master_id, other_zero_target_reason, + remarks, created_by, created_at, is_active) + VALUES + (?, ?, ?, ?, + ?, ?, ?, + ?, ?, ?, + ?, ?, ?, + ?, ?, ?, + ?, ?, ?, + ?, ?, ?, + ?, ?, ?, ?, + ?, ?, + ?, ?, + ?, ?, + ?, ?, NOW(), 1)`, + [ + newSubmissionId, + product.product_id, + product.unit_id, + product.annual_installed_capacity, + // Previous = Current from source (shift forward) + product.current_quantity_period_one || 0, + product.current_quantity_period_two || 0, + product.current_quantity_period_three || 0, + product.current_cost_period_one || 0, + product.current_cost_period_two || 0, + product.current_cost_period_three || 0, + // Current = Forecast from source (use forecast values) + product.forecast_quantity_period_one || 0, + product.forecast_quantity_period_two || 0, + product.forecast_quantity_period_three || 0, + product.forecast_cost_period_one || 0, + product.forecast_cost_period_two || 0, + product.forecast_cost_period_three || 0, + // Forecast = Same as current (or could apply growth rate) + product.forecast_quantity_period_one || 0, + product.forecast_quantity_period_two || 0, + product.forecast_quantity_period_three || 0, + product.forecast_cost_period_one || 0, + product.forecast_cost_period_two || 0, + product.forecast_cost_period_three || 0, + // Totals + this.calculateTotal( + product.current_quantity_period_one, + product.current_quantity_period_two, + product.current_quantity_period_three + ), + this.calculateTotal( + product.current_cost_period_one, + product.current_cost_period_two, + product.current_cost_period_three + ), + this.calculateTotal( + product.forecast_quantity_period_one, + product.forecast_quantity_period_two, + product.forecast_quantity_period_three + ), + this.calculateTotal( + product.forecast_cost_period_one, + product.forecast_cost_period_two, + product.forecast_cost_period_three + ), + this.calculateTotal( + product.forecast_quantity_period_one, + product.forecast_quantity_period_two, + product.forecast_quantity_period_three + ), + this.calculateTotal( + product.forecast_cost_period_one, + product.forecast_cost_period_two, + product.forecast_cost_period_three + ), + // Other fields + product.variation_reason_master_id, + product.other_variation_reason, + product.zero_target_reason_master_id, + product.other_zero_target_reason, + 'Auto-generated from forecast values', + product.created_by || 1 + ] + ); + } + + console.log(` ✓ Created submission ID ${newSubmissionId} for ${quarter} using forecast from submission ${sourceSubmissionId}`); + + return newSubmissionId; + } + + + // Helper: Calculate total from three periods + calculateTotal(value1, value2, value3) { + const v1 = parseFloat(value1) || 0; + const v2 = parseFloat(value2) || 0; + const v3 = parseFloat(value3) || 0; + return (v1 + v2 + v3).toFixed(2); + } + + + // Get Report: Show which establishments were auto-filled + async getAutoFillReport(year, quarter = null) { + const connection = await this.pool.getConnection(); + + try { + let query = ` + SELECT + e.id as establishment_id, + e.establishment_code, + e.factory_name, + s.quarter, + s.year, + MAX(s.created_at) AS created_at, -- Fix + MIN(sp.remarks) AS remarks, -- or MAX(sp.remarks) + COUNT(sp.id) as products_count + FROM submission s + JOIN establishments e ON e.id = s.establishment_id + LEFT JOIN submission_products sp ON sp.submission_id = s.id + WHERE s.year = ? + AND sp.remarks LIKE '%Auto-generated%' + `; + + const params = [year]; + + if (quarter) { + query += ` AND s.quarter = ?`; + params.push(quarter); + } + + query += ` + GROUP BY e.id, s.quarter + ORDER BY e.id, FIELD(s.quarter, 'Q1', 'Q2', 'Q3', 'Q4') + `; + + const [results] = await connection.query(query, params); + + console.log('\n========================================'); + console.log(`Auto-Fill Report for Year ${year}${quarter ? ' - ' + quarter : ''}`); + console.log('========================================\n'); + + if (results.length === 0) { + console.log('No auto-generated submissions found.'); + } else { + console.table(results); + } + + return results; + + } finally { + connection.release(); + } + } + + + // Verify completeness after auto-fill + async verifyCompleteness(year, quarter = null) { + const connection = await this.pool.getConnection(); + + try { + if (quarter) { + // Check specific quarter + const [results] = await connection.query(` + SELECT + COUNT(DISTINCT e.id) as total_establishments, + COUNT(DISTINCT s.establishment_id) as submitted_establishments, + (COUNT(DISTINCT e.id) - COUNT(DISTINCT s.establishment_id)) as missing_establishments + FROM establishments e + LEFT JOIN submission s + ON s.establishment_id = e.id + AND s.year = ? + AND s.quarter = ? + `, [year, quarter]); + + const stats = results[0]; + + console.log('\n========================================'); + console.log(`Completeness Report for ${year} - ${quarter}`); + console.log('========================================'); + console.log(`Total Establishments: ${stats.total_establishments}`); + console.log(`Submitted ${quarter}: ${stats.submitted_establishments}`); + console.log(`Missing ${quarter}: ${stats.missing_establishments}`); + console.log('========================================\n'); + + return stats; + + } else { + // Check all quarters + const [results] = await connection.query(` + SELECT + COUNT(DISTINCT e.id) as total_establishments, + COUNT(DISTINCT CASE WHEN quarter_count = 4 THEN e.id END) as complete_establishments, + COUNT(DISTINCT CASE WHEN quarter_count < 4 THEN e.id END) as incomplete_establishments + FROM establishments e + LEFT JOIN ( + SELECT + establishment_id, + COUNT(DISTINCT quarter) as quarter_count + FROM submission + WHERE year = ? + GROUP BY establishment_id + ) as sub ON sub.establishment_id = e.id + `, [year]); + + const stats = results[0]; + + console.log('\n========================================'); + console.log(`Completeness Report for Year ${year}`); + console.log('========================================'); + console.log(`Total Establishments: ${stats.total_establishments}`); + console.log(`Complete (4 quarters): ${stats.complete_establishments}`); + console.log(`Incomplete (< 4 quarters): ${stats.incomplete_establishments}`); + console.log('========================================\n'); + + return stats; + } + + } finally { + connection.release(); + } + } + + // Close pool + async close() { + await this.pool.end(); + } +} + +// ========================================================================== +// USAGE EXAMPLE +// ========================================================================== + +const dbConfig = { + host: 'localhost', + user: 'your_user', + password: 'your_password', + database: 'your_database', + waitForConnections: true, + connectionLimit: 10, + queueLimit: 0 +}; + +// Example usage +async function main() { + const service = new AutoFillSubmissionService(dbConfig); + + try { + const year = 2022; + const quarter = 'Q1'; + + // Auto-fill missing submissions for specific year and quarter + const result = await service.autoFillMissingSubmissions(year, quarter); + + // Get report for specific quarter + await service.getAutoFillReport(year, quarter); + + // Verify completeness for specific quarter + await service.verifyCompleteness(year, quarter); + + // Or verify entire year + // await service.verifyCompleteness(year); + + } catch (error) { + console.error('Error:', error); + } finally { + await service.close(); + } +} + +// Uncomment to run +// main(); + +module.exports = AutoFillSubmissionService; \ No newline at end of file diff --git a/app/services/ipi_calculation_service.js b/app/services/ipi_calculation_service.js new file mode 100644 index 0000000..3fb3e84 --- /dev/null +++ b/app/services/ipi_calculation_service.js @@ -0,0 +1,822 @@ +// IPI Index Calculation Service +const mysql = require('mysql2/promise'); +const logger = require("./logger"); + +class IPICalculationService { + + + constructor(dbConfig) { + this.pool = mysql.createPool(dbConfig); + } + + + // STEP 1: Calculate and Store Base Year Average Production (2022) + async calculateBaseYearProduction(baseYear = 2022, forceRecalculate = false) { + + const connection = await this.pool.getConnection(); + let logId = null; + + try { + await connection.beginTransaction(); + + logger.info(`calculateBaseYearProduction - Calculating Base Year Production for ${baseYear}...`); + + // If force recalculate, delete existing data + if (forceRecalculate) { + logger.info('calculateBaseYearProduction - Force recalculate enabled , clearing existing data...'); + await connection.query( + `DELETE FROM base_year_production WHERE base_year = ?`, + [baseYear] + ); + } + + // Log calculation start + logId = await this.logRecord( 'base_year', baseYear, null, 'Started', null, 0, null, connection); + + // Your existing query to calculate base year production + const query = ` + SELECT + p.id AS product_id, + p.hs_code AS product_hs_code, + u.uom_short_name AS unit, + SUM(CASE WHEN s.quarter = 'Q1' THEN sp.current_quantity_period_one END) AS Jan, + SUM(CASE WHEN s.quarter = 'Q1' THEN sp.current_quantity_period_two END) AS Feb, + SUM(CASE WHEN s.quarter = 'Q1' THEN sp.current_quantity_period_three END) AS Mar, + SUM(CASE WHEN s.quarter = 'Q2' THEN sp.current_quantity_period_one END) AS Apr, + SUM(CASE WHEN s.quarter = 'Q2' THEN sp.current_quantity_period_two END) AS May, + SUM(CASE WHEN s.quarter = 'Q2' THEN sp.current_quantity_period_three END) AS Jun, + SUM(CASE WHEN s.quarter = 'Q3' THEN sp.current_quantity_period_one END) AS Jul, + SUM(CASE WHEN s.quarter = 'Q3' THEN sp.current_quantity_period_two END) AS Aug, + SUM(CASE WHEN s.quarter = 'Q3' THEN sp.current_quantity_period_three END) AS Sep, + SUM(CASE WHEN s.quarter = 'Q4' THEN sp.current_quantity_period_one END) AS Oct, + SUM(CASE WHEN s.quarter = 'Q4' THEN sp.current_quantity_period_two END) AS Nov, + SUM(CASE WHEN s.quarter = 'Q4' THEN sp.current_quantity_period_three END) AS \`Dec\`, + ROUND(( + COALESCE(SUM(CASE WHEN s.quarter='Q1' THEN sp.current_quantity_period_one END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q1' THEN sp.current_quantity_period_two END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q1' THEN sp.current_quantity_period_three END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q2' THEN sp.current_quantity_period_one END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q2' THEN sp.current_quantity_period_two END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q2' THEN sp.current_quantity_period_three END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q3' THEN sp.current_quantity_period_one END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q3' THEN sp.current_quantity_period_two END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q3' THEN sp.current_quantity_period_three END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q4' THEN sp.current_quantity_period_one END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q4' THEN sp.current_quantity_period_two END),0) + + COALESCE(SUM(CASE WHEN s.quarter='Q4' THEN sp.current_quantity_period_three END),0) + ) / 12, 2) AS avg_by_production + FROM submission_products sp + JOIN submission s ON s.id = sp.submission_id + JOIN products p ON p.id = sp.product_id + LEFT JOIN unit_master u ON u.id = p.unit_id + WHERE s.year = ? AND sp.is_active = 1 AND s.status = 'Approved' + GROUP BY p.id + `; + + const [products] = await connection.query(query, [baseYear]); + + + logger.info(`calculateBaseYearProduction - Found ${products.length} products to process...`); + + + // Insert into base_year_production table + let processedCount = 0; + for (const product of products) { + await connection.query( + `INSERT INTO base_year_production + (product_id, product_hs_code, base_year, unit, jan, feb, mar, apr, may, jun, + jul, aug, sep, oct, nov, \`dec\`, avg_by_production) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + product.product_id, product.product_hs_code, baseYear, product.unit, + product.Jan || 0, product.Feb || 0, product.Mar || 0, + product.Apr || 0, product.May || 0, product.Jun || 0, + product.Jul || 0, product.Aug || 0, product.Sep || 0, + product.Oct || 0, product.Nov || 0, product.Dec || 0, + product.avg_by_production + ] + ); + processedCount++; + + // Log progress every 50 products + if (processedCount % 50 === 0) { + logger.info(`calculateBaseYearProduction - Processed ${processedCount}/${products.length} products...`); + } + } + + // Update log + await this.logRecord('base_year', baseYear, null, 'Completed', logId, products.length, null,connection); + + + await connection.commit(); + logger.info(`calculateBaseYearProduction - Base Year Production calculated: ${products.length} products`); + return { success: true, productsProcessed: products.length }; + + } catch (error) { + await connection.rollback(); + console.error('Error calculating base year production:', error); + logger.error(`calculateBaseYearProduction - Error calculating base year production: ${error} `); + // Update log OUTSIDE the failed transaction + if (logId) { + await this.logRecord('base_year', baseYear, null, 'Failed', logId, 0, error.message); + } + + throw error; + } finally { + connection.release(); + } + } + + + // STEP 2: Aggregate Monthly Production + async aggregateMonthlyProduction(year, month) { + const connection = await this.pool.getConnection(); + let logId = null; + + try { + await connection.beginTransaction(); + + logger.info(`aggregateMonthlyProduction - Aggregating production for ${year}-${month}...`); + + // Log calculation start + logId = await this.logRecord( 'monthly_production', year, month, 'Started', null, 0, null, connection); + + + // Determine quarter and period based on month + const quarterMap = { + 1: { quarter: 'Q1', period: 'current_quantity_period_one' }, + 2: { quarter: 'Q1', period: 'current_quantity_period_two' }, + 3: { quarter: 'Q1', period: 'current_quantity_period_three' }, + 4: { quarter: 'Q2', period: 'current_quantity_period_one' }, + 5: { quarter: 'Q2', period: 'current_quantity_period_two' }, + 6: { quarter: 'Q2', period: 'current_quantity_period_three' }, + 7: { quarter: 'Q3', period: 'current_quantity_period_one' }, + 8: { quarter: 'Q3', period: 'current_quantity_period_two' }, + 9: { quarter: 'Q3', period: 'current_quantity_period_three' }, + 10: { quarter: 'Q4', period: 'current_quantity_period_one' }, + 11: { quarter: 'Q4', period: 'current_quantity_period_two' }, + 12: { quarter: 'Q4', period: 'current_quantity_period_three' } + }; + + const { quarter, period } = quarterMap[month]; + const monthNames = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + + // Aggregate production by product + const query = ` + SELECT + p.id AS product_id, + p.hs_code AS product_hs_code, + u.uom_short_name AS unit, + SUM(sp.${period}) AS production_quantity + FROM submission_products sp + JOIN submission s ON s.id = sp.submission_id + JOIN products p ON p.id = sp.product_id + LEFT JOIN unit_master u ON u.id = p.unit_id + WHERE s.year = ? AND s.quarter = ? + AND sp.is_active = 1 + AND s.status = 'Approved' + GROUP BY p.id + `; + + const [products] = await connection.query(query, [year, quarter]); + + // Insert monthly production + for (const product of products) { + await connection.query( + `INSERT INTO monthly_production + (product_id, product_hs_code, year, month, month_name, + production_quantity, unit) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + production_quantity=VALUES(production_quantity), + unit=VALUES(unit)`, + [ + product.product_id, product.product_hs_code, year, month, + monthNames[month], product.production_quantity || 0, product.unit + ] + ); + } + + + // Update log + await this.logRecord('monthly_production', year, month, 'Completed', logId, products.length, null,connection); + + await connection.commit(); + logger.info(`aggregateMonthlyProduction - Monthly production aggregated: ${products.length} products`); + return { success: true, productsProcessed: products.length }; + + } catch (error) { + await connection.rollback(); + console.error('Error aggregating monthly production:', error); + logger.error(`aggregateMonthlyProduction - Error aggregating monthly production: ${error}`); + // Update log OUTSIDE the failed transaction + if (logId) { + await this.logRecord('monthly_production', year, month, 'Failed', logId, 0, error.message); + } + throw error; + } finally { + connection.release(); + } + } + + + // STEP 3: Calculate Item Level Indices + async calculateItemLevelIndices(year, month) { + const connection = await this.pool.getConnection(); + let logId = null; + try { + await connection.beginTransaction(); + + logger.info(`calculateItemLevelIndices - Calculating item level indices for ${year}-${month}...`); + + // Log calculation start + logId = await this.logRecord( 'item_level_indices', year, month, 'Started', null, 0, null, connection); + + const monthNames = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + + // Calculate indices: Ri = Current Production / Base Year Avg Production + // Ii = Ri × 100 + const query = ` + SELECT + mp.product_id, + mp.product_hs_code, + mp.production_quantity AS current_production, + byp.avg_by_production AS base_year_avg_production, + (mp.production_quantity / byp.avg_by_production) AS production_relative, + ((mp.production_quantity / byp.avg_by_production) * 100) AS item_index + FROM monthly_production mp + JOIN base_year_production byp ON byp.product_id = mp.product_id + WHERE mp.year = ? AND mp.month = ? + AND byp.avg_by_production > 0 + `; + + const [items] = await connection.query(query, [year, month]); + + // Insert item level indices + for (const item of items) { + await connection.query( + `INSERT INTO item_level_indices + (product_id, product_hs_code, year, month, month_name, + current_production, base_year_avg_production, + production_relative, item_index) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + current_production=VALUES(current_production), + base_year_avg_production=VALUES(base_year_avg_production), + production_relative=VALUES(production_relative), + item_index=VALUES(item_index)`, + [ + item.product_id, item.product_hs_code, year, month, monthNames[month], + item.current_production, item.base_year_avg_production, + item.production_relative, item.item_index + ] + ); + } + + // Update log + await this.logRecord('item_level_indices', year, month, 'Completed', logId, items.length, null,connection); + + await connection.commit(); + logger.info(`calculateItemLevelIndices - Item level indices calculated: ${items.length} items`); + return { success: true, itemsProcessed: items.length }; + + } catch (error) { + await connection.rollback(); + console.error('Error calculating item indices:', error); + logger.error(`calculateItemLevelIndices - Error calculating item indices: ${error}`); + // Update log OUTSIDE the failed transaction + if (logId) { + await this.logRecord('item_level_indices', year, month, 'Failed', logId, 0, error.message); + } + throw error; + } finally { + connection.release(); + } + } + + + // STEP 4: Calculate ISIC 4-Digit Level Indices + async calculateISIC4DigitIndices(year, month) { + const connection = await this.pool.getConnection(); + let logId = null; + try { + await connection.beginTransaction(); + + logger.info(`calculateISIC4DigitIndices - Calculating ISIC 4-digit indices for ${year}-${month}...`); + + // Log calculation start + logId = await this.logRecord( 'isic_4digit_indices', year, month, 'Started', null, 0, null, connection); + + const monthNames = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + + // Get ISIC code from establishments table directly + // Using simple average of item indices for each ISIC 4-digit code + const query = ` + SELECT + LEFT(e.isic_code , 4) as isic_4digit_code, + COUNT(DISTINCT ili.product_id) AS total_weight, + SUM(ili.item_index) AS weighted_index_sum, + AVG(ili.item_index) AS isic_4digit_index + FROM item_level_indices ili + JOIN products p ON p.id = ili.product_id + JOIN submission_products sp ON sp.product_id = p.id + JOIN submission s ON s.id = sp.submission_id + AND s.year = ili.year + JOIN establishments e ON e.id = s.establishment_id + WHERE ili.year = ? AND ili.month = ? + AND e.isic_code IS NOT NULL + AND e.isic_code != '' + GROUP BY LEFT(e.isic_code , 4) + `; + + const [isic4Results] = await connection.query(query, [year, month]); + + for (const result of isic4Results) { + await connection.query( + `INSERT INTO isic_4digit_indices + (isic_4digit_code, year, month, month_name, total_weight, + weighted_index_sum, isic_4digit_index) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + total_weight=VALUES(total_weight), + weighted_index_sum=VALUES(weighted_index_sum), + isic_4digit_index=VALUES(isic_4digit_index)`, + [ + result.isic_4digit_code, year, month, monthNames[month], + result.total_weight, + result.weighted_index_sum, + result.isic_4digit_index + ] + ); + } + + // Update log + await this.logRecord('isic_4digit_indices', year, month, 'Completed', logId, isic4Results.length, null,connection); + + await connection.commit(); + logger.info(`calculateISIC4DigitIndices - ISIC 4-digit indices calculated: ${isic4Results.length} codes`); + return { success: true, codesProcessed: isic4Results.length }; + + } catch (error) { + await connection.rollback(); + console.error('Error calculating ISIC 4-digit indices:', error); + logger.error(`calculateISIC4DigitIndices - Error calculating ISIC 4-digit indices: ${error} `); + // Update log OUTSIDE the failed transaction + if (logId) { + await this.logRecord('isic_4digit_indices', year, month, 'Failed', logId, 0, error.message); + } + throw error; + } finally { + connection.release(); + } + } + + + // STEP 5: Calculate ISIC 3-Digit Level Indices + async calculateISIC3DigitIndices(year, month) { + const connection = await this.pool.getConnection(); + let logId = null; + try { + await connection.beginTransaction(); + + logger.info(`calculateISIC3DigitIndices - Calculating ISIC 3-digit indices for ${year}-${month}...`); + + // Log calculation start + logId = await this.logRecord( 'isic_3digit_indices', year, month, 'Started', null, 0, null, connection); + + const monthNames = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + + const query = ` + SELECT + LEFT(isic_4digit_code, 3) AS isic_3digit_code, + SUM(total_weight) AS total_weight, + SUM(total_weight * isic_4digit_index) AS weighted_index_sum, + (SUM(total_weight * isic_4digit_index) / SUM(total_weight)) AS isic_3digit_index + FROM isic_4digit_indices + WHERE year = ? AND month = ? + GROUP BY LEFT(isic_4digit_code, 3) + `; + + const [isic3Results] = await connection.query(query, [year, month]); + + for (const result of isic3Results) { + await connection.query( + `INSERT INTO isic_3digit_indices + (isic_3digit_code, year, month, month_name, total_weight, + weighted_index_sum, isic_3digit_index) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + total_weight=VALUES(total_weight), + weighted_index_sum=VALUES(weighted_index_sum), + isic_3digit_index=VALUES(isic_3digit_index)`, + [ + result.isic_3digit_code, year, month, monthNames[month], + result.total_weight, result.weighted_index_sum, + result.isic_3digit_index + ] + ); + } + + // Update log + await this.logRecord('isic_3digit_indices', year, month, 'Completed', logId, isic3Results.length, null,connection); + + await connection.commit(); + logger.info(`calculateISIC3DigitIndices - ISIC 3-digit indices calculated: ${isic3Results.length} codes`); + return { success: true, codesProcessed: isic3Results.length }; + + } catch (error) { + await connection.rollback(); + console.error('Error calculating ISIC 3-digit indices:', error); + logger.error(`calculateISIC3DigitIndices - Error calculating ISIC 3-digit indices: ${error} `); + // Update log OUTSIDE the failed transaction + if (logId) { + await this.logRecord('isic_3digit_indices', year, month, 'Failed', logId, 0, error.message); + } + throw error; + } finally { + connection.release(); + } + } + + + // STEP 6: Calculate ISIC 2-Digit Level Indices + async calculateISIC2DigitIndices(year, month) { + const connection = await this.pool.getConnection(); + let logId = null; + try { + await connection.beginTransaction(); + + logger.info(`calculateISIC2DigitIndices - Calculating ISIC 2-digit indices for ${year}-${month}...`); + + // Log calculation start + logId = await this.logRecord( 'isic_2digit_indices', year, month, 'Started', null, 0, null, connection); + + const monthNames = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + + const query = ` + SELECT + LEFT(isic_3digit_code, 2) AS isic_2digit_code, + SUM(total_weight) AS total_weight, + SUM(total_weight * isic_3digit_index) AS weighted_index_sum, + (SUM(total_weight * isic_3digit_index) / SUM(total_weight)) AS isic_2digit_index + FROM isic_3digit_indices + WHERE year = ? AND month = ? + GROUP BY LEFT(isic_3digit_code, 2) + `; + + const [isic2Results] = await connection.query(query, [year, month]); + + for (const result of isic2Results) { + await connection.query( + `INSERT INTO isic_2digit_indices + (isic_2digit_code, year, month, month_name, total_weight, + weighted_index_sum, isic_2digit_index) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + total_weight=VALUES(total_weight), + weighted_index_sum=VALUES(weighted_index_sum), + isic_2digit_index=VALUES(isic_2digit_index)`, + [ + result.isic_2digit_code, year, month, monthNames[month], + result.total_weight, result.weighted_index_sum, + result.isic_2digit_index + ] + ); + } + + // Update log + await this.logRecord('isic_2digit_indices', year, month, 'Completed', logId, isic2Results.length, null,connection); + + await connection.commit(); + logger.info(`calculateISIC2DigitIndices - ISIC 2-digit indices calculated: ${isic2Results.length} codes`); + return { success: true, codesProcessed: isic2Results.length }; + + } catch (error) { + await connection.rollback(); + console.error('Error calculating ISIC 2-digit indices:', error); + logger.error(`calculateISIC2DigitIndices - Error calculating ISIC 2-digit indices: ${error}`); + // Update log OUTSIDE the failed transaction + if (logId) { + await this.logRecord('isic_2digit_indices', year, month, 'Failed', logId, 0, error.message); + } + throw error; + } finally { + connection.release(); + } + } + + + // STEP 7: Calculate Manufacturing IPI (Headline Index) + async calculateManufacturingIPI(year, month) { + const connection = await this.pool.getConnection(); + let logId = null; + try { + await connection.beginTransaction(); + + logger.info(`calculateManufacturingIPI - Calculating Manufacturing indices for ${year}-${month}...`); + + // Log calculation start + logId = await this.logRecord( 'manufacturing_ipi', year, month, 'Started', null, 0, null, connection); + + const monthNames = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + + // Calculate Manufacturing IPI + const query = ` + SELECT + SUM(total_weight) AS total_weight, + SUM(total_weight * isic_2digit_index) AS weighted_index_sum, + (SUM(total_weight * isic_2digit_index) / SUM(total_weight)) AS manufacturing_index + FROM isic_2digit_indices + WHERE year = ? AND month = ? + `; + + const [result] = await connection.query(query, [year, month]); + + // If no data found, return empty response + if (!result || result.length === 0) { + return { + success: true, + index: null, + momChange: null, + yoyChange: null, + message: "No IPI data found for the given year and month" + }; + } + + const ipiData = result[0]; + + // Calculate MoM and YoY changes + let momChange = null; + let yoyChange = null; + + // Get previous month index for MoM + const prevMonth = month === 1 ? 12 : month - 1; + const prevYear = month === 1 ? year - 1 : year; + + const [prevMonthData] = await connection.query( + `SELECT manufacturing_index FROM manufacturing_ipi + WHERE year = ? AND month = ?`, + [prevYear, prevMonth] + ); + + if (prevMonthData.length > 0) { + const prevIndex = prevMonthData[0].manufacturing_index; + momChange = ((ipiData.manufacturing_index - prevIndex) / prevIndex) * 100; + } + + // Get same month last year for YoY + const [lastYearData] = await connection.query( + `SELECT manufacturing_index FROM manufacturing_ipi + WHERE year = ? AND month = ?`, + [year - 1, month] + ); + + if (lastYearData.length > 0) { + const lastYearIndex = lastYearData[0].manufacturing_index; + yoyChange = ((ipiData.manufacturing_index - lastYearIndex) / lastYearIndex) * 100; + } + + // Reference date is the first day of the month + const referenceDate = `${year}-${String(month).padStart(2, '0')}-01`; + + // Insert Manufacturing IPI + await connection.query( + `INSERT INTO manufacturing_ipi + (year, month, month_name, reference_date, total_weight, + weighted_index_sum, manufacturing_index, mom_change, yoy_change, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'Completed') + ON DUPLICATE KEY UPDATE + total_weight=VALUES(total_weight), + weighted_index_sum=VALUES(weighted_index_sum), + manufacturing_index=VALUES(manufacturing_index), + mom_change=VALUES(mom_change), + yoy_change=VALUES(yoy_change), + status=VALUES(status)`, + [ + year, month, monthNames[month], referenceDate, + ipiData.total_weight, ipiData.weighted_index_sum, + ipiData.manufacturing_index, momChange, yoyChange + ] + ); + + + // Update log + await this.logRecord('manufacturing_ipi', year, month, 'Completed', logId, result.length, null,connection); + + await connection.commit(); + + + logger.info(`calculateManufacturingIPI - Manufacturing IPI calculated: ${ipiData.manufacturing_index}`); + logger.info(`calculateManufacturingIPI - MoM: ${momChange ? momChange.toFixed(2) + '%' : 'N/A'}`); + logger.info(`calculateManufacturingIPI - YoY: ${yoyChange ? yoyChange.toFixed(2) + '%' : 'N/A'}`); + + return { + success: true, + index: ipiData.manufacturing_index, + momChange, + yoyChange + }; + + } catch (error) { + await connection.rollback(); + console.error('Error calculating Manufacturing IPI:', error); + logger.error(`calculateManufacturingIPI - Error calculating Manufacturing IPI: ${error}`); + // Update log OUTSIDE the failed transaction + if (logId) { + await this.logRecord('manufacturing_ipi', year, month, 'Failed', logId, 0, error.message); + } + throw error; + } finally { + connection.release(); + } + } + + + // MASTER FUNCTION: Run Complete Calculation Pipeline + async runCompleteCalculation(year, month) { + console.log(`\n========================================`); + console.log(`Starting IPI Calculation for ${year}-${month}`); + console.log(`========================================\n`); + + try { + // Step 1: Aggregate monthly production + await this.aggregateMonthlyProduction(year, month); + + // Step 2: Calculate item level indices + await this.calculateItemLevelIndices(year, month); + + // Step 3: Calculate ISIC 4-digit indices + await this.calculateISIC4DigitIndices(year, month); + + // Step 4: Calculate ISIC 3-digit indices + await this.calculateISIC3DigitIndices(year, month); + + // Step 5: Calculate ISIC 2-digit indices + await this.calculateISIC2DigitIndices(year, month); + + // Step 6: Calculate Manufacturing IPI + const result = await this.calculateManufacturingIPI(year, month); + + console.log(`\n========================================`); + console.log(`✓ IPI Calculation Completed Successfully`); + console.log(`========================================\n`); + + return result; + + } catch (error) { + console.error('Error in complete calculation:', error); + throw error; + } + } + + +// HELPER : calculation log +async logRecord(calculation_type, reference_year, reference_month = null, status = 'Started', logId = null, records_processed = 0, error_message = null, useConnection = null) { + + const connection = useConnection || await this.pool.getConnection(); + try { + if (!logId) { + const [result] = await connection.query( + `INSERT INTO calculation_log (calculation_type, reference_year, reference_month, status) + VALUES (?, ?, ?, ?)`, + [calculation_type, reference_year, reference_month, status] + ); + return result.insertId; + } + + if(records_processed == 0) + { + status = 'Failed'; + } + + await connection.query( + `UPDATE calculation_log + SET status=?, records_processed=?, error_message=?, completed_at=NOW() + WHERE id=?`, + [status, records_processed, error_message?.substring(0,65535) || null, logId] + ); + + } catch (err) { + console.error('[LOG ERROR]', err.message); + } finally { + if (!useConnection) connection.release(); + } +} + + + // HELPER: Verify Base Year Data + async verifyBaseYearData(baseYear = 2022) { + const connection = await this.pool.getConnection(); + + try { + console.log(`\nVerifying Base Year ${baseYear} Data...`); + + // Count products + const [countResult] = await connection.query( + `SELECT COUNT(*) as count FROM base_year_production WHERE base_year = ?`, + [baseYear] + ); + + // Get sample data + const [sampleData] = await connection.query( + `SELECT + product_id, + product_hs_code, + avg_by_production, + jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, \`dec\` + FROM base_year_production + WHERE base_year = ? + ORDER BY product_id + LIMIT 5`, + [baseYear] + ); + + // Check for products with zero average + const [zeroAvgResult] = await connection.query( + `SELECT COUNT(*) as count + FROM base_year_production + WHERE base_year = ? AND (avg_by_production = 0 OR avg_by_production IS NULL)`, + [baseYear] + ); + + console.log(`\n✓ Total products in base year: ${countResult[0].count}`); + console.log(`✓ Products with zero/null average: ${zeroAvgResult[0].count}`); + console.log(`\nSample data (first 5 products):`); + console.table(sampleData); + + return { + totalProducts: countResult[0].count, + zeroAverageProducts: zeroAvgResult[0].count, + sampleData: sampleData + }; + + } finally { + connection.release(); + } + } + + + // HELPER: Clear All Calculated Data for a Period + async clearCalculatedData(year, month) { + const connection = await this.pool.getConnection(); + + try { + await connection.beginTransaction(); + + console.log(`Clearing calculated data for ${year}-${month}...`); + + await connection.query( + `DELETE FROM monthly_production WHERE year = ? AND month = ?`, + [year, month] + ); + + await connection.query( + `DELETE FROM item_level_indices WHERE year = ? AND month = ?`, + [year, month] + ); + + await connection.query( + `DELETE FROM isic_4digit_indices WHERE year = ? AND month = ?`, + [year, month] + ); + + await connection.query( + `DELETE FROM isic_3digit_indices WHERE year = ? AND month = ?`, + [year, month] + ); + + await connection.query( + `DELETE FROM isic_2digit_indices WHERE year = ? AND month = ?`, + [year, month] + ); + + await connection.query( + `DELETE FROM manufacturing_ipi WHERE year = ? AND month = ?`, + [year, month] + ); + + await connection.commit(); + console.log(`✓ Data cleared for ${year}-${month}`); + + return { success: true }; + + } catch (error) { + await connection.rollback(); + console.error('Error clearing data:', error); + throw error; + } finally { + connection.release(); + } + } + + // Close pool + async close() { + await this.pool.end(); + } + +} + + + + +module.exports = IPICalculationService; \ No newline at end of file diff --git a/app/views/logs/viewer.ejs b/app/views/logs/viewer.ejs new file mode 100644 index 0000000..8ec4f96 --- /dev/null +++ b/app/views/logs/viewer.ejs @@ -0,0 +1,352 @@ + + + + + + + <%= title %> + + + +
+

📋 <%= title %>

+
+ FCSC_IPI_BACKEND +
+
+ +
+ + +
+
+ + + + + +
+ +
+
+ Select a log file from the sidebar to view its contents +
+
+ +
+ Ready +
+
+
+ + + + \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 4823430..23da90b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "csurf": "^1.11.0", "csv-parser": "^3.2.0", "dotenv": "^17.2.3", + "ejs": "^3.1.10", "exceljs": "^4.4.0", "express": "^5.1.0", "express-validator": "^7.2.1", @@ -1025,6 +1026,20 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/ejs": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/enabled": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", @@ -1213,6 +1228,33 @@ "moment": "^2.29.1" } }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -1619,6 +1661,22 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, + "node_modules/jake": { + "version": "10.9.4", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.4.tgz", + "integrity": "sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA==", + "dependencies": { + "async": "^3.2.6", + "filelist": "^1.0.4", + "picocolors": "^1.1.1" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", diff --git a/package.json b/package.json index c3325a8..43ebd03 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "csurf": "^1.11.0", "csv-parser": "^3.2.0", "dotenv": "^17.2.3", + "ejs": "^3.1.10", "exceljs": "^4.4.0", "express": "^5.1.0", "express-validator": "^7.2.1", diff --git a/server.js b/server.js index d8927ad..7e055b3 100644 --- a/server.js +++ b/server.js @@ -35,6 +35,14 @@ const csrfProtection = csrf({ const app = express(); +/** + * ========================= + * VIEW ENGINE SETUP (EJS) + * ========================= + */ +app.set('view engine', 'ejs'); +app.set('views', path.join(__dirname, 'app/views')); + /** * REQUIRED for HSTS when behind proxy */ @@ -184,6 +192,10 @@ app.use( * AUTH (NO CSRF) * ========================= */ + +const logRoutes = require('./app/routes/logRoutes'); +app.use('/logs', logRoutes); + app.post("/api/auth/login", [verifySignature], authController.login); app.post( "/api/forgot-password/request-otp",