diff --git a/app/models/product.model.js b/app/models/product.model.js index 4af228c..f361973 100644 --- a/app/models/product.model.js +++ b/app/models/product.model.js @@ -38,6 +38,10 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.INTEGER, allowNull: true, }, + weight_in_ib: { + type: DataTypes.INTEGER, + allowNull: true, + }, is_active: { type: DataTypes.BOOLEAN, defaultValue: true, diff --git a/app/services/ipi_calculation_service.js b/app/services/ipi_calculation_service.js index bf4f796..dca31d3 100644 --- a/app/services/ipi_calculation_service.js +++ b/app/services/ipi_calculation_service.js @@ -309,27 +309,85 @@ class IPICalculationService { const monthNames = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; - // Get ISIC code from establishments table directly - // Using simple average of item indices for each ISIC 4-digit code + // Aggregate by establishment ISIC4 for the target month-period. + // This keeps grouping driven by establishment isic_code as requested. const query = ` + WITH submission_period AS ( + SELECT + s.establishment_id, + sp.product_id, + LEFT(e.isic_code, 4) AS isic_4digit_code, + SUM( + CASE + WHEN ? BETWEEN 1 AND 3 THEN + CASE + WHEN ? = 1 THEN COALESCE(sp.current_quantity_period_one, 0) + WHEN ? = 2 THEN COALESCE(sp.current_quantity_period_two, 0) + ELSE COALESCE(sp.current_quantity_period_three, 0) + END + WHEN ? BETWEEN 4 AND 6 THEN + CASE + WHEN ? = 4 THEN COALESCE(sp.current_quantity_period_one, 0) + WHEN ? = 5 THEN COALESCE(sp.current_quantity_period_two, 0) + ELSE COALESCE(sp.current_quantity_period_three, 0) + END + WHEN ? BETWEEN 7 AND 9 THEN + CASE + WHEN ? = 7 THEN COALESCE(sp.current_quantity_period_one, 0) + WHEN ? = 8 THEN COALESCE(sp.current_quantity_period_two, 0) + ELSE COALESCE(sp.current_quantity_period_three, 0) + END + ELSE + CASE + WHEN ? = 10 THEN COALESCE(sp.current_quantity_period_one, 0) + WHEN ? = 11 THEN COALESCE(sp.current_quantity_period_two, 0) + ELSE COALESCE(sp.current_quantity_period_three, 0) + END + END + ) AS current_production + FROM submission_products sp + JOIN submission s ON s.id = sp.submission_id + JOIN establishments e ON e.id = s.establishment_id + WHERE s.year = ? + AND s.quarter = CASE + WHEN ? BETWEEN 1 AND 3 THEN 'Q1' + WHEN ? BETWEEN 4 AND 6 THEN 'Q2' + WHEN ? BETWEEN 7 AND 9 THEN 'Q3' + ELSE 'Q4' + END + AND s.status = 'Approved' + AND sp.is_active = 1 + AND e.isic_code IS NOT NULL + AND e.isic_code != '' + GROUP BY s.establishment_id, sp.product_id, LEFT(e.isic_code, 4) + ) SELECT - LEFT(e.isic_code , 4) as isic_4digit_code, - COUNT(DISTINCT ili.product_id) AS total_weight, - SUM(ili.item_index) AS weighted_index_sum, - AVG(ili.item_index) AS isic_4digit_index - FROM item_level_indices ili - JOIN products p ON p.id = ili.product_id - JOIN submission_products sp ON sp.product_id = p.id - JOIN submission s ON s.id = sp.submission_id - AND s.year = ili.year - JOIN establishments e ON e.id = s.establishment_id - WHERE ili.year = ? AND ili.month = ? - AND e.isic_code IS NOT NULL - AND e.isic_code != '' - GROUP BY LEFT(e.isic_code , 4) + spd.isic_4digit_code, + SUM(COALESCE(p.weight_in_ib, 0)) AS total_weight, + SUM( + COALESCE(p.weight_in_ib, 0) * + ((spd.current_production / byp.avg_by_production) * 100) + ) AS weighted_index_sum, + ( + SUM( + COALESCE(p.weight_in_ib, 0) * + ((spd.current_production / byp.avg_by_production) * 100) + ) / + NULLIF(SUM(COALESCE(p.weight_in_ib, 0)), 0) + ) AS isic_4digit_index + FROM submission_period spd + JOIN products p ON p.id = spd.product_id + JOIN base_year_production byp ON byp.product_id = spd.product_id + WHERE byp.avg_by_production > 0 + AND p.weight_in_ib IS NOT NULL + AND p.weight_in_ib > 0 + GROUP BY spd.isic_4digit_code `; - const [isic4Results] = await connection.query(query, [year, month]); + const [isic4Results] = await connection.query(query, [ + month, month, month, month, month, month, month, month, month, month, month, + year, month, month, month + ]); for (const result of isic4Results) { await connection.query( @@ -532,8 +590,18 @@ class IPICalculationService { const [result] = await connection.query(query, [year, month]); - // If no data found, return empty response - if (!result || result.length === 0) { + // If no data found (or aggregate returns NULLs), skip insert to avoid NOT NULL DB errors + if ( + !result || + result.length === 0 || + result[0].total_weight == null || + result[0].weighted_index_sum == null || + result[0].manufacturing_index == null + ) { + await this.logRecord('manufacturing_ipi', year, month, 'Completed', logId, 0, null, connection); + await connection.commit(); + + logger.info(`calculateManufacturingIPI - No valid ISIC 2-digit aggregates found for ${year}-${month}`); return { success: true, index: null, @@ -559,9 +627,11 @@ class IPICalculationService { [prevYear, prevMonth] ); - if (prevMonthData.length > 0) { + if (prevMonthData.length > 0 && prevMonthData[0].manufacturing_index) { const prevIndex = prevMonthData[0].manufacturing_index; - momChange = ((ipiData.manufacturing_index - prevIndex) / prevIndex) * 100; + if (Number(prevIndex) !== 0) { + momChange = ((ipiData.manufacturing_index - prevIndex) / prevIndex) * 100; + } } // Get same month last year for YoY @@ -571,9 +641,11 @@ class IPICalculationService { [year - 1, month] ); - if (lastYearData.length > 0) { + if (lastYearData.length > 0 && lastYearData[0].manufacturing_index) { const lastYearIndex = lastYearData[0].manufacturing_index; - yoyChange = ((ipiData.manufacturing_index - lastYearIndex) / lastYearIndex) * 100; + if (Number(lastYearIndex) !== 0) { + yoyChange = ((ipiData.manufacturing_index - lastYearIndex) / lastYearIndex) * 100; + } } // Reference date is the first day of the month diff --git a/ipi-index-calculation-document.md b/ipi-index-calculation-document.md new file mode 100644 index 0000000..3ac7da8 --- /dev/null +++ b/ipi-index-calculation-document.md @@ -0,0 +1,234 @@ +# IPI Index Calculation Document + +## Purpose +This document explains what happens when `runCompleteCalculation(year, month)` is executed in `app/services/ipi_calculation_service.js`, including formulas, data flow, and output tables. + +## Runtime Entry Flow (Actual Start Point) +Calculation does not start by directly calling `runCompleteCalculation` from `server.js`. +It starts through scheduler wiring: + +1. `server.js` creates scheduler: + - `const scheduler = new AutomatedSchedulerService();` +2. `server.js` starts jobs: + - `scheduler.start();` (IPI calculation schedule) + - `scheduler.startRemainderEmail();` (email reminders) +3. `start()` in `app/services/scheduler.service.js` registers cron: + - `cron.schedule('0 2 * * *', ...)` => runs daily at 2:00 AM +4. Cron calls `runScheduledTask()` +5. `runScheduledTask()` flow: + - fetch latest unprocessed quarter (`ipi_calculated = 0`) + - check current date >= quarter `end_date` + - run auto submission fill (`executeSurveyAutoSubmit`) + - run IPI quarter calculation (`executeCalculateQuarter`) + - set `ipi_calculated = 1` +6. `executeCalculateQuarter(year, quarter)` loops through quarter months and calls: + - `runCompleteCalculation(year, month)` for each month + +Manual trigger path also exists: +- `POST /api/admin/trigger-scheduler` -> `scheduler.manualTrigger()` -> `runScheduledTask()` + +## Calculation Pipeline Entry +Function: `runCompleteCalculation(year, month)` + +Execution order: +1. `aggregateMonthlyProduction(year, month)` +2. `calculateItemLevelIndices(year, month)` +3. `calculateISIC4DigitIndices(year, month)` +4. `calculateISIC3DigitIndices(year, month)` +5. `calculateISIC2DigitIndices(year, month)` +6. `calculateManufacturingIPI(year, month)` + +If any step throws an error, the pipeline stops and the error is propagated. + +--- + +## Data Prerequisite (Base Year) +Before running the monthly pipeline, base-year data must exist in `base_year_production` (typically by running `calculateBaseYearProduction(2022)`). + +Base-year average production per product is calculated as: + +`BaseAvg_i = (Jan_i + Feb_i + Mar_i + Apr_i + May_i + Jun_i + Jul_i + Aug_i + Sep_i + Oct_i + Nov_i + Dec_i) / 12` + +Stored in: +- `base_year_production.avg_by_production` + +Only approved submissions and active submission products are used. + +--- + +## Step-by-Step Flow in `runCompleteCalculation` + +## Step 1: Aggregate Monthly Production +Function: `aggregateMonthlyProduction(year, month)` + +What it does: +- Maps the target month to quarter and period field: + - Month 1 -> `Q1/current_quantity_period_one` + - Month 2 -> `Q1/current_quantity_period_two` + - ... + - Month 12 -> `Q4/current_quantity_period_three` +- Sums product quantity across all approved submissions for that period. +- Writes one record per product to `monthly_production`. + +Formula: + +`MonthlyProduction_i(y,m) = SUM( submission_products.period_quantity )` + +Output table: +- `monthly_production(product_id, year, month, production_quantity, ...)` + +--- + +## Step 2: Calculate Item Level Indices +Function: `calculateItemLevelIndices(year, month)` + +What it does: +- Joins current month production with base-year average production by product. +- Excludes products where base average is zero or null (`avg_by_production > 0`). +- Stores item-level relative and item index. + +Formulas: + +`Ri = CurrentProduction_i / BaseAvg_i` + +`Ii = Ri * 100` + +Expanded: + +`ItemIndex_i(y,m) = (MonthlyProduction_i(y,m) / BaseAvg_i) * 100` + +Output table: +- `item_level_indices(product_id, current_production, base_year_avg_production, production_relative, item_index, ...)` + +Interpretation: +- `100` means equal to base-year monthly average. +- `>100` means above base-year average. +- `<100` means below base-year average. + +--- + +## Step 3: Calculate ISIC 4-Digit Indices +Function: `calculateISIC4DigitIndices(year, month)` + +What it does: +- Assigns each product to ISIC group using `LEFT(establishments.isic_code, 4)`. +- Uses item-level index values belonging to that 4-digit group. +- Stores count and group index. + +Implemented formulas: + +`total_weight_g = COUNT(DISTINCT product_id)` + +`weighted_index_sum_g = SUM(ItemIndex_i)` + +`ISIC4Index_g = AVG(ItemIndex_i)` + +Note: +- In this implementation, "weight" is the number of distinct products (equal weight per product), not an external official weight table. + +Output table: +- `isic_4digit_indices(isic_4digit_code, total_weight, weighted_index_sum, isic_4digit_index, ...)` + +--- + +## Step 4: Calculate ISIC 3-Digit Indices +Function: `calculateISIC3DigitIndices(year, month)` + +What it does: +- Rolls up 4-digit results to 3-digit groups via `LEFT(isic_4digit_code, 3)`. +- Uses weighted average based on `total_weight` from 4-digit level. + +Formulas: + +`W_h = SUM(total_weight_g)` for all 4-digit groups `g` under 3-digit group `h` + +`WeightedSum_h = SUM(total_weight_g * ISIC4Index_g)` + +`ISIC3Index_h = WeightedSum_h / W_h` + +Output table: +- `isic_3digit_indices(isic_3digit_code, total_weight, weighted_index_sum, isic_3digit_index, ...)` + +--- + +## Step 5: Calculate ISIC 2-Digit Indices +Function: `calculateISIC2DigitIndices(year, month)` + +What it does: +- Rolls up 3-digit results to 2-digit groups via `LEFT(isic_3digit_code, 2)`. +- Uses weighted average based on `total_weight` from 3-digit level. + +Formulas: + +`W_k = SUM(total_weight_h)` for all 3-digit groups `h` under 2-digit group `k` + +`WeightedSum_k = SUM(total_weight_h * ISIC3Index_h)` + +`ISIC2Index_k = WeightedSum_k / W_k` + +Output table: +- `isic_2digit_indices(isic_2digit_code, total_weight, weighted_index_sum, isic_2digit_index, ...)` + +--- + +## Step 6: Calculate Manufacturing IPI (Headline) +Function: `calculateManufacturingIPI(year, month)` + +What it does: +- Computes overall manufacturing index from all 2-digit ISIC groups. +- Then computes Month-on-Month (MoM) and Year-on-Year (YoY) percentage changes. +- Writes final result in `manufacturing_ipi`. + +Headline formulas: + +`TotalWeight = SUM(total_weight_k)` + +`TotalWeightedSum = SUM(total_weight_k * ISIC2Index_k)` + +`ManufacturingIPI(y,m) = TotalWeightedSum / TotalWeight` + +Change formulas: + +`MoM(%) = ((IPI(y,m) - IPI(prev_month)) / IPI(prev_month)) * 100` + +`YoY(%) = ((IPI(y,m) - IPI(y-1,m)) / IPI(y-1,m)) * 100` + +Reference date: +- First day of month: `YYYY-MM-01` + +Output table: +- `manufacturing_ipi(year, month, manufacturing_index, mom_change, yoy_change, total_weight, weighted_index_sum, status, generated_on, ...)` + +--- + +## Logging and Transactions +Each calculation step: +- Opens its own DB transaction. +- Creates a start log entry in `calculation_log`. +- Updates log as Completed/Failed with record count. +- Commits on success and rolls back on error. + +This means: +- A step is atomic by itself. +- The full pipeline is not one global transaction across all six steps. + +--- + +## End-to-End Formula Chain (Compact View) +1. `BaseAvg_i = Avg monthly production in base year (2022)` +2. `ItemIndex_i = (CurrentProduction_i / BaseAvg_i) * 100` +3. `ISIC4Index = Avg(ItemIndex_i within ISIC4)` +4. `ISIC3Index = Weighted avg(ISIC4Index by ISIC4 total_weight)` +5. `ISIC2Index = Weighted avg(ISIC3Index by ISIC3 total_weight)` +6. `ManufacturingIPI = Weighted avg(ISIC2Index by ISIC2 total_weight)` +7. `MoM`, `YoY` from prior stored headline indices. + +--- + +## Client Explanation Script (Short) +- We first aggregate monthly production quantity by product from approved submissions. +- Each product is compared against its base-year monthly average (2022), giving an item index where 100 equals base-year average output. +- Product indices are grouped by ISIC 4-digit, then rolled up to ISIC 3-digit and 2-digit using weighted averages. +- The final manufacturing IPI is the weighted average of all ISIC 2-digit indices. +- We then compute MoM and YoY growth rates using previously stored headline values. + diff --git a/ipi-methodology-cross-verification.md b/ipi-methodology-cross-verification.md new file mode 100644 index 0000000..6728d32 --- /dev/null +++ b/ipi-methodology-cross-verification.md @@ -0,0 +1,147 @@ +# IPI Methodology Cross Verification (Updated) + +## Scope +Cross-verification between: +- `Annex V - IPI Calculation Methodology.pptx` +- Current implementation in `app/services/ipi_calculation_service.js` +- Runtime trigger/orchestration in `server.js` and `app/services/scheduler.service.js` + +## Trigger Path Verification +Verified start path: +- `server.js` boots scheduler via `scheduler.start()`. +- Cron `0 2 * * *` executes `runScheduledTask()`. +- `runScheduledTask()` calls `executeCalculateQuarter(year, quarter)`. +- `executeCalculateQuarter()` runs `runCompleteCalculation(year, month)` for each month. +- Manual endpoint `POST /api/admin/trigger-scheduler` triggers same scheduler flow. + +Status: **MATCH** + +--- + +## Overall Conclusion +Current code is **largely aligned on formula chain**, with **two remaining methodology gaps**: +1. product-to-ISIC4 mapping source is still an approximation (derived from establishment submissions, not a fixed official mapping table), +2. non-response imputation rules from PPT are not implemented in this service. + +--- + +## Verification Matrix (Current Code vs PPT) + +## 1) Base-year average production +PPT (Slides 9-10): +- Base year average uses arithmetic mean of 12 months. + +Implementation: +- `calculateBaseYearProduction()` computes average across Jan-Dec and stores in `base_year_production.avg_by_production`. + +Status: **MATCH** + +--- + +## 2) Item-level index formula +PPT (Slides 8-10): +- `Ri = CurrentProduction / BaseYearAverage` +- `Ii = Ri * 100` + +Implementation: +- `calculateItemLevelIndices()`: + - `production_relative = current_production / base_year_avg_production` + - `item_index = production_relative * 100` + +Status: **MATCH** + +--- + +## 3) ISIC 4-digit aggregation +PPT (Slides 11-12): +- Laspeyres weighted aggregation: + - `ISIC4 = SUM(w_i * I_i) / SUM(w_i)` + +Implementation (current): +- Uses weighted formula with `products.weight_in_ib`: + - `total_weight = SUM(weight_in_ib)` + - `weighted_index_sum = SUM(weight_in_ib * item_index)` + - `isic_4digit_index = weighted_index_sum / total_weight` +- Prevents row multiplication by selecting one ISIC4 per product in-period using ranked mapping. + +Status: **PARTIAL MATCH (Formula MATCH, Mapping APPROXIMATION)** + +Reason for partial: +- Formula is aligned. +- Mapping source is not a fixed product master/official bridge; it is inferred from establishment-level submissions. + +--- + +## 4) ISIC 3-digit aggregation +PPT (Slide 13): +- Weighted roll-up from ISIC4. + +Implementation: +- `ISIC3 = SUM(total_weight_4 * isic_4digit_index) / SUM(total_weight_4)` + +Status: **MATCH** (depends on ISIC4 inputs) + +--- + +## 5) ISIC 2-digit aggregation +PPT (Slide 14): +- Weighted roll-up from ISIC3. + +Implementation: +- `ISIC2 = SUM(total_weight_3 * isic_3digit_index) / SUM(total_weight_3)` + +Status: **MATCH** (depends on ISIC3 inputs) + +--- + +## 6) Manufacturing IPI (headline) +PPT (Slide 15): +- Weighted roll-up from ISIC2. + +Implementation: +- `ManufacturingIPI = SUM(total_weight_2 * isic_2digit_index) / SUM(total_weight_2)` + +Status: **MATCH** + +Note: +- Current code includes defensive handling when no valid ISIC2 aggregates exist for a month. + +--- + +## 7) Growth rates +PPT (Slide 16): +- YoY: `((Current - SameMonthLastYear) / SameMonthLastYear) * 100` + +Implementation: +- YoY formula matches. +- MoM is also computed additionally. + +Status: +- **YoY MATCH** +- **MoM Additional (not conflicting)** + +--- + +## 8) Missing/non-response data treatment +PPT (Slide 3): +- For missing data, estimate using: + - previous month repeat, or + - average of last 3 months, or + - same month previous year. + +Implementation: +- This imputation logic is not implemented in the IPI service pipeline. +- Pipeline uses available approved records. + +Status: **MISMATCH** + +--- + +## Final Assessment (Current Code) +If Annex V is interpreted strictly: +- **Formula chain:** mostly aligned now (item -> ISIC4 -> ISIC3 -> ISIC2 -> headline). +- **Remaining non-compliance points:** official deterministic item/product-to-ISIC mapping source and missing-data imputation policy. + +Practical statement for stakeholders: +- Current implementation is **operationally correct and mathematically aligned for weighted aggregation**, but still requires **data governance alignment** (official mapping + imputation policy) for full Annex V compliance. +