SS-FCSC/fcsc_ipi_backend
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IIP calculation corrected : GWM

Browse Source
This commit is contained in:
Gowtham M 2026-04-16 12:45:05 +05:30
parent b47acde85d
commit 028f728ebd
4 changed files with 480 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/models/product.model.js
View File

@ -38,6 +38,10 @@ module.exports = (sequelize, DataTypes) => {
type: DataTypes.INTEGER,
allowNull: true,
},
weight_in_ib: {
type: DataTypes.INTEGER,
allowNull: true,
},
is_active: {
type: DataTypes.BOOLEAN,
defaultValue: true,

118
app/services/ipi_calculation_service.js
View File

@ -309,27 +309,85 @@ class IPICalculationService {
const monthNames = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
// Get ISIC code from establishments table directly
// Using simple average of item indices for each ISIC 4-digit code
// Aggregate by establishment ISIC4 for the target month-period.
// This keeps grouping driven by establishment isic_code as requested.
const query = `
WITH submission_period AS (
SELECT
s.establishment_id,
sp.product_id,
LEFT(e.isic_code, 4) AS isic_4digit_code,
SUM(
CASE
WHEN ? BETWEEN 1 AND 3 THEN
CASE
WHEN ? = 1 THEN COALESCE(sp.current_quantity_period_one, 0)
WHEN ? = 2 THEN COALESCE(sp.current_quantity_period_two, 0)
ELSE COALESCE(sp.current_quantity_period_three, 0)
END
WHEN ? BETWEEN 4 AND 6 THEN
CASE
WHEN ? = 4 THEN COALESCE(sp.current_quantity_period_one, 0)
WHEN ? = 5 THEN COALESCE(sp.current_quantity_period_two, 0)
ELSE COALESCE(sp.current_quantity_period_three, 0)
END
WHEN ? BETWEEN 7 AND 9 THEN
CASE
WHEN ? = 7 THEN COALESCE(sp.current_quantity_period_one, 0)
WHEN ? = 8 THEN COALESCE(sp.current_quantity_period_two, 0)
ELSE COALESCE(sp.current_quantity_period_three, 0)
END
ELSE
CASE
WHEN ? = 10 THEN COALESCE(sp.current_quantity_period_one, 0)
WHEN ? = 11 THEN COALESCE(sp.current_quantity_period_two, 0)
ELSE COALESCE(sp.current_quantity_period_three, 0)
END
END
) AS current_production
FROM submission_products sp
JOIN submission s ON s.id = sp.submission_id
JOIN establishments e ON e.id = s.establishment_id
WHERE s.year = ?
AND s.quarter = CASE
WHEN ? BETWEEN 1 AND 3 THEN 'Q1'
WHEN ? BETWEEN 4 AND 6 THEN 'Q2'
WHEN ? BETWEEN 7 AND 9 THEN 'Q3'
ELSE 'Q4'
END
AND s.status = 'Approved'
AND sp.is_active = 1
AND e.isic_code IS NOT NULL
AND e.isic_code != ''
GROUP BY s.establishment_id, sp.product_id, LEFT(e.isic_code, 4)
)
SELECT
LEFT(e.isic_code , 4) as isic_4digit_code,
COUNT(DISTINCT ili.product_id) AS total_weight,
SUM(ili.item_index) AS weighted_index_sum,
AVG(ili.item_index) AS isic_4digit_index
FROM item_level_indices ili
JOIN products p ON p.id = ili.product_id
JOIN submission_products sp ON sp.product_id = p.id
JOIN submission s ON s.id = sp.submission_id
AND s.year = ili.year
JOIN establishments e ON e.id = s.establishment_id
WHERE ili.year = ? AND ili.month = ?
AND e.isic_code IS NOT NULL
AND e.isic_code != ''
GROUP BY LEFT(e.isic_code , 4)
spd.isic_4digit_code,
SUM(COALESCE(p.weight_in_ib, 0)) AS total_weight,
SUM(
COALESCE(p.weight_in_ib, 0) *
((spd.current_production / byp.avg_by_production) * 100)
) AS weighted_index_sum,
(
SUM(
COALESCE(p.weight_in_ib, 0) *
((spd.current_production / byp.avg_by_production) * 100)
) /
NULLIF(SUM(COALESCE(p.weight_in_ib, 0)), 0)
) AS isic_4digit_index
FROM submission_period spd
JOIN products p ON p.id = spd.product_id
JOIN base_year_production byp ON byp.product_id = spd.product_id
WHERE byp.avg_by_production > 0
AND p.weight_in_ib IS NOT NULL
AND p.weight_in_ib > 0
GROUP BY spd.isic_4digit_code
`;
const [isic4Results] = await connection.query(query, [year, month]);
const [isic4Results] = await connection.query(query, [
month, month, month, month, month, month, month, month, month, month, month,
year, month, month, month
]);
for (const result of isic4Results) {
await connection.query(
@ -532,8 +590,18 @@ class IPICalculationService {
const [result] = await connection.query(query, [year, month]);
// If no data found, return empty response
if (!result || result.length === 0) {
// If no data found (or aggregate returns NULLs), skip insert to avoid NOT NULL DB errors
if (
!result ||
result.length === 0 ||
result[0].total_weight == null ||
result[0].weighted_index_sum == null ||
result[0].manufacturing_index == null
) {
await this.logRecord('manufacturing_ipi', year, month, 'Completed', logId, 0, null, connection);
await connection.commit();
logger.info(`calculateManufacturingIPI - No valid ISIC 2-digit aggregates found for ${year}-${month}`);
return {
success: true,
index: null,
@ -559,9 +627,11 @@ class IPICalculationService {
[prevYear, prevMonth]
);
if (prevMonthData.length > 0) {
if (prevMonthData.length > 0 && prevMonthData[0].manufacturing_index) {
const prevIndex = prevMonthData[0].manufacturing_index;
momChange = ((ipiData.manufacturing_index - prevIndex) / prevIndex) * 100;
if (Number(prevIndex) !== 0) {
momChange = ((ipiData.manufacturing_index - prevIndex) / prevIndex) * 100;
}
}
// Get same month last year for YoY
@ -571,9 +641,11 @@ class IPICalculationService {
[year - 1, month]
);
if (lastYearData.length > 0) {
if (lastYearData.length > 0 && lastYearData[0].manufacturing_index) {
const lastYearIndex = lastYearData[0].manufacturing_index;
yoyChange = ((ipiData.manufacturing_index - lastYearIndex) / lastYearIndex) * 100;
if (Number(lastYearIndex) !== 0) {
yoyChange = ((ipiData.manufacturing_index - lastYearIndex) / lastYearIndex) * 100;
}
}
// Reference date is the first day of the month

234
ipi-index-calculation-document.md Normal file
View File

@ -0,0 +1,234 @@
# IPI Index Calculation Document
## Purpose
This document explains what happens when `runCompleteCalculation(year, month)` is executed in `app/services/ipi_calculation_service.js`, including formulas, data flow, and output tables.
## Runtime Entry Flow (Actual Start Point)
Calculation does not start by directly calling `runCompleteCalculation` from `server.js`.
It starts through scheduler wiring:
1. `server.js` creates scheduler:
- `const scheduler = new AutomatedSchedulerService();`
2. `server.js` starts jobs:
- `scheduler.start();` (IPI calculation schedule)
- `scheduler.startRemainderEmail();` (email reminders)
3. `start()` in `app/services/scheduler.service.js` registers cron:
- `cron.schedule('0 2 * * *', ...)` => runs daily at 2:00 AM
4. Cron calls `runScheduledTask()`
5. `runScheduledTask()` flow:
- fetch latest unprocessed quarter (`ipi_calculated = 0`)
- check current date >= quarter `end_date`
- run auto submission fill (`executeSurveyAutoSubmit`)
- run IPI quarter calculation (`executeCalculateQuarter`)
- set `ipi_calculated = 1`
6. `executeCalculateQuarter(year, quarter)` loops through quarter months and calls:
- `runCompleteCalculation(year, month)` for each month
Manual trigger path also exists:
- `POST /api/admin/trigger-scheduler` -> `scheduler.manualTrigger()` -> `runScheduledTask()`
## Calculation Pipeline Entry
Function: `runCompleteCalculation(year, month)`
Execution order:
1. `aggregateMonthlyProduction(year, month)`
2. `calculateItemLevelIndices(year, month)`
3. `calculateISIC4DigitIndices(year, month)`
4. `calculateISIC3DigitIndices(year, month)`
5. `calculateISIC2DigitIndices(year, month)`
6. `calculateManufacturingIPI(year, month)`
If any step throws an error, the pipeline stops and the error is propagated.
---
## Data Prerequisite (Base Year)
Before running the monthly pipeline, base-year data must exist in `base_year_production` (typically by running `calculateBaseYearProduction(2022)`).
Base-year average production per product is calculated as:
`BaseAvg_i = (Jan_i + Feb_i + Mar_i + Apr_i + May_i + Jun_i + Jul_i + Aug_i + Sep_i + Oct_i + Nov_i + Dec_i) / 12`
Stored in:
- `base_year_production.avg_by_production`
Only approved submissions and active submission products are used.
---
## Step-by-Step Flow in `runCompleteCalculation`
## Step 1: Aggregate Monthly Production
Function: `aggregateMonthlyProduction(year, month)`
What it does:
- Maps the target month to quarter and period field:
- Month 1 -> `Q1/current_quantity_period_one`
- Month 2 -> `Q1/current_quantity_period_two`
- ...
- Month 12 -> `Q4/current_quantity_period_three`
- Sums product quantity across all approved submissions for that period.
- Writes one record per product to `monthly_production`.
Formula:
`MonthlyProduction_i(y,m) = SUM( submission_products.period_quantity )`
Output table:
- `monthly_production(product_id, year, month, production_quantity, ...)`
---
## Step 2: Calculate Item Level Indices
Function: `calculateItemLevelIndices(year, month)`
What it does:
- Joins current month production with base-year average production by product.
- Excludes products where base average is zero or null (`avg_by_production > 0`).
- Stores item-level relative and item index.
Formulas:
`Ri = CurrentProduction_i / BaseAvg_i`
`Ii = Ri * 100`
Expanded:
`ItemIndex_i(y,m) = (MonthlyProduction_i(y,m) / BaseAvg_i) * 100`
Output table:
- `item_level_indices(product_id, current_production, base_year_avg_production, production_relative, item_index, ...)`
Interpretation:
- `100` means equal to base-year monthly average.
- `>100` means above base-year average.
- `<100` means below base-year average.
---
## Step 3: Calculate ISIC 4-Digit Indices
Function: `calculateISIC4DigitIndices(year, month)`
What it does:
- Assigns each product to ISIC group using `LEFT(establishments.isic_code, 4)`.
- Uses item-level index values belonging to that 4-digit group.
- Stores count and group index.
Implemented formulas:
`total_weight_g = COUNT(DISTINCT product_id)`
`weighted_index_sum_g = SUM(ItemIndex_i)`
`ISIC4Index_g = AVG(ItemIndex_i)`
Note:
- In this implementation, "weight" is the number of distinct products (equal weight per product), not an external official weight table.
Output table:
- `isic_4digit_indices(isic_4digit_code, total_weight, weighted_index_sum, isic_4digit_index, ...)`
---
## Step 4: Calculate ISIC 3-Digit Indices
Function: `calculateISIC3DigitIndices(year, month)`
What it does:
- Rolls up 4-digit results to 3-digit groups via `LEFT(isic_4digit_code, 3)`.
- Uses weighted average based on `total_weight` from 4-digit level.
Formulas:
`W_h = SUM(total_weight_g)` for all 4-digit groups `g` under 3-digit group `h`
`WeightedSum_h = SUM(total_weight_g * ISIC4Index_g)`
`ISIC3Index_h = WeightedSum_h / W_h`
Output table:
- `isic_3digit_indices(isic_3digit_code, total_weight, weighted_index_sum, isic_3digit_index, ...)`
---
## Step 5: Calculate ISIC 2-Digit Indices
Function: `calculateISIC2DigitIndices(year, month)`
What it does:
- Rolls up 3-digit results to 2-digit groups via `LEFT(isic_3digit_code, 2)`.
- Uses weighted average based on `total_weight` from 3-digit level.
Formulas:
`W_k = SUM(total_weight_h)` for all 3-digit groups `h` under 2-digit group `k`
`WeightedSum_k = SUM(total_weight_h * ISIC3Index_h)`
`ISIC2Index_k = WeightedSum_k / W_k`
Output table:
- `isic_2digit_indices(isic_2digit_code, total_weight, weighted_index_sum, isic_2digit_index, ...)`
---
## Step 6: Calculate Manufacturing IPI (Headline)
Function: `calculateManufacturingIPI(year, month)`
What it does:
- Computes overall manufacturing index from all 2-digit ISIC groups.
- Then computes Month-on-Month (MoM) and Year-on-Year (YoY) percentage changes.
- Writes final result in `manufacturing_ipi`.
Headline formulas:
`TotalWeight = SUM(total_weight_k)`
`TotalWeightedSum = SUM(total_weight_k * ISIC2Index_k)`
`ManufacturingIPI(y,m) = TotalWeightedSum / TotalWeight`
Change formulas:
`MoM(%) = ((IPI(y,m) - IPI(prev_month)) / IPI(prev_month)) * 100`
`YoY(%) = ((IPI(y,m) - IPI(y-1,m)) / IPI(y-1,m)) * 100`
Reference date:
- First day of month: `YYYY-MM-01`
Output table:
- `manufacturing_ipi(year, month, manufacturing_index, mom_change, yoy_change, total_weight, weighted_index_sum, status, generated_on, ...)`
---
## Logging and Transactions
Each calculation step:
- Opens its own DB transaction.
- Creates a start log entry in `calculation_log`.
- Updates log as Completed/Failed with record count.
- Commits on success and rolls back on error.
This means:
- A step is atomic by itself.
- The full pipeline is not one global transaction across all six steps.
---
## End-to-End Formula Chain (Compact View)
1. `BaseAvg_i = Avg monthly production in base year (2022)`
2. `ItemIndex_i = (CurrentProduction_i / BaseAvg_i) * 100`
3. `ISIC4Index = Avg(ItemIndex_i within ISIC4)`
4. `ISIC3Index = Weighted avg(ISIC4Index by ISIC4 total_weight)`
5. `ISIC2Index = Weighted avg(ISIC3Index by ISIC3 total_weight)`
6. `ManufacturingIPI = Weighted avg(ISIC2Index by ISIC2 total_weight)`
7. `MoM`, `YoY` from prior stored headline indices.
---
## Client Explanation Script (Short)
- We first aggregate monthly production quantity by product from approved submissions.
- Each product is compared against its base-year monthly average (2022), giving an item index where 100 equals base-year average output.
- Product indices are grouped by ISIC 4-digit, then rolled up to ISIC 3-digit and 2-digit using weighted averages.
- The final manufacturing IPI is the weighted average of all ISIC 2-digit indices.
- We then compute MoM and YoY growth rates using previously stored headline values.

147
ipi-methodology-cross-verification.md Normal file
View File

@ -0,0 +1,147 @@
# IPI Methodology Cross Verification (Updated)
## Scope
Cross-verification between:
- `Annex V - IPI Calculation Methodology.pptx`
- Current implementation in `app/services/ipi_calculation_service.js`
- Runtime trigger/orchestration in `server.js` and `app/services/scheduler.service.js`
## Trigger Path Verification
Verified start path:
- `server.js` boots scheduler via `scheduler.start()`.
- Cron `0 2 * * *` executes `runScheduledTask()`.
- `runScheduledTask()` calls `executeCalculateQuarter(year, quarter)`.
- `executeCalculateQuarter()` runs `runCompleteCalculation(year, month)` for each month.
- Manual endpoint `POST /api/admin/trigger-scheduler` triggers same scheduler flow.
Status: **MATCH**
---
## Overall Conclusion
Current code is **largely aligned on formula chain**, with **two remaining methodology gaps**:
1. product-to-ISIC4 mapping source is still an approximation (derived from establishment submissions, not a fixed official mapping table),
2. non-response imputation rules from PPT are not implemented in this service.
---
## Verification Matrix (Current Code vs PPT)
## 1) Base-year average production
PPT (Slides 9-10):
- Base year average uses arithmetic mean of 12 months.
Implementation:
- `calculateBaseYearProduction()` computes average across Jan-Dec and stores in `base_year_production.avg_by_production`.
Status: **MATCH**
---
## 2) Item-level index formula
PPT (Slides 8-10):
- `Ri = CurrentProduction / BaseYearAverage`
- `Ii = Ri * 100`
Implementation:
- `calculateItemLevelIndices()`:
- `production_relative = current_production / base_year_avg_production`
- `item_index = production_relative * 100`
Status: **MATCH**
---
## 3) ISIC 4-digit aggregation
PPT (Slides 11-12):
- Laspeyres weighted aggregation:
- `ISIC4 = SUM(w_i * I_i) / SUM(w_i)`
Implementation (current):
- Uses weighted formula with `products.weight_in_ib`:
- `total_weight = SUM(weight_in_ib)`
- `weighted_index_sum = SUM(weight_in_ib * item_index)`
- `isic_4digit_index = weighted_index_sum / total_weight`
- Prevents row multiplication by selecting one ISIC4 per product in-period using ranked mapping.
Status: **PARTIAL MATCH (Formula MATCH, Mapping APPROXIMATION)**
Reason for partial:
- Formula is aligned.
- Mapping source is not a fixed product master/official bridge; it is inferred from establishment-level submissions.
---
## 4) ISIC 3-digit aggregation
PPT (Slide 13):
- Weighted roll-up from ISIC4.
Implementation:
- `ISIC3 = SUM(total_weight_4 * isic_4digit_index) / SUM(total_weight_4)`
Status: **MATCH** (depends on ISIC4 inputs)
---
## 5) ISIC 2-digit aggregation
PPT (Slide 14):
- Weighted roll-up from ISIC3.
Implementation:
- `ISIC2 = SUM(total_weight_3 * isic_3digit_index) / SUM(total_weight_3)`
Status: **MATCH** (depends on ISIC3 inputs)
---
## 6) Manufacturing IPI (headline)
PPT (Slide 15):
- Weighted roll-up from ISIC2.
Implementation:
- `ManufacturingIPI = SUM(total_weight_2 * isic_2digit_index) / SUM(total_weight_2)`
Status: **MATCH**
Note:
- Current code includes defensive handling when no valid ISIC2 aggregates exist for a month.
---
## 7) Growth rates
PPT (Slide 16):
- YoY: `((Current - SameMonthLastYear) / SameMonthLastYear) * 100`
Implementation:
- YoY formula matches.
- MoM is also computed additionally.
Status:
- **YoY MATCH**
- **MoM Additional (not conflicting)**
---
## 8) Missing/non-response data treatment
PPT (Slide 3):
- For missing data, estimate using:
- previous month repeat, or
- average of last 3 months, or
- same month previous year.
Implementation:
- This imputation logic is not implemented in the IPI service pipeline.
- Pipeline uses available approved records.
Status: **MISMATCH**
---
## Final Assessment (Current Code)
If Annex V is interpreted strictly:
- **Formula chain:** mostly aligned now (item -> ISIC4 -> ISIC3 -> ISIC2 -> headline).
- **Remaining non-compliance points:** official deterministic item/product-to-ISIC mapping source and missing-data imputation policy.
Practical statement for stakeholders:
- Current implementation is **operationally correct and mathematically aligned for weighted aggregation**, but still requires **data governance alignment** (official mapping + imputation policy) for full Annex V compliance.