SS-FCSC/fcsc_ipi_backend
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Csv file changes added

Browse Source
This commit is contained in:
unknown 2025-12-11 15:22:14 +05:30
parent 180f34d483
commit 87f9b87a0d
5 changed files with 112 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
app/controllers/products.controller.js
View File

@ -217,7 +217,12 @@ const filePath = uploadedPath;
const cleanRow = {};
for (const key in row) {
const normalizedKey = key.replace(/[\s\W]+/g, "_").trim().toLowerCase();
let cleanedHeader = key
.replace(/\*/g, "")
.replace(/\(.*?\)/g, "")
.trim();
const normalizedKey = cleanedHeader.replace(/[\s\W]+/g, "_").trim().toLowerCase();
let mappedKey = normalizedKey;
if (/^hs.*code$/.test(normalizedKey) || normalizedKey === "hscode") {
@ -281,7 +286,10 @@ const filePath = uploadedPath;
errors.push({ row: index + 1, error: "HS Code must be max 10 digits" });
continue;
}
if (productName.length > 1000) {
errors.push({ row: index + 1, error: "Product Name is too long. Maximum allowed length is 1000 characters." });
continue;
}
if (description.length > 1000) {
errors.push({ row: index + 1, error: "HS Description is too long. Maximum allowed length is 1000 characters." });
continue;

164
app/controllers/unitMasterController.js
View File

@ -209,7 +209,7 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
});
}
const results = [];
let results = [];
fs.createReadStream(filePath)
.pipe(csv())
@ -224,24 +224,68 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
});
}
//Expected columns
const requiredCols = ["Unit Name", "Description"];
const headers = Object.keys(results[0] || {});
const missingCols = requiredCols.filter((col) => !headers.includes(col));
const extraCols = headers.filter((col) => !requiredCols.includes(col));
// -------------------------------------------------------------------
// NORMALIZE HEADERS (Very Important!)
// -------------------------------------------------------------------
const normalizeHeader = (h) => {
return h
.replace(/\*/g, "") // remove *
.replace(/\(.*?\)/g, "") // remove (Mandatory)
.trim()
.replace(/[\s\W]+/g, "_") // spaces & special chars -> _
.toLowerCase();
};
// Required normalized fields
const mappedRequiredCols = {
unit_name: "Unit Name",
description: "Description",
};
const incomingHeaders = Object.keys(results[0] || {});
const normalizedIncoming = incomingHeaders.map(h => normalizeHeader(h));
// Check missing columns
const missingCols = Object.keys(mappedRequiredCols).filter(
req => !normalizedIncoming.includes(req)
);
// Check unexpected columns
const extraCols = normalizedIncoming.filter(
col => !Object.keys(mappedRequiredCols).includes(col)
);
if (missingCols.length > 0 || extraCols.length > 0) {
fs.unlinkSync(filePath);
let msg = "";
if (missingCols.length > 0)
msg += `Missing required columns: ${missingCols.join(", ")}. `;
msg += `Missing required columns: ${missingCols.map(c => mappedRequiredCols[c]).join(", ")}. `;
if (extraCols.length > 0)
msg += `Unexpected columns found: ${extraCols.join(
", "
)}. Only 'Unit Name' and 'Description' are allowed.`;
return res.status(400).send({ status: "failed", message: msg.trim() });
msg += `Unexpected columns found: ${extraCols.join(", ")}. Only 'Unit Name' and 'Description' are allowed.`;
return res.status(400).send({
status: "failed",
message: msg.trim(),
});
}
// Remap row keys to clean headers
results = results.map(row => {
const newRow = {};
for (const key in row) {
const normalized = normalizeHeader(key);
const mapped = mappedRequiredCols[normalized];
if (mapped) newRow[mapped] = row[key];
}
return newRow;
});
// -------------------------------------------------------------------
// VALIDATION AND PROCESSING
// -------------------------------------------------------------------
const inserted = [];
const duplicates = [];
const errors = [];
@ -251,7 +295,7 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
const generateShortName = (unitName, existingShorts = new Set()) => {
const cleaned = unitName.replace(/[^a-zA-Z]/g, "").toUpperCase();
// Predefined abbreviations
const abbreviationMap = {
METER: "MT",
METRE: "MT",
@ -272,54 +316,59 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
UNIT: "UNT",
};
//if Unit Name is ≤ 5 characters, just use it in uppercase
if (cleaned.length <= 5) return cleaned;
// Otherwise, use abbreviation map or generate automatically
let shortName = abbreviationMap[cleaned] || cleaned.substring(0, 3);
if (shortName.length < 2) shortName = shortName.padEnd(2, "X");
// Random 2-letter suffix for uniqueness
const randomLetters = () =>
Array.from({ length: 2 }, () =>
String.fromCharCode(65 + Math.floor(Math.random() * 26))
).join("");
let finalShort = `${shortName}${randomLetters()}`;
while (existingShorts.has(finalShort.toLowerCase())) {
finalShort = `${shortName}${randomLetters()}`;
let final = `${shortName}${randomLetters()}`;
while (existingShorts.has(final.toLowerCase())) {
final = `${shortName}${randomLetters()}`;
}
return finalShort.substring(0, 5);
return final.substring(0, 5);
};
//Validate & prepare records
for (const [index, row] of results.entries()) {
// Row validation
for (const [i, row] of results.entries()) {
const uom = row["Unit Name"]?.trim();
const description = row["Description"]?.trim() || null;
if (description.length > 1000) {
errors.push({ row: i + 1, reason: "HS Description is too long. Maximum allowed length is 1000 characters." });
continue;
}
if (!uom) {
errors.push({ row: index + 1, reason: "Missing 'Unit Name'." });
errors.push({ row: i + 1, reason: "Missing 'Unit Name'." });
continue;
}
const uomKey = uom.toLowerCase();
if (seenUnitNames.has(uomKey)) {
fileDuplicates.push({
row: index + 1,
row: i + 1,
reason: "Duplicate Unit Name in file.",
uom,
});
continue;
}
let uomShort = generateShortName(uom, seenShortNames);
const uomShort = generateShortName(uom, seenShortNames);
seenUnitNames.add(uomKey);
seenShortNames.add(uomShort.toLowerCase());
results[index]._generatedShort = uomShort;
results[index]._description = description;
results[i]._generatedShort = uomShort;
results[i]._description = description;
}
// Stop if file has internal duplicates
if (fileDuplicates.length > 0) {
fs.unlinkSync(filePath);
return res.status(400).send({
@ -329,10 +378,10 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
});
}
//Insert into DB with safe duplicate handling
for (const [index, row] of results.entries()) {
// Insert into DB
for (const [i, row] of results.entries()) {
try {
const uom = row["Unit Name"]?.trim();
const uom = row["Unit Name"].trim();
let uomShort = row._generatedShort;
const description = row._description;
@ -352,34 +401,12 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
});
if (existing) {
// Skip if same Unit Name already exists
if (existing.uom.toLowerCase() === uom.toLowerCase()) {
duplicates.push({
id: existing.id,
uom: existing.uom,
uom_short_name: existing.uom_short_name,
});
continue;
}
//Safe regeneration using ternary inside while loop
let newShort = uomShort;
let existsInDb;
do {
existsInDb = await UnitMaster.findOne({
where: Sequelize.where(
Sequelize.fn("LOWER", Sequelize.col("uom_short_name")),
newShort.toLowerCase()
),
});
newShort = existsInDb
? `${uomShort}${Math.floor(Math.random() * 1000)}`.substring(0, 5)
: newShort;
} while (existsInDb);
uomShort = newShort;
duplicates.push({
id: existing.id,
uom: existing.uom,
uom_short_name: existing.uom_short_name,
});
continue;
}
const newUnit = await UnitMaster.create({
@ -391,13 +418,18 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
});
inserted.push(newUnit);
} catch (err) {
errors.push({ row: index + 1, reason: err.message });
errors.push({ row: i + 1, reason: err.message });
}
}
fs.unlinkSync(filePath);
// -------------------------------------------------------------------
// FINAL RESPONSE
// -------------------------------------------------------------------
let finalStatus = "success";
let message = `${inserted.length} units inserted successfully.`;
@ -421,6 +453,7 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
duplicates,
error_details: errors,
});
} catch (err) {
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
return res.status(500).send({
@ -431,7 +464,9 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
});
} catch (error) {
console.error("Error uploading Unit Master CSV:", error);
if (req.file && fs.existsSync(path.resolve(req.file.path))) fs.unlinkSync(path.resolve(req.file.path));
if (req.file && fs.existsSync(path.resolve(req.file.path)))
fs.unlinkSync(path.resolve(req.file.path));
return res.status(500).send({
status: "failed",
message: "Error processing CSV file.",
@ -440,15 +475,6 @@ exports.uploadUnitMasterFromCSV = async (req, res) => {
}
};
// exports.downloadUnitMasterFile = async (req, res) => {
// try {
// const filePath = path.join(__dirname, "../uploads/unit_master_sample.csv");
// return res.download(filePath, "unit_master_sample.csv");
// } catch (error) {
// return res.status(500).send({ status: "failed", message: error.message });
// }
// };
exports.downloadUnitMasterFile = async (req, res) => {
try {
const safeBasePath = path.resolve(__dirname, "../downloads_csv");

2
app/downloads_csv/products_upload_sample.csv
View File

@ -1 +1 @@
HS Code,Product Name,Unit,Description
HS Code * (Mandatory),Product Name * (Mandatory),Unit * (Mandatory),Description

1 HS Code HS Code * (Mandatory) Product Name Product Name * (Mandatory) Unit Unit * (Mandatory) Description

2
app/downloads_csv/unit_master_sample.csv
View File

@ -1 +1 @@
Unit Name,Description
Unit Name * (Mandatory),Description * (Mandatory)

1 Unit Name Unit Name * (Mandatory) Description Description * (Mandatory)

5
app/models/UnitMaster.model.js
View File

@ -31,6 +31,11 @@ module.exports = (sequelize, DataTypes) => {
description: {
type: DataTypes.STRING,
allowNull: true,
validate: {
len: {
args: [0, 1000],
msg: "Description is too long. Maximum allowed length is 1000 characters."
}}
},
is_active: {
type: DataTypes.BOOLEAN,