bulk upload csv file changes
This commit is contained in:
unknown
parent
14da0fb727
commit
acf9a83558
@ -1107,7 +1107,7 @@ exports.establishmentBulkUpload = async (req, res) => {
|
|||||||
const rows = [];
|
const rows = [];
|
||||||
|
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
// READ CSV AND SKIP EMPTY ROWS
|
// READ CSV AND NORMALIZE HEADERS
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
await new Promise((resolve, reject) => {
|
await new Promise((resolve, reject) => {
|
||||||
fs.createReadStream(filePath)
|
fs.createReadStream(filePath)
|
||||||
@ -1115,20 +1115,18 @@ exports.establishmentBulkUpload = async (req, res) => {
|
|||||||
.on("data", (rawRow) => {
|
.on("data", (rawRow) => {
|
||||||
const row = {};
|
const row = {};
|
||||||
|
|
||||||
// Normalize headers
|
|
||||||
for (const key in rawRow) {
|
for (const key in rawRow) {
|
||||||
const normalizedKey = key.replace(/[\s\W]+/g, "_")
|
const normalizedKey = key
|
||||||
|
.replace(/[\s\W]+/g, "_")
|
||||||
.trim()
|
.trim()
|
||||||
.toLowerCase();
|
.toLowerCase();
|
||||||
|
|
||||||
row[normalizedKey] = rawRow[key]?.trim() || "";
|
row[normalizedKey] = rawRow[key]?.trim() || "";
|
||||||
}
|
}
|
||||||
|
|
||||||
const cleanedValues = Object.values(row).map(v =>(v || "").replace(/\s+/g, "").trim());
|
const cleaned = Object.values(row).map(v => (v || "").replace(/\s+/g, "").trim());
|
||||||
const isEmpty = cleanedValues.every(v => v === "");
|
const empty = cleaned.every(v => v === "");
|
||||||
if (isEmpty) return;
|
if (!empty) rows.push(row);
|
||||||
|
|
||||||
rows.push(row);
|
|
||||||
})
|
})
|
||||||
.on("end", resolve)
|
.on("end", resolve)
|
||||||
.on("error", reject);
|
.on("error", reject);
|
||||||
@ -1138,86 +1136,66 @@ exports.establishmentBulkUpload = async (req, res) => {
|
|||||||
fs.unlinkSync(filePath);
|
fs.unlinkSync(filePath);
|
||||||
return res.status(400).send({
|
return res.status(400).send({
|
||||||
status: "failed",
|
status: "failed",
|
||||||
message: "CSV file is empty.",
|
message: "CSV file is empty."
|
||||||
errors: [{ error: "Uploaded file is empty." }]
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
// -------------------------------------------------------
|
|
||||||
// REQUIRED HEADER CHECK
|
|
||||||
// -------------------------------------------------------
|
|
||||||
const required = [
|
|
||||||
"establishment_id",
|
|
||||||
"factory_name",
|
|
||||||
"email",
|
|
||||||
"emirate",
|
|
||||||
"total_employment"
|
|
||||||
];
|
|
||||||
|
|
||||||
const firstRowKeys = Object.keys(rows[0]);
|
// -------------------------------------------------------
|
||||||
|
// REQUIRED HEADERS
|
||||||
|
// -------------------------------------------------------
|
||||||
|
const required = ["establishment_id", "factory_name", "user_name", "email", "emirate"];
|
||||||
|
const firstKeys = Object.keys(rows[0]);
|
||||||
|
|
||||||
const missing = required.filter(r => !firstRowKeys.includes(r));
|
const missingHeaders = required.filter(h => !firstKeys.includes(h));
|
||||||
if (missing.length > 0) {
|
if (missingHeaders.length > 0) {
|
||||||
fs.unlinkSync(filePath);
|
fs.unlinkSync(filePath);
|
||||||
return res.status(400).send({
|
return res.status(400).send({
|
||||||
status: "failed",
|
status: "failed",
|
||||||
message: "Invalid data. Please check the instructions given and upload again.",
|
message: "Missing required columns: " + missingHeaders.join(", ")
|
||||||
errors: [
|
|
||||||
{ error: "Missing required columns: " + missing.join(", ") }
|
|
||||||
]
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
// LOAD REFERENCE TABLES
|
// LOAD REFERENCE TABLES ONCE
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
const emirates = await Emirate.findAll({ attributes: ["id", "name"] });
|
const emirates = await Emirate.findAll({ attributes: ["id", "name"] });
|
||||||
const emirateMap = {};
|
const emirateMap = {};
|
||||||
emirates.forEach(e => {
|
emirates.forEach(e => (emirateMap[e.name.trim().toLowerCase()] = e.id));
|
||||||
emirateMap[e.name.trim().toLowerCase()] = e.id;
|
|
||||||
});
|
const cityTowns = await CityTown.findAll({ attributes: ["id", "name"] });
|
||||||
|
const cityTownMap = {};
|
||||||
|
cityTowns.forEach(ct => (cityTownMap[ct.name.trim().toLowerCase()] = ct.id));
|
||||||
|
|
||||||
function normalizeHS(val) {
|
function normalizeHS(val) {
|
||||||
if (!val) return "";
|
if (!val) return "";
|
||||||
|
return val.toString().normalize("NFKD").replace(/[^\d]/g, "").trim();
|
||||||
return val
|
|
||||||
.toString()
|
|
||||||
.normalize("NFKD")
|
|
||||||
.replace(/[^\d]/g, "")
|
|
||||||
.trim();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const products = await Product.findAll({ attributes: ["id", "hs_code"] });
|
const products = await Product.findAll({ attributes: ["id", "hs_code"] });
|
||||||
|
|
||||||
const productMap = {};
|
const productMap = {};
|
||||||
|
|
||||||
products.forEach(p => {
|
products.forEach(p => {
|
||||||
const cleaned = normalizeHS(p.hs_code);
|
const c = normalizeHS(p.hs_code);
|
||||||
|
if (c.length >= 1 && c.length <= 10) productMap[c] = p.id;
|
||||||
// Only store valid HS codes 1–10 digits
|
|
||||||
if (cleaned.length >= 1 && cleaned.length <= 10) {
|
|
||||||
productMap[cleaned] = p.id;
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
const existingEsts = await Establishment.findAll({
|
|
||||||
|
const existing = await Establishment.findAll({
|
||||||
attributes: ["establishment_code", "factory_name", "establishment_contact_email"]
|
attributes: ["establishment_code", "factory_name", "establishment_contact_email"]
|
||||||
});
|
});
|
||||||
|
|
||||||
const existingEstIdSet = new Set(existingEsts.map(e => e.establishment_code));
|
const existingEstSet = new Set(existing.map(e => e.establishment_code));
|
||||||
const existingFactorySet = new Set(existingEsts.map(e => e.factory_name.toLowerCase()));
|
const existingFactorySet = new Set(existing.map(e => e.factory_name.toLowerCase()));
|
||||||
const existingEmailSet = new Set(existingEsts.map(e => e.establishment_contact_email.toLowerCase()));
|
const existingEmailSet = new Set(existing.map(e => e.establishment_contact_email.toLowerCase()));
|
||||||
|
|
||||||
// -------------------------------------------------------
|
// FILE duplicate trackers
|
||||||
// FILE-LEVEL DUPLICATE TRACKING
|
|
||||||
// -------------------------------------------------------
|
|
||||||
const errors = [];
|
|
||||||
const fileEstSet = new Set();
|
const fileEstSet = new Set();
|
||||||
const fileFactorySet = new Set();
|
const fileFactorySet = new Set();
|
||||||
const fileEmailSet = new Set();
|
const fileEmailSet = new Set();
|
||||||
|
|
||||||
|
const errors = [];
|
||||||
const prepared = [];
|
const prepared = [];
|
||||||
|
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
// VALIDATE EACH ROW
|
// PHASE 1: VALIDATE ALL ROWS (NO DB INSERT HERE)
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
for (let i = 0; i < rows.length; i++) {
|
for (let i = 0; i < rows.length; i++) {
|
||||||
const r = rows[i];
|
const r = rows[i];
|
||||||
@ -1227,42 +1205,24 @@ exports.establishmentBulkUpload = async (req, res) => {
|
|||||||
const factory = r.factory_name;
|
const factory = r.factory_name;
|
||||||
const email = r.email;
|
const email = r.email;
|
||||||
const emirate = r.emirate;
|
const emirate = r.emirate;
|
||||||
const employment = r.total_employment;
|
|
||||||
|
|
||||||
// Required validation
|
const missing = [];
|
||||||
const missingFields = [];
|
if (!est) missing.push("Establishment ID");
|
||||||
|
if (!factory) missing.push("Factory Name");
|
||||||
|
if (!r.user_name) missing.push("User Name");
|
||||||
|
if (!email) missing.push("Email");
|
||||||
|
if (!emirate) missing.push("Emirate");
|
||||||
|
|
||||||
if (!est) missingFields.push("Establishment ID");
|
if (missing.length) {
|
||||||
if (!factory) missingFields.push("Factory Name");
|
errors.push({ row: rowNum, error: `Missing required fields: ${missing.join(", ")}` });
|
||||||
if (!email) missingFields.push("Email");
|
|
||||||
if (!emirate) missingFields.push("Emirate");
|
|
||||||
if (!employment) missingFields.push("Total Employment");
|
|
||||||
|
|
||||||
if (missingFields.length > 0) {
|
|
||||||
errors.push({
|
|
||||||
row: rowNum,
|
|
||||||
error: `Missing Data: ${missingFields.join(", ")} ${missingFields.length === 1 ? "is" : "are"} required.`
|
|
||||||
});
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isNaN(employment) || !/^\d+$/.test(employment)) {
|
if (!/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email)) {
|
||||||
errors.push({
|
errors.push({ row: rowNum, error: "Invalid Email format" });
|
||||||
row: rowNum,
|
|
||||||
error: "Total Employment must be numeric."
|
|
||||||
});
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
|
||||||
|
|
||||||
if (!emailRegex.test(email)) {
|
|
||||||
errors.push({
|
|
||||||
row: rowNum,
|
|
||||||
error: "Invalid Email: Please check the Email address."
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// FILE-level duplicates
|
// FILE-level duplicates
|
||||||
const estKey = est.trim();
|
const estKey = est.trim();
|
||||||
const factoryKey = factory.trim().toLowerCase();
|
const factoryKey = factory.trim().toLowerCase();
|
||||||
@ -1286,7 +1246,7 @@ exports.establishmentBulkUpload = async (req, res) => {
|
|||||||
fileEmailSet.add(emailKey);
|
fileEmailSet.add(emailKey);
|
||||||
|
|
||||||
// DB-level duplicates
|
// DB-level duplicates
|
||||||
if (existingEstIdSet.has(estKey)) {
|
if (existingEstSet.has(estKey)) {
|
||||||
errors.push({ row: rowNum, error: `Establishment ID already exists: ${est}` });
|
errors.push({ row: rowNum, error: `Establishment ID already exists: ${est}` });
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@ -1306,116 +1266,127 @@ exports.establishmentBulkUpload = async (req, res) => {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// -------------------------------------------------------
|
// HS Code validation
|
||||||
// HS CODE VALIDATION
|
|
||||||
// -------------------------------------------------------
|
|
||||||
const hsCols = Object.keys(r).filter(k => k.startsWith("hs"));
|
const hsCols = Object.keys(r).filter(k => k.startsWith("hs"));
|
||||||
const hsRaw = hsCols.map(k => r[k]).filter(Boolean);
|
const hsRaw = hsCols.map(k => r[k]).filter(Boolean);
|
||||||
|
|
||||||
if (hsRaw.length === 0) {
|
if (hsRaw.length === 0) {
|
||||||
errors.push({ row: rowNum, error: "At least one HS Code is required" });
|
errors.push({ row: rowNum, error: "At least one HS code is required" });
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clean & remove leading zeros
|
const cleaned = hsRaw.map(h => normalizeHS(h));
|
||||||
const cleaned = hsRaw.map(v =>
|
const invalid = cleaned.filter(c => !productMap[c]);
|
||||||
v.replace(/[^0-9]/g, "").replace(/^0+/, "")
|
|
||||||
);
|
|
||||||
|
|
||||||
// Check duplicates in same row
|
|
||||||
const rowDuplicates = cleaned.filter((c, idx) => cleaned.indexOf(c) !== idx);
|
|
||||||
if (rowDuplicates.length > 0) {
|
|
||||||
errors.push({
|
|
||||||
row: rowNum,
|
|
||||||
error: `Duplicate HS Code(s) in the same row: ${[...new Set(rowDuplicates)].join(", ")}`
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// DB-level HS Code validation
|
|
||||||
const invalid = [];
|
|
||||||
|
|
||||||
for (const raw of hsRaw) {
|
|
||||||
const cleaned = normalizeHS(raw);
|
|
||||||
|
|
||||||
if (cleaned.length < 1 || cleaned.length > 10) {
|
|
||||||
invalid.push(raw);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!productMap[cleaned]) {
|
|
||||||
invalid.push(raw);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (invalid.length > 0) {
|
if (invalid.length > 0) {
|
||||||
errors.push({
|
errors.push({ row: rowNum, error: `Invalid HS Code(s): ${invalid.join(", ")}` });
|
||||||
row: rowNum,
|
|
||||||
error: `HS Code not found in database: ${invalid.join(", ")}`
|
|
||||||
});
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
prepared.push({
|
prepared.push({
|
||||||
|
r,
|
||||||
|
rowNum,
|
||||||
est,
|
est,
|
||||||
factory,
|
factory,
|
||||||
email,
|
email,
|
||||||
emirateId,
|
emirateId,
|
||||||
employment: parseInt(employment),
|
productIds: cleaned.map(c => productMap[c])
|
||||||
productIds: cleaned.map(c => productMap[c]),
|
|
||||||
rowNum
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
// STOP IF ERRORS
|
// STOP IF ANY ERRORS FROM PHASE 1
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
if (errors.length > 0) {
|
if (errors.length > 0) {
|
||||||
logger.error("Establishment Upload Failed: " + JSON.stringify(errors));
|
|
||||||
fs.unlinkSync(filePath);
|
fs.unlinkSync(filePath);
|
||||||
return res.status(400).send({
|
return res.status(400).send({
|
||||||
status: "failed",
|
status: "failed",
|
||||||
message: "Invalid data. Please check the instructions given and upload again.",
|
message: "Invalid data. Please check and upload again.",
|
||||||
errors
|
errors
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
// INSERT VALID RECORDS
|
// PHASE 2: CITY/TOWN VALIDATION (OPTIONAL BUT MUST BE VALID)
|
||||||
// -------------------------------------------------------
|
// -------------------------------------------------------
|
||||||
for (const rec of prepared) {
|
for (const p of prepared) {
|
||||||
|
const r = p.r;
|
||||||
|
const name = r.city_town?.trim().toLowerCase() || "";
|
||||||
|
|
||||||
|
if (name) {
|
||||||
|
const id = cityTownMap[name];
|
||||||
|
if (!id) {
|
||||||
|
fs.unlinkSync(filePath);
|
||||||
|
return res.status(400).send({
|
||||||
|
status: "failed",
|
||||||
|
message: "Invalid City/Town found. Upload stopped.",
|
||||||
|
errors: [{ row: p.rowNum, error: `Invalid City/Town: ${r.city_town}` }]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
r.city_town_id = id;
|
||||||
|
} else {
|
||||||
|
r.city_town_id = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -------------------------------------------------------
|
||||||
|
// PHASE 3: INSERT INTO DATABASE
|
||||||
|
// -------------------------------------------------------
|
||||||
|
for (const p of prepared) {
|
||||||
|
const r = p.r;
|
||||||
|
|
||||||
const est = await Establishment.create({
|
const est = await Establishment.create({
|
||||||
establishment_code: rec.est,
|
establishment_code: p.est,
|
||||||
factory_name: rec.factory,
|
factory_name: p.factory,
|
||||||
establishment_contact_email: rec.email,
|
establishment_contact_email: p.email,
|
||||||
establishment_emirate_id: rec.emirateId,
|
establishment_emirate_id: p.emirateId,
|
||||||
total_employees: rec.employment,
|
|
||||||
|
permanent_factory_code: r.permanent_factory_code,
|
||||||
|
industry_code: r.industry_code_business_register,
|
||||||
|
industry_code_production: r.industry_code_current_production,
|
||||||
|
industry_code_mismatch_remarks: r.industry_code_mismatch_remarks,
|
||||||
|
description: r.description,
|
||||||
|
establishment_address: r.establishment_address,
|
||||||
|
establishment_city_town_id: r.city_town_id,
|
||||||
|
establishment_postal_code: r.postal_code,
|
||||||
|
establishment_po_box: r.po_box,
|
||||||
|
establishment_makani_number: r.makani_number,
|
||||||
|
establishment_contact_person_name: r.contact_person_name,
|
||||||
|
establishment_contact_person_designation: r.contact_person_designation,
|
||||||
|
establishment_mobile_number: r.mobile_number,
|
||||||
|
establishment_website: r.website,
|
||||||
|
|
||||||
|
emirati_male: Number(r.number_of_emirati_male || 0),
|
||||||
|
emirati_female: Number(r.number_of_emirati_female || 0),
|
||||||
|
non_emirati_male: Number(r.number_of_non_emirati_male || 0),
|
||||||
|
non_emirati_female: Number(r.number_of_non_emirati_female || 0),
|
||||||
|
|
||||||
|
total_emirati:
|
||||||
|
Number(r.number_of_emirati_male || 0) +
|
||||||
|
Number(r.number_of_emirati_female || 0),
|
||||||
|
|
||||||
|
total_employees:
|
||||||
|
Number(r.number_of_emirati_male || 0) +
|
||||||
|
Number(r.number_of_emirati_female || 0) +
|
||||||
|
Number(r.number_of_non_emirati_male || 0) +
|
||||||
|
Number(r.number_of_non_emirati_female || 0),
|
||||||
|
|
||||||
created_by: req.user.id
|
created_by: req.user.id
|
||||||
});
|
});
|
||||||
|
|
||||||
const autoPassword = Math.random().toString(36).slice(-10);
|
const autoPassword = Math.random().toString(36).slice(-10);
|
||||||
const hashedPassword = await bcrypt.hash(autoPassword, 10);
|
const hashed = await bcrypt.hash(autoPassword, 10);
|
||||||
const user = await EstablishmentUser.create({
|
|
||||||
|
await EstablishmentUser.create({
|
||||||
establishment_id: est.id,
|
establishment_id: est.id,
|
||||||
name: rec.factory,
|
name: r.user_name,
|
||||||
email: rec.email,
|
email: p.email,
|
||||||
password: hashedPassword,
|
password: hashed,
|
||||||
created_by: req.user.id
|
created_by: req.user.id
|
||||||
});
|
});
|
||||||
|
|
||||||
const placeHolderData = {
|
|
||||||
contact_name: rec.factory || rec.est,
|
|
||||||
portal_url: process.env.FE_BASE_URL,
|
|
||||||
username: rec.email,
|
|
||||||
password: autoPassword,
|
|
||||||
support_email: process.env.SUPPORT_EMAIL,
|
|
||||||
support_phone: process.env.SUPPORT_PHONE
|
|
||||||
};
|
|
||||||
|
|
||||||
await sendEmailService(rec.email, "establishment_user_creation_to_user", placeHolderData);
|
|
||||||
|
|
||||||
await EstablishmentProduct.bulkCreate(
|
await EstablishmentProduct.bulkCreate(
|
||||||
rec.productIds.map(pid => ({
|
p.productIds.map(pid => ({
|
||||||
establishment_id: est.id,
|
establishment_id: est.id,
|
||||||
product_id: pid,
|
product_id: pid,
|
||||||
created_by: req.user.id
|
created_by: req.user.id
|
||||||
@ -1436,15 +1407,375 @@ exports.establishmentBulkUpload = async (req, res) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error("Establishment Import Fatal Error: " + err.message);
|
logger.error("Fatal Error: " + err.message);
|
||||||
|
|
||||||
return res.status(500).send({
|
return res.status(500).send({
|
||||||
status: "failed",
|
status: "failed",
|
||||||
message: "Invalid data. Please check the instructions given and upload again."
|
message: "Unexpected error occurred."
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
// exports.establishmentBulkUpload = async (req, res) => {
|
||||||
|
// try {
|
||||||
|
// if (!req.file) {
|
||||||
|
// return res.status(400).send({
|
||||||
|
// status: "failed",
|
||||||
|
// message: "No file uploaded."
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
|
||||||
|
// const filePath = req.file.path;
|
||||||
|
|
||||||
|
// if (!req.file.originalname.toLowerCase().endsWith(".csv")) {
|
||||||
|
// fs.unlinkSync(filePath);
|
||||||
|
// return res.status(400).send({
|
||||||
|
// status: "failed",
|
||||||
|
// message: "Invalid file type. Only CSV allowed."
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
|
||||||
|
// const rows = [];
|
||||||
|
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// // READ CSV AND SKIP EMPTY ROWS
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// await new Promise((resolve, reject) => {
|
||||||
|
// fs.createReadStream(filePath)
|
||||||
|
// .pipe(csv())
|
||||||
|
// .on("data", (rawRow) => {
|
||||||
|
// const row = {};
|
||||||
|
|
||||||
|
// // Normalize headers
|
||||||
|
// for (const key in rawRow) {
|
||||||
|
// const normalizedKey = key.replace(/[\s\W]+/g, "_")
|
||||||
|
// .trim()
|
||||||
|
// .toLowerCase();
|
||||||
|
|
||||||
|
// row[normalizedKey] = rawRow[key]?.trim() || "";
|
||||||
|
// }
|
||||||
|
|
||||||
|
// const cleanedValues = Object.values(row).map(v =>(v || "").replace(/\s+/g, "").trim());
|
||||||
|
// const isEmpty = cleanedValues.every(v => v === "");
|
||||||
|
// if (isEmpty) return;
|
||||||
|
|
||||||
|
// rows.push(row);
|
||||||
|
// })
|
||||||
|
// .on("end", resolve)
|
||||||
|
// .on("error", reject);
|
||||||
|
// });
|
||||||
|
|
||||||
|
// if (rows.length === 0) {
|
||||||
|
// fs.unlinkSync(filePath);
|
||||||
|
// return res.status(400).send({
|
||||||
|
// status: "failed",
|
||||||
|
// message: "CSV file is empty.",
|
||||||
|
// errors: [{ error: "Uploaded file is empty." }]
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// // REQUIRED HEADER CHECK
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// const required = [
|
||||||
|
// "establishment_id",
|
||||||
|
// "factory_name",
|
||||||
|
// "email",
|
||||||
|
// "emirate",
|
||||||
|
// "total_employment"
|
||||||
|
// ];
|
||||||
|
|
||||||
|
// const firstRowKeys = Object.keys(rows[0]);
|
||||||
|
|
||||||
|
// const missing = required.filter(r => !firstRowKeys.includes(r));
|
||||||
|
// if (missing.length > 0) {
|
||||||
|
// fs.unlinkSync(filePath);
|
||||||
|
// return res.status(400).send({
|
||||||
|
// status: "failed",
|
||||||
|
// message: "Invalid data. Please check the instructions given and upload again.",
|
||||||
|
// errors: [
|
||||||
|
// { error: "Missing required columns: " + missing.join(", ") }
|
||||||
|
// ]
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// // LOAD REFERENCE TABLES
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// const emirates = await Emirate.findAll({ attributes: ["id", "name"] });
|
||||||
|
// const emirateMap = {};
|
||||||
|
// emirates.forEach(e => {
|
||||||
|
// emirateMap[e.name.trim().toLowerCase()] = e.id;
|
||||||
|
// });
|
||||||
|
|
||||||
|
// function normalizeHS(val) {
|
||||||
|
// if (!val) return "";
|
||||||
|
|
||||||
|
// return val
|
||||||
|
// .toString()
|
||||||
|
// .normalize("NFKD")
|
||||||
|
// .replace(/[^\d]/g, "")
|
||||||
|
// .trim();
|
||||||
|
// }
|
||||||
|
|
||||||
|
// const products = await Product.findAll({ attributes: ["id", "hs_code"] });
|
||||||
|
|
||||||
|
// const productMap = {};
|
||||||
|
|
||||||
|
// products.forEach(p => {
|
||||||
|
// const cleaned = normalizeHS(p.hs_code);
|
||||||
|
|
||||||
|
// // Only store valid HS codes 1–10 digits
|
||||||
|
// if (cleaned.length >= 1 && cleaned.length <= 10) {
|
||||||
|
// productMap[cleaned] = p.id;
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
// const existingEsts = await Establishment.findAll({
|
||||||
|
// attributes: ["establishment_code", "factory_name", "establishment_contact_email"]
|
||||||
|
// });
|
||||||
|
|
||||||
|
// const existingEstIdSet = new Set(existingEsts.map(e => e.establishment_code));
|
||||||
|
// const existingFactorySet = new Set(existingEsts.map(e => e.factory_name.toLowerCase()));
|
||||||
|
// const existingEmailSet = new Set(existingEsts.map(e => e.establishment_contact_email.toLowerCase()));
|
||||||
|
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// // FILE-LEVEL DUPLICATE TRACKING
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// const errors = [];
|
||||||
|
// const fileEstSet = new Set();
|
||||||
|
// const fileFactorySet = new Set();
|
||||||
|
// const fileEmailSet = new Set();
|
||||||
|
|
||||||
|
// const prepared = [];
|
||||||
|
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// // VALIDATE EACH ROW
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// for (let i = 0; i < rows.length; i++) {
|
||||||
|
// const r = rows[i];
|
||||||
|
// const rowNum = i + 2;
|
||||||
|
|
||||||
|
// const est = r.establishment_id;
|
||||||
|
// const factory = r.factory_name;
|
||||||
|
// const email = r.email;
|
||||||
|
// const emirate = r.emirate;
|
||||||
|
// const employment = r.total_employment;
|
||||||
|
|
||||||
|
// // Required validation
|
||||||
|
// const missingFields = [];
|
||||||
|
|
||||||
|
// if (!est) missingFields.push("Establishment ID");
|
||||||
|
// if (!factory) missingFields.push("Factory Name");
|
||||||
|
// if (!email) missingFields.push("Email");
|
||||||
|
// if (!emirate) missingFields.push("Emirate");
|
||||||
|
// if (!employment) missingFields.push("Total Employment");
|
||||||
|
|
||||||
|
// if (missingFields.length > 0) {
|
||||||
|
// errors.push({
|
||||||
|
// row: rowNum,
|
||||||
|
// error: `Missing Data: ${missingFields.join(", ")} ${missingFields.length === 1 ? "is" : "are"} required.`
|
||||||
|
// });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// if (isNaN(employment) || !/^\d+$/.test(employment)) {
|
||||||
|
// errors.push({
|
||||||
|
// row: rowNum,
|
||||||
|
// error: "Total Employment must be numeric."
|
||||||
|
// });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
||||||
|
|
||||||
|
// if (!emailRegex.test(email)) {
|
||||||
|
// errors.push({
|
||||||
|
// row: rowNum,
|
||||||
|
// error: "Invalid Email: Please check the Email address."
|
||||||
|
// });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
// // FILE-level duplicates
|
||||||
|
// const estKey = est.trim();
|
||||||
|
// const factoryKey = factory.trim().toLowerCase();
|
||||||
|
// const emailKey = email.trim().toLowerCase();
|
||||||
|
|
||||||
|
// if (fileEstSet.has(estKey)) {
|
||||||
|
// errors.push({ row: rowNum, error: `Duplicate Establishment ID in file: ${est}` });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
// if (fileFactorySet.has(factoryKey)) {
|
||||||
|
// errors.push({ row: rowNum, error: `Duplicate Factory Name in file: ${factory}` });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
// if (fileEmailSet.has(emailKey)) {
|
||||||
|
// errors.push({ row: rowNum, error: `Duplicate Email in file: ${email}` });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// fileEstSet.add(estKey);
|
||||||
|
// fileFactorySet.add(factoryKey);
|
||||||
|
// fileEmailSet.add(emailKey);
|
||||||
|
|
||||||
|
// // DB-level duplicates
|
||||||
|
// if (existingEstIdSet.has(estKey)) {
|
||||||
|
// errors.push({ row: rowNum, error: `Establishment ID already exists: ${est}` });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
// if (existingFactorySet.has(factoryKey)) {
|
||||||
|
// errors.push({ row: rowNum, error: `Factory Name already exists: ${factory}` });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
// if (existingEmailSet.has(emailKey)) {
|
||||||
|
// errors.push({ row: rowNum, error: `Email already exists: ${email}` });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // Emirate validation
|
||||||
|
// const emirateId = emirateMap[emirate.toLowerCase()];
|
||||||
|
// if (!emirateId) {
|
||||||
|
// errors.push({ row: rowNum, error: `Invalid Emirate: ${emirate}` });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// // HS CODE VALIDATION
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// const hsCols = Object.keys(r).filter(k => k.startsWith("hs"));
|
||||||
|
// const hsRaw = hsCols.map(k => r[k]).filter(Boolean);
|
||||||
|
|
||||||
|
// if (hsRaw.length === 0) {
|
||||||
|
// errors.push({ row: rowNum, error: "At least one HS Code is required" });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // Clean & remove leading zeros
|
||||||
|
// const cleaned = hsRaw.map(v =>
|
||||||
|
// v.replace(/[^0-9]/g, "").replace(/^0+/, "")
|
||||||
|
// );
|
||||||
|
|
||||||
|
// // Check duplicates in same row
|
||||||
|
// const rowDuplicates = cleaned.filter((c, idx) => cleaned.indexOf(c) !== idx);
|
||||||
|
// if (rowDuplicates.length > 0) {
|
||||||
|
// errors.push({
|
||||||
|
// row: rowNum,
|
||||||
|
// error: `Duplicate HS Code(s) in the same row: ${[...new Set(rowDuplicates)].join(", ")}`
|
||||||
|
// });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // DB-level HS Code validation
|
||||||
|
// const invalid = [];
|
||||||
|
|
||||||
|
// for (const raw of hsRaw) {
|
||||||
|
// const cleaned = normalizeHS(raw);
|
||||||
|
|
||||||
|
// if (cleaned.length < 1 || cleaned.length > 10) {
|
||||||
|
// invalid.push(raw);
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// if (!productMap[cleaned]) {
|
||||||
|
// invalid.push(raw);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// if (invalid.length > 0) {
|
||||||
|
// errors.push({
|
||||||
|
// row: rowNum,
|
||||||
|
// error: `HS Code not found in database: ${invalid.join(", ")}`
|
||||||
|
// });
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// prepared.push({
|
||||||
|
// est,
|
||||||
|
// factory,
|
||||||
|
// email,
|
||||||
|
// emirateId,
|
||||||
|
// employment: parseInt(employment),
|
||||||
|
// productIds: cleaned.map(c => productMap[c]),
|
||||||
|
// rowNum
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// // STOP IF ERRORS
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// if (errors.length > 0) {
|
||||||
|
// logger.error("Establishment Upload Failed: " + JSON.stringify(errors));
|
||||||
|
// fs.unlinkSync(filePath);
|
||||||
|
// return res.status(400).send({
|
||||||
|
// status: "failed",
|
||||||
|
// message: "Invalid data. Please check the instructions given and upload again.",
|
||||||
|
// errors
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// // INSERT VALID RECORDS
|
||||||
|
// // -------------------------------------------------------
|
||||||
|
// for (const rec of prepared) {
|
||||||
|
// const est = await Establishment.create({
|
||||||
|
// establishment_code: rec.est,
|
||||||
|
// factory_name: rec.factory,
|
||||||
|
// establishment_contact_email: rec.email,
|
||||||
|
// establishment_emirate_id: rec.emirateId,
|
||||||
|
// total_employees: rec.employment,
|
||||||
|
// created_by: req.user.id
|
||||||
|
// });
|
||||||
|
|
||||||
|
// const autoPassword = Math.random().toString(36).slice(-10);
|
||||||
|
// const hashedPassword = await bcrypt.hash(autoPassword, 10);
|
||||||
|
// const user = await EstablishmentUser.create({
|
||||||
|
// establishment_id: est.id,
|
||||||
|
// name: rec.factory,
|
||||||
|
// email: rec.email,
|
||||||
|
// password: hashedPassword,
|
||||||
|
// created_by: req.user.id
|
||||||
|
// });
|
||||||
|
|
||||||
|
// const placeHolderData = {
|
||||||
|
// contact_name: rec.factory || rec.est,
|
||||||
|
// portal_url: process.env.FE_BASE_URL,
|
||||||
|
// username: rec.email,
|
||||||
|
// password: autoPassword,
|
||||||
|
// support_email: process.env.SUPPORT_EMAIL,
|
||||||
|
// support_phone: process.env.SUPPORT_PHONE
|
||||||
|
// };
|
||||||
|
|
||||||
|
// await sendEmailService(rec.email, "establishment_user_creation_to_user", placeHolderData);
|
||||||
|
|
||||||
|
// await EstablishmentProduct.bulkCreate(
|
||||||
|
// rec.productIds.map(pid => ({
|
||||||
|
// establishment_id: est.id,
|
||||||
|
// product_id: pid,
|
||||||
|
// created_by: req.user.id
|
||||||
|
// }))
|
||||||
|
// );
|
||||||
|
// }
|
||||||
|
|
||||||
|
// fs.unlinkSync(filePath);
|
||||||
|
|
||||||
|
// return res.status(200).send({
|
||||||
|
// status: "success",
|
||||||
|
// message: `${prepared.length} establishments inserted successfully.`,
|
||||||
|
// summary: {
|
||||||
|
// total_records: rows.length,
|
||||||
|
// imported: prepared.length,
|
||||||
|
// errors: []
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
|
||||||
|
// } catch (err) {
|
||||||
|
// logger.error("Establishment Import Fatal Error: " + err.message);
|
||||||
|
|
||||||
|
// return res.status(500).send({
|
||||||
|
// status: "failed",
|
||||||
|
// message: "Invalid data. Please check the instructions given and upload again."
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
// };
|
||||||
|
|
||||||
// exports.establishmentBulkUpload = async (req, res) => {
|
// exports.establishmentBulkUpload = async (req, res) => {
|
||||||
// try {
|
// try {
|
||||||
// if (!req.file) {
|
// if (!req.file) {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user