diff --git a/app/Controllers/EndorsementController.php b/app/Controllers/EndorsementController.php index 2416522..6765577 100644 --- a/app/Controllers/EndorsementController.php +++ b/app/Controllers/EndorsementController.php @@ -610,15 +610,35 @@ class EndorsementController extends ResourceController try { $id = $this->request->getGet('id'); $type = $this->request->getGet('type'); // 'completion' or 'original' + $fileType = (!$type || $type === 'original') ? 'original' : $type; + + $storageDriver = config('Storage')->driver ?? 'local'; + $storageBucket = config('Storage')->s3Bucket ?? ''; + $s3LogCtx = [ + 'endorsement_id' => (int) $id, + 'driver' => $storageDriver, + 'bucket' => $storageBucket, + 'action' => 'download', + 'request_type' => $type, + 'file_type' => $fileType, + ]; + + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][START] downloadEndorsementCompletionFile started | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); if (!$id) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] endorsement id missing | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 400, 'data' => 'ID is required'], 200); } // Fetch record from DB $fileRecord = $this->EndorsementModel->find((int)$id); + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][RESOLVE] endorsement record resolved | ' . json_encode(array_merge($s3LogCtx, [ + 'record_found' => $fileRecord !== null, + ]), JSON_UNESCAPED_SLASHES)); + if (!$fileRecord) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] endorsement record not found in DB | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 404, 'data' => 'File not found'], 200); } @@ -632,19 +652,64 @@ class EndorsementController extends ResourceController } else if ($type === 'completion') { $fileName = $fileRecord['endorsement_completion_file'] ?? null; $subPath = 'endorsement_pdf/'; + } else { + $fileName = null; + $subPath = ''; } + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][RESOLVE] endorsement file name resolved | ' . json_encode(array_merge($s3LogCtx, [ + 'file_name' => $fileName, + 'sub_folder' => endorsement_subfolder_for_type($fileType), + 'original_file_name' => $fileRecord['endorsement_file_name'] ?? null, + 'completion_file_name' => $fileRecord['endorsement_completion_file'] ?? null, + ]), JSON_UNESCAPED_SLASHES)); + if (!$fileName) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] no file uploaded yet for requested type | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 404, 'data' => 'No file uploaded yet'], 200); } - if (!endorsement_exists_by_type($fileName, $type ?: 'original')) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][EXISTS] checking endorsement file in storage | ' . json_encode(array_merge($s3LogCtx, [ + 'module' => 'endorsement', + 'sub_folder' => endorsement_subfolder_for_type($fileType), + 'file_name' => $fileName, + ]), JSON_UNESCAPED_SLASHES)); + + $exists = endorsement_exists_by_type($fileName, $fileType); + + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][EXISTS] endorsement file exists check complete | ' . json_encode(array_merge($s3LogCtx, [ + 'file_name' => $fileName, + 'exists' => $exists, + ]), JSON_UNESCAPED_SLASHES)); + + if (!$exists) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] endorsement file missing in storage | ' . json_encode(array_merge($s3LogCtx, [ + 'file_name' => $fileName, + ]), JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 404, 'data' => 'File missing on server'], 200); } - return endorsement_download_by_type($fileName, $type ?: 'original'); + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][DOWNLOAD] streaming endorsement file from storage | ' . json_encode(array_merge($s3LogCtx, [ + 'file_name' => $fileName, + 'inline' => false, + 'sub_folder' => endorsement_subfolder_for_type($fileType), + ]), JSON_UNESCAPED_SLASHES)); + + $response = endorsement_download_by_type($fileName, $fileType); + + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][END] endorsement download response ready | ' . json_encode(array_merge($s3LogCtx, [ + 'file_name' => $fileName, + ]), JSON_UNESCAPED_SLASHES)); + + return $response; } catch (\Exception $e) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] downloadEndorsementCompletionFile fatal error | ' . json_encode([ + 'endorsement_id' => (int) ($this->request->getGet('id') ?? 0), + 'error' => $e->getMessage(), + 'file' => $e->getFile(), + 'line' => $e->getLine(), + ], JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 500, 'message' => $e->getMessage()], 500); } } @@ -694,15 +759,32 @@ class EndorsementController extends ResourceController { try { $endorsementId = $this->request->getGet('endorsement_id'); + + $storageDriver = config('Storage')->driver ?? 'local'; + $storageBucket = config('Storage')->s3Bucket ?? ''; + $s3LogCtx = [ + 'endorsement_id' => (int) $endorsementId, + 'driver' => $storageDriver, + 'bucket' => $storageBucket, + 'action' => 'inline_view', + ]; + + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][START] EndorsementFilePath started | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); if (!$endorsementId) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] endorsement_id missing | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 400, 'data' => 'endorsement_id is required'], 200); } // Fetch record from DB $fileRecord = $this->EndorsementModel->where('is_active', 1)->find((int)$endorsementId); + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][RESOLVE] endorsement record resolved | ' . json_encode(array_merge($s3LogCtx, [ + 'record_found' => $fileRecord !== null, + ]), JSON_UNESCAPED_SLASHES)); + if (!$fileRecord) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] endorsement record not found in DB | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 404, 'data' => 'File not found in database'], 200); } @@ -717,16 +799,57 @@ class EndorsementController extends ResourceController $fileName = $originalFile; $fileType = 'original'; } else { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] no original or completion file in DB | ' . json_encode(array_merge($s3LogCtx, [ + 'original_file_name' => $originalFile, + 'completion_file_name' => $completionFile, + ]), JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 404, 'data' => 'No file uploaded yet'], 200); } - if (!endorsement_exists_by_type($fileName, $fileType)) { + $s3LogCtx['file_type'] = $fileType; + $s3LogCtx['file_name'] = $fileName; + + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][RESOLVE] endorsement file name resolved | ' . json_encode(array_merge($s3LogCtx, [ + 'sub_folder' => endorsement_subfolder_for_type($fileType), + 'original_file_name' => $originalFile, + 'completion_file_name' => $completionFile, + 'selected_source' => $fileType, + ]), JSON_UNESCAPED_SLASHES)); + + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][EXISTS] checking endorsement file in storage | ' . json_encode(array_merge($s3LogCtx, [ + 'module' => 'endorsement', + 'sub_folder' => endorsement_subfolder_for_type($fileType), + ]), JSON_UNESCAPED_SLASHES)); + + $exists = endorsement_exists_by_type($fileName, $fileType); + + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][EXISTS] endorsement file exists check complete | ' . json_encode(array_merge($s3LogCtx, [ + 'exists' => $exists, + ]), JSON_UNESCAPED_SLASHES)); + + if (!$exists) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] endorsement file missing in storage | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 404, 'data' => 'File missing on server'], 200); } - return endorsement_download_by_type($fileName, $fileType, true); + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][DOWNLOAD] streaming endorsement file inline from storage | ' . json_encode(array_merge($s3LogCtx, [ + 'inline' => true, + 'sub_folder' => endorsement_subfolder_for_type($fileType), + ]), JSON_UNESCAPED_SLASHES)); + + $response = endorsement_download_by_type($fileName, $fileType, true); + + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][END] endorsement inline view response ready | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); + + return $response; } catch (\Exception $e) { + log_message('error', '[S3_FILE_GET][ENDORSEMENT_READ][FAILED] EndorsementFilePath fatal error | ' . json_encode([ + 'endorsement_id' => (int) ($this->request->getGet('endorsement_id') ?? 0), + 'error' => $e->getMessage(), + 'file' => $e->getFile(), + 'line' => $e->getLine(), + ], JSON_UNESCAPED_SLASHES)); return $this->respond(['status' => 'failed', 'code' => 500, 'message' => $e->getMessage()], 500); } } diff --git a/app/Controllers/JobWorker.php b/app/Controllers/JobWorker.php index 09c8857..04d1520 100755 --- a/app/Controllers/JobWorker.php +++ b/app/Controllers/JobWorker.php @@ -18,7 +18,7 @@ class JobWorker extends BaseController [ 'readFileAndCalculateCommission' => [ 'type' => 'CC', // Handler Category (Possible values: HC, CC, HF) - 'handler' => 'App\Controllers\PolicyRagController', + 'handler' => 'App\Controllers\PolicyController', ], ]; diff --git a/app/Controllers/PolicyController.php b/app/Controllers/PolicyController.php index dc8d69e..1d952f1 100644 --- a/app/Controllers/PolicyController.php +++ b/app/Controllers/PolicyController.php @@ -742,6 +742,16 @@ class PolicyController extends ResourceController { helper('policy_pdf_helper'); + $storageDriver = config('Storage')->driver ?? 'local'; + $storageBucket = config('Storage')->s3Bucket ?? ''; + $s3LogCtx = [ + 'policy_id' => (int) $policyId, + 'driver' => $storageDriver, + 'bucket' => $storageBucket, + ]; + + log_message('error', '[S3_FILE_GET][POLICY_READ][START] checkPolicyDoc file fetch started | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); + // Replace with your actual Gemini API key $apiKey = getenv('GEMINI_API_KEY'); // The model to use and the API endpoint @@ -754,9 +764,40 @@ class PolicyController extends ResourceController $pdfFileName = $record['policy_pdf_file_name'] ?? ''; $mdFileName = policy_md_file_name($pdfFileName); - if ($pdfFileName === '' || !policy_exists_by_type($pdfFileName, 'policy_pdf')) { + log_message('error', '[S3_FILE_GET][POLICY_READ][RESOLVE] policy record resolved | ' . json_encode(array_merge($s3LogCtx, [ + 'pdf_file_name' => $pdfFileName, + 'md_file_name' => $mdFileName, + 'record_found' => $record !== null, + ]), JSON_UNESCAPED_SLASHES)); + + if ($pdfFileName === '') { + log_message('error', '[S3_FILE_GET][POLICY_READ][FAILED] policy_pdf_file_name empty in DB | ' . json_encode($s3LogCtx, JSON_UNESCAPED_SLASHES)); + if ($return == true) { + return $this->respond(['status' => 'failed', 'message' => 'File not found: policy_pdf_file_name is empty'], 200); + } + + return ['status' => 'failed', 'message' => 'File not found: policy_pdf_file_name is empty']; + } + + log_message('error', '[S3_FILE_GET][POLICY_READ][EXISTS] checking policy PDF in storage | ' . json_encode(array_merge($s3LogCtx, [ + 'module' => 'policy', + 'sub_folder' => 'policy_pdf', + 'pdf_file_name'=> $pdfFileName, + ]), JSON_UNESCAPED_SLASHES)); + + $pdfExists = policy_exists_by_type($pdfFileName, 'policy_pdf'); + + log_message('error', '[S3_FILE_GET][POLICY_READ][EXISTS] policy PDF exists check complete | ' . json_encode(array_merge($s3LogCtx, [ + 'pdf_file_name' => $pdfFileName, + 'exists' => $pdfExists, + ]), JSON_UNESCAPED_SLASHES)); + + if (!$pdfExists) { $missingPath = policy_local_pdf_path($pdfFileName); - log_message('info',"Error: File not found at {$missingPath}"); + log_message('error', '[S3_FILE_GET][POLICY_READ][FAILED] policy PDF not found in storage | ' . json_encode(array_merge($s3LogCtx, [ + 'pdf_file_name' => $pdfFileName, + 'resolved_path' => $missingPath, + ]), JSON_UNESCAPED_SLASHES)); if($return == true) { return $this->respond(['status'=>"failed", 'message'=> "File not found at {$missingPath}"], 200); @@ -765,11 +806,34 @@ class PolicyController extends ResourceController } } + log_message('error', '[S3_FILE_GET][POLICY_READ][DOWNLOAD] resolving local processing path (S3 downloadToTemp if driver=s3) | ' . json_encode(array_merge($s3LogCtx, [ + 'module' => 'policy', + 'sub_folder' => 'policy_pdf', + 'pdf_file_name' => $pdfFileName, + ]), JSON_UNESCAPED_SLASHES)); + $pdfFilePath = policy_local_pdf_path($pdfFileName); + log_message('error', '[S3_FILE_GET][POLICY_READ][DOWNLOAD] local processing path ready | ' . json_encode(array_merge($s3LogCtx, [ + 'pdf_file_name' => $pdfFileName, + 'local_path' => $pdfFilePath, + 'is_file' => is_file($pdfFilePath), + 'file_size' => is_file($pdfFilePath) ? filesize($pdfFilePath) : null, + ]), JSON_UNESCAPED_SLASHES)); + + log_message('error', '[S3_FILE_GET][POLICY_READ][CONVERT] PDF to markdown started | ' . json_encode(array_merge($s3LogCtx, [ + 'pdf_file_name' => $pdfFileName, + 'md_file_name' => $mdFileName, + 'pdf_local_path'=> $pdfFilePath, + ]), JSON_UNESCAPED_SLASHES)); + $markdownResult = convert_policy_pdf_to_markdown($pdfFilePath); if ($markdownResult['status'] !== 'success') { - log_message('info', 'Policy PDF to markdown conversion failed: ' . $markdownResult['message']); + log_message('error', '[S3_FILE_GET][POLICY_READ][CONVERT][FAILED] PDF to markdown conversion failed | ' . json_encode(array_merge($s3LogCtx, [ + 'pdf_file_name' => $pdfFileName, + 'md_file_name' => $mdFileName, + 'message' => $markdownResult['message'] ?? null, + ]), JSON_UNESCAPED_SLASHES)); if ($return == true) { return $this->respond(['status' => 'failed', 'message' => $markdownResult['message']], 200); } @@ -777,8 +841,30 @@ class PolicyController extends ResourceController return ['status' => 'failed', 'message' => $markdownResult['message']]; } - if (!policy_md_exists($mdFileName)) { - log_message('info',"Error: Markdown file not found for {$mdFileName}"); + log_message('error', '[S3_FILE_GET][POLICY_READ][CONVERT] PDF to markdown success | ' . json_encode(array_merge($s3LogCtx, [ + 'pdf_file_name' => $pdfFileName, + 'md_file_name' => $markdownResult['md_file_name'] ?? $mdFileName, + 'message' => $markdownResult['message'] ?? null, + 'content_bytes' => isset($markdownResult['content']) ? strlen($markdownResult['content']) : null, + ]), JSON_UNESCAPED_SLASHES)); + + log_message('error', '[S3_FILE_GET][POLICY_READ][EXISTS] checking markdown in storage | ' . json_encode(array_merge($s3LogCtx, [ + 'module' => 'policy', + 'sub_folder' => 'policy_md', + 'md_file_name' => $mdFileName, + ]), JSON_UNESCAPED_SLASHES)); + + $mdExists = policy_md_exists($mdFileName); + + log_message('error', '[S3_FILE_GET][POLICY_READ][EXISTS] markdown exists check complete | ' . json_encode(array_merge($s3LogCtx, [ + 'md_file_name' => $mdFileName, + 'exists' => $mdExists, + ]), JSON_UNESCAPED_SLASHES)); + + if (!$mdExists) { + log_message('error', '[S3_FILE_GET][POLICY_READ][FAILED] markdown file not found in storage | ' . json_encode(array_merge($s3LogCtx, [ + 'md_file_name' => $mdFileName, + ]), JSON_UNESCAPED_SLASHES)); if($return == true) { return $this->respond(['status'=>"failed", 'message'=> "Markdown file not found for {$mdFileName}"], 200); @@ -787,7 +873,29 @@ class PolicyController extends ResourceController } } - $fileContent = $markdownResult['content'] ?? policy_read_md($mdFileName); + if (!empty($markdownResult['content'])) { + $fileContent = $markdownResult['content']; + log_message('error', '[S3_FILE_GET][POLICY_READ][CONTENT] content taken from convert result (no extra storage read) | ' . json_encode(array_merge($s3LogCtx, [ + 'md_file_name' => $mdFileName, + 'content_bytes' => strlen($fileContent), + 'source' => 'convert_result', + ]), JSON_UNESCAPED_SLASHES)); + } else { + log_message('error', '[S3_FILE_GET][POLICY_READ][READ][START] reading markdown from storage | ' . json_encode(array_merge($s3LogCtx, [ + 'module' => 'policy', + 'sub_folder' => 'policy_md', + 'md_file_name' => $mdFileName, + ]), JSON_UNESCAPED_SLASHES)); + + $fileContent = policy_read_md($mdFileName); + + log_message('error', '[S3_FILE_GET][POLICY_READ][READ][END] markdown read from storage complete | ' . json_encode(array_merge($s3LogCtx, [ + 'md_file_name' => $mdFileName, + 'content_bytes' => strlen($fileContent), + 'source' => 'storage_read', + ]), JSON_UNESCAPED_SLASHES)); + } + $mimeType = 'text/markdown'; // Define supported inline MIME types @@ -806,7 +914,9 @@ class PolicyController extends ResourceController // Conditionally handle the file upload based on MIME type if (in_array($mimeType, $supportedInlineMimeTypes)) { if ($fileContent === '' || trim($fileContent) === '') { - log_message('info', "Markdown file is empty for {$mdFileName}"); + log_message('error', '[S3_FILE_GET][POLICY_READ][FAILED] markdown content empty | ' . json_encode(array_merge($s3LogCtx, [ + 'md_file_name' => $mdFileName, + ]), JSON_UNESCAPED_SLASHES)); if ($return == true) { return $this->respond(['status' => 'failed', 'message' => "Markdown file is empty for {$mdFileName}"], 200); } @@ -815,6 +925,13 @@ class PolicyController extends ResourceController } $base64Content = base64_encode($fileContent); + log_message('error', '[S3_FILE_GET][POLICY_READ] markdown prepared for Gemini payload | ' . json_encode(array_merge($s3LogCtx, [ + 'md_file_name' => $mdFileName, + 'mime_type' => $mimeType, + 'content_bytes' => strlen($fileContent), + 'base64_bytes' => strlen($base64Content), + ]), JSON_UNESCAPED_SLASHES)); + $payloadPart = [ "inlineData" => [ "mimeType" => $mimeType, @@ -828,8 +945,16 @@ class PolicyController extends ResourceController $payloadPart ] ]; + + log_message('error', '[S3_FILE_GET][POLICY_READ][END] storage file get/read flow complete, Gemini request starting | ' . json_encode(array_merge($s3LogCtx, [ + 'pdf_file_name' => $pdfFileName, + 'md_file_name' => $mdFileName, + ]), JSON_UNESCAPED_SLASHES)); } else { - log_message('info', "Unsupported inline MIME type ({$mimeType}) for markdown file."); + log_message('error', '[S3_FILE_GET][POLICY_READ][FAILED] unsupported mime type | ' . json_encode(array_merge($s3LogCtx, [ + 'md_file_name' => $mdFileName, + 'mime_type' => $mimeType, + ]), JSON_UNESCAPED_SLASHES)); if($return == true) { return $this->respond(['status'=>"failed", 'message'=> "Unsupported inline MIME type ({$mimeType})"], 200); @@ -863,7 +988,7 @@ class PolicyController extends ResourceController // Check for cURL errors if (curl_errno($ch)) { $curl_error = curl_error($ch); - log_message('info',"cURL Error: " . $curl_error); + log_message('error',"cURL Error: " . $curl_error); if($return == true) { return $this->respond(['status'=>"failed", 'message'=> "cURL Error: " . $curl_error], 200); @@ -878,7 +1003,7 @@ class PolicyController extends ResourceController $ch = null; // Mark handle as closed // Check for non-successful HTTP status codes if ($http_code < 200 || $http_code >= 300) { - log_message('info',"API returned non-successful HTTP status code: $http_code. Response: " . substr($response, 0, 200)); + log_message('error',"API returned non-successful HTTP status code: $http_code. Response: " . substr($response, 0, 200)); if($return == true) { return $this->respond(['status'=>"failed", 'message'=> "API returned non-successful HTTP status code: $http_code. Response: " . substr($response, 0, 200)], 200); @@ -893,7 +1018,7 @@ class PolicyController extends ResourceController if ($text_path) { $generatedText = $text_path; - log_message('info', "Successfully received generated text."); + log_message('error', "Successfully received generated text."); // 9. Attempt to extract and decode the exact JSON from the generated text $json_string = ''; @@ -912,7 +1037,7 @@ class PolicyController extends ResourceController } if (empty($json_string)) { - log_message('info',"Could not extract a valid JSON string from the generated text."); + log_message('error',"Could not extract a valid JSON string from the generated text."); if($return == true) { return $this->respond(['status'=>"failed", 'message'=> "Could not extract a valid JSON string from the generated text."], 200); @@ -924,7 +1049,7 @@ class PolicyController extends ResourceController // Decode the extracted JSON string $final_data = json_decode($json_string, true, 512, JSON_THROW_ON_ERROR); - log_message('info', "Extracted and decoded final JSON data successfully."); + log_message('error', "Extracted and decoded final JSON data successfully."); if($return == true) { @@ -935,7 +1060,7 @@ class PolicyController extends ResourceController } else { - log_message('info',"Response structure invalid or no generated text candidate found."); + log_message('error',"Response structure invalid or no generated text candidate found."); if($return == true) { return $this->respond(['status'=>"failed", 'message'=> "Response structure invalid or no generated text candidate found"], 200); @@ -949,7 +1074,12 @@ class PolicyController extends ResourceController // if ($ch !== null) { // curl_close($ch); // } - log_message('error', "Fatal Error: " . $e->getMessage()); + log_message('error', '[S3_FILE_GET][POLICY_READ][FAILED] checkPolicyDoc fatal error | ' . json_encode([ + 'policy_id' => (int) $policyId, + 'error' => $e->getMessage(), + 'file' => $e->getFile(), + 'line' => $e->getLine(), + ], JSON_UNESCAPED_SLASHES)); if($return == true) { return $this->respond(['status'=>"failed", 'message'=> "Error: " . $e->getMessage()], 200);