Nhances/nhance_partner_be
Nhances/nhance_partner_be
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GWM : rag file read log message updated

Browse Source
This commit is contained in:
Gowtham M 2026-07-01 10:48:35 +05:30
parent 6a13032714
commit 5d6b9058f9
1 changed files with 218 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

249
app/Controllers/PolicyRagController.php
View File

@ -66,7 +66,7 @@ class PolicyRagController extends ResourceController
{ {
$policyId = $data['policy_id']; $policyId = $data['policy_id'];
log_message('debug', 'PolicyRag readFileAndCalculateCommission started for policy_id: ' . $policyId); $this->logRagStep('JOB_START', 'started', 'readFileAndCalculateCommission', ['policy_id' => $policyId]);
$readDoc = $this->readPolicyDocViaRag($policyId); $readDoc = $this->readPolicyDocViaRag($policyId);
$calculateCommission = ['status' => 'failed']; $calculateCommission = ['status' => 'failed'];
@ -75,21 +75,43 @@ class PolicyRagController extends ResourceController
$extracted = $readDoc['data']; $extracted = $readDoc['data'];
$policyData = $this->mapExtractedDataToPolicyFields($extracted); $policyData = $this->mapExtractedDataToPolicyFields($extracted);
log_message('debug', 'Policy RAG Update Payload: ' . json_encode($policyData)); $this->logRagStep('DB_UPDATE', 'started', 'Updating policy from RAG data', [
'policy_id' => $policyId,
'payload' => $policyData,
]);
$updateStatus = $this->PolicyModel->update($policyId, $policyData); $updateStatus = $this->PolicyModel->update($policyId, $policyData);
if ($updateStatus) { if ($updateStatus) {
log_message('debug', 'Policy updated successfully via RAG for ID: ' . $policyId); $this->logRagStep('DB_UPDATE', 'success', 'Policy updated from RAG data', ['policy_id' => $policyId]);
$this->logRagStep('COMMISSION', 'started', 'Calculating commission', ['policy_id' => $policyId]);
$policyController = new PolicyController(); $policyController = new PolicyController();
$calculateCommission = $policyController->calculateCommission($policyId); $calculateCommission = $policyController->calculateCommission($policyId);
$commissionStatus = $calculateCommission['status'] ?? 'failed';
$this->logRagStep(
'COMMISSION',
$commissionStatus === 'success' ? 'success' : 'failed',
'Commission calculation finished',
['policy_id' => $policyId, 'result' => $commissionStatus]
);
} else { } else {
log_message('error', 'Failed to update policy via RAG for ID: ' . $policyId); $this->logRagStep('DB_UPDATE', 'failed', 'Failed to update policy from RAG data', ['policy_id' => $policyId]);
} }
} else { } else {
log_message('error', 'readPolicyDocViaRag returned failure: ' . json_encode($readDoc)); $this->logRagStep('JOB_READ', 'failed', 'readPolicyDocViaRag failed', [
'policy_id' => $policyId,
'response' => $readDoc,
]);
} }
$this->logRagStep('JOB_END', 'completed', 'readFileAndCalculateCommission finished', [
'policy_id' => $policyId,
'readPolicyDocViaRag' => $readDoc['status'],
'calculateCommission' => $calculateCommission['status'] ?? 'failed',
]);
return [ return [
'readPolicyDocViaRag' => $readDoc['status'], 'readPolicyDocViaRag' => $readDoc['status'],
'calculateCommission' => $calculateCommission['status'] ?? 'failed', 'calculateCommission' => $calculateCommission['status'] ?? 'failed',
@ -130,67 +152,126 @@ class PolicyRagController extends ResourceController
{ {
$fileId = null; $fileId = null;
$this->logRagStep('READ_START', 'started', 'readPolicyDocViaRag', ['policy_id' => $policyId]);
try { try {
$record = $this->PolicyModel->find((int) $policyId); $record = $this->PolicyModel->find((int) $policyId);
if (!$record) { if (!$record) {
$this->logRagStep('POLICY_LOOKUP', 'failed', 'Policy not found', ['policy_id' => $policyId]);
return $this->formatReadResponse('failed', 'Policy not found'); return $this->formatReadResponse('failed', 'Policy not found');
} }
$this->logRagStep('POLICY_LOOKUP', 'success', 'Policy found', [
'policy_id' => $policyId,
'pdf_file' => $record['policy_pdf_file_name'] ?? null,
]);
$uploadedPath = WRITEPATH . 'uploads/policy/policy_pdf/'; $uploadedPath = WRITEPATH . 'uploads/policy/policy_pdf/';
$pdfFilePath = $uploadedPath . $record['policy_pdf_file_name']; $pdfFilePath = $uploadedPath . $record['policy_pdf_file_name'];
if (!file_exists($pdfFilePath)) { if (!file_exists($pdfFilePath)) {
log_message('info', "RAG read: PDF not found at {$pdfFilePath}"); $this->logRagStep('PDF_CHECK', 'failed', 'PDF file not found', ['path' => $pdfFilePath]);
return $this->formatReadResponse('failed', "File not found at {$pdfFilePath}"); return $this->formatReadResponse('failed', "File not found at {$pdfFilePath}");
} }
$uploadResult = $this->uploadFileToRag($pdfFilePath); $this->logRagStep('PDF_CHECK', 'success', 'PDF file found', ['path' => $pdfFilePath]);
$this->logRagStep('UPLOAD', 'started', 'Uploading PDF to RAG API', ['path' => $pdfFilePath]);
$uploadResult = $this->uploadFileToRag($pdfFilePath, (int) $policyId);
if ($uploadResult['status'] !== 'success') { if ($uploadResult['status'] !== 'success') {
return $this->formatReadResponse('failed', $uploadResult['message']); return $this->formatReadResponse('failed', $uploadResult['message']);
} }
$fileId = $uploadResult['data']['file_id'] ?? null; $fileId = $uploadResult['data']['file_id'] ?? null;
if (empty($fileId)) { if (empty($fileId)) {
$this->logRagStep('UPLOAD', 'failed', 'RAG upload did not return file_id', ['policy_id' => $policyId]);
return $this->formatReadResponse('failed', 'RAG upload did not return file_id'); return $this->formatReadResponse('failed', 'RAG upload did not return file_id');
} }
$this->logRagStep('SAVE_RAG_FILE_ID', 'started', 'Saving rag_file_id', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
$updated = $this->PolicyModel->update((int) $policyId, ['rag_file_id' => $fileId]); $updated = $this->PolicyModel->update((int) $policyId, ['rag_file_id' => $fileId]);
if ($updated === false) { if ($updated === false) {
log_message('error', "Failed to save rag_file_id={$fileId} for policy_id={$policyId}"); $this->logRagStep('SAVE_RAG_FILE_ID', 'failed', 'Failed to save rag_file_id', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
} else {
$this->logRagStep('SAVE_RAG_FILE_ID', 'success', 'rag_file_id saved', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
} }
log_message('info', "RAG file uploaded. file_id={$fileId}, policy_id={$policyId}");
$indexResult = $this->waitForFileIndexed($fileId); $this->logRagStep('INDEX_POLL', 'started', 'Waiting for RAG file to be indexed', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
$indexResult = $this->waitForFileIndexed($fileId, (int) $policyId);
if ($indexResult['status'] !== 'success') { if ($indexResult['status'] !== 'success') {
return $this->formatReadResponse('failed', $indexResult['message']); return $this->formatReadResponse('failed', $indexResult['message']);
} }
$chatResult = $this->chatWithRag($fileId, $this->getPolicyReadPrompt()); $this->logRagStep('CHAT', 'started', 'Sending extraction prompt to RAG API', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
$chatResult = $this->chatWithRag($fileId, $this->getPolicyReadPrompt(), (int) $policyId);
if ($chatResult['status'] !== 'success') { if ($chatResult['status'] !== 'success') {
return $this->formatReadResponse('failed', $chatResult['message']); return $this->formatReadResponse('failed', $chatResult['message']);
} }
$answer = $chatResult['data']['answer'] ?? ''; $answer = $chatResult['data']['answer'] ?? '';
// dd($answer); $finalData = $this->extractJsonFromText($answer, (int) $policyId);
$finalData = $this->extractJsonFromText($answer);
if ($finalData === null) { if ($finalData === null) {
$this->logRagStep('JSON_EXTRACT', 'failed', 'Could not extract valid JSON from RAG answer', [
'policy_id' => $policyId,
'file_id' => $fileId,
'answer_snippet' => substr($answer, 0, 500),
]);
return $this->formatReadResponse('failed', 'Could not extract valid JSON from RAG answer'); return $this->formatReadResponse('failed', 'Could not extract valid JSON from RAG answer');
} }
log_message('info', 'RAG policy doc read successful for policy_id=' . $policyId); $this->logRagStep('READ_COMPLETE', 'success', 'Policy document read successfully', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
return $this->formatReadResponse('success', 'Doc read success', $finalData); return $this->formatReadResponse('success', 'Doc read success', $finalData);
} catch (\Throwable $e) { } catch (\Throwable $e) {
log_message('error', 'PolicyRag readPolicyDocViaRag error: ' . $e->getMessage()); $this->logRagStep('READ_ERROR', 'failed', $e->getMessage(), [
'policy_id' => $policyId,
'file_id' => $fileId,
'line' => $e->getLine(),
]);
return $this->formatReadResponse('failed', 'Error: ' . $e->getMessage()); return $this->formatReadResponse('failed', 'Error: ' . $e->getMessage());
} finally { } finally {
if (!empty($fileId)) { if (!empty($fileId)) {
$deleteResult = $this->deleteRagFile($fileId); $this->logRagStep('DELETE', 'started', 'Deleting RAG file', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
$deleteResult = $this->deleteRagFile($fileId, (int) $policyId);
if ($deleteResult['status'] !== 'success') { if ($deleteResult['status'] !== 'success') {
log_message('error', 'RAG file delete failed for file_id=' . $fileId . ': ' . ($deleteResult['message'] ?? '')); $this->logRagStep('DELETE', 'failed', $deleteResult['message'] ?? 'RAG file delete failed', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
} }
} }
$this->logRagStep('READ_END', 'completed', 'readPolicyDocViaRag finished', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
} }
} }
@ -344,7 +425,7 @@ class PolicyRagController extends ResourceController
return $baseUrl; return $baseUrl;
} }
private function uploadFileToRag(string $pdfFilePath): array private function uploadFileToRag(string $pdfFilePath, ?int $policyId = null): array
{ {
$url = $this->ragApiBaseUrl() . '/api/files/upload'; $url = $this->ragApiBaseUrl() . '/api/files/upload';
$cfile = new \CURLFile($pdfFilePath, 'application/pdf', basename($pdfFilePath)); $cfile = new \CURLFile($pdfFilePath, 'application/pdf', basename($pdfFilePath));
@ -363,24 +444,41 @@ class PolicyRagController extends ResourceController
curl_close($ch); curl_close($ch);
if ($curlError) { if ($curlError) {
log_message('error', 'RAG upload cURL error: ' . $curlError); $this->logRagStep('UPLOAD', 'failed', 'cURL error: ' . $curlError, [
'policy_id' => $policyId,
'url' => $url,
]);
return ['status' => 'failed', 'message' => 'RAG upload cURL error: ' . $curlError]; return ['status' => 'failed', 'message' => 'RAG upload cURL error: ' . $curlError];
} }
if ($httpCode < 200 || $httpCode >= 300) { if ($httpCode < 200 || $httpCode >= 300) {
log_message('error', "RAG upload HTTP {$httpCode}: " . substr((string) $response, 0, 500)); $this->logRagStep('UPLOAD', 'failed', "HTTP {$httpCode}", [
'policy_id' => $policyId,
'response' => substr((string) $response, 0, 500),
]);
return ['status' => 'failed', 'message' => "RAG upload failed with HTTP {$httpCode}"]; return ['status' => 'failed', 'message' => "RAG upload failed with HTTP {$httpCode}"];
} }
$responseData = json_decode((string) $response, true); $responseData = json_decode((string) $response, true);
if (!is_array($responseData) || empty($responseData['success'])) { if (!is_array($responseData) || empty($responseData['success'])) {
$this->logRagStep('UPLOAD', 'failed', 'Invalid API response', [
'policy_id' => $policyId,
'response' => substr((string) $response, 0, 500),
]);
return ['status' => 'failed', 'message' => 'RAG upload returned invalid response']; return ['status' => 'failed', 'message' => 'RAG upload returned invalid response'];
} }
$this->logRagStep('UPLOAD', 'success', 'PDF uploaded to RAG API', [
'policy_id' => $policyId,
'file_id' => $responseData['file_id'] ?? null,
'filename' => $responseData['filename'] ?? null,
'status' => $responseData['status'] ?? null,
]);
return ['status' => 'success', 'data' => $responseData]; return ['status' => 'success', 'data' => $responseData];
} }
private function getFileStatus(string $fileId): array private function getFileStatus(string $fileId, ?int $policyId = null): array
{ {
$url = $this->ragApiBaseUrl() . '/api/files/status?file_id=' . urlencode($fileId); $url = $this->ragApiBaseUrl() . '/api/files/status?file_id=' . urlencode($fileId);
@ -393,40 +491,67 @@ class PolicyRagController extends ResourceController
curl_close($ch); curl_close($ch);
if ($curlError) { if ($curlError) {
$this->logRagStep('INDEX_POLL', 'failed', 'Status check cURL error: ' . $curlError, [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
return ['status' => 'failed', 'message' => 'RAG status cURL error: ' . $curlError]; return ['status' => 'failed', 'message' => 'RAG status cURL error: ' . $curlError];
} }
if ($httpCode < 200 || $httpCode >= 300) { if ($httpCode < 200 || $httpCode >= 300) {
$this->logRagStep('INDEX_POLL', 'failed', "Status check HTTP {$httpCode}", [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
return ['status' => 'failed', 'message' => "RAG status check failed with HTTP {$httpCode}"]; return ['status' => 'failed', 'message' => "RAG status check failed with HTTP {$httpCode}"];
} }
$responseData = json_decode((string) $response, true); $responseData = json_decode((string) $response, true);
if (!is_array($responseData) || empty($responseData['success'])) { if (!is_array($responseData) || empty($responseData['success'])) {
$this->logRagStep('INDEX_POLL', 'failed', 'Status check invalid response', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
return ['status' => 'failed', 'message' => 'RAG status returned invalid response']; return ['status' => 'failed', 'message' => 'RAG status returned invalid response'];
} }
return ['status' => 'success', 'data' => $responseData]; return ['status' => 'success', 'data' => $responseData];
} }
private function waitForFileIndexed(string $fileId): array private function waitForFileIndexed(string $fileId, ?int $policyId = null): array
{ {
$interval = (int) (getenv('RAG_API_STATUS_POLL_INTERVAL') ?: 10); $interval = (int) (getenv('RAG_API_STATUS_POLL_INTERVAL') ?: 10);
$maxAttempts = (int) (getenv('RAG_API_STATUS_MAX_ATTEMPTS') ?: 60); $maxAttempts = (int) (getenv('RAG_API_STATUS_MAX_ATTEMPTS') ?: 60);
for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) { for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) {
$statusResult = $this->getFileStatus($fileId); $statusResult = $this->getFileStatus($fileId, $policyId);
if ($statusResult['status'] !== 'success') { if ($statusResult['status'] !== 'success') {
return $statusResult; return $statusResult;
} }
$fileStatus = strtolower((string) ($statusResult['data']['status'] ?? '')); $fileStatus = strtolower((string) ($statusResult['data']['status'] ?? ''));
log_message('info', "RAG file status poll attempt {$attempt}/{$maxAttempts}: file_id={$fileId}, status={$fileStatus}"); $this->logRagStep('INDEX_POLL', 'progress', "Poll attempt {$attempt}/{$maxAttempts}", [
'policy_id' => $policyId,
'file_id' => $fileId,
'status' => $fileStatus,
]);
if ($fileStatus === 'indexed') { if ($fileStatus === 'indexed') {
$this->logRagStep('INDEX_POLL', 'success', 'File indexed', [
'policy_id' => $policyId,
'file_id' => $fileId,
'chunk_count' => $statusResult['data']['chunk_count'] ?? null,
'attempts' => $attempt,
]);
return $statusResult; return $statusResult;
} }
if (in_array($fileStatus, ['failed', 'error'], true)) { if (in_array($fileStatus, ['failed', 'error'], true)) {
$this->logRagStep('INDEX_POLL', 'failed', 'Indexing failed', [
'policy_id' => $policyId,
'file_id' => $fileId,
'status' => $fileStatus,
]);
return ['status' => 'failed', 'message' => 'RAG file indexing failed with status: ' . $fileStatus]; return ['status' => 'failed', 'message' => 'RAG file indexing failed with status: ' . $fileStatus];
} }
@ -435,10 +560,17 @@ class PolicyRagController extends ResourceController
} }
} }
$this->logRagStep('INDEX_POLL', 'failed', 'Indexing timed out', [
'policy_id' => $policyId,
'file_id' => $fileId,
'max_attempts' => $maxAttempts,
'interval_sec' => $interval,
]);
return ['status' => 'failed', 'message' => 'RAG file indexing timed out']; return ['status' => 'failed', 'message' => 'RAG file indexing timed out'];
} }
private function chatWithRag(string $fileId, string $question): array private function chatWithRag(string $fileId, string $question, ?int $policyId = null): array
{ {
$url = $this->ragApiBaseUrl() . '/api/chat'; $url = $this->ragApiBaseUrl() . '/api/chat';
@ -459,23 +591,41 @@ class PolicyRagController extends ResourceController
curl_close($ch); curl_close($ch);
if ($curlError) { if ($curlError) {
$this->logRagStep('CHAT', 'failed', 'cURL error: ' . $curlError, [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
return ['status' => 'failed', 'message' => 'RAG chat cURL error: ' . $curlError]; return ['status' => 'failed', 'message' => 'RAG chat cURL error: ' . $curlError];
} }
if ($httpCode < 200 || $httpCode >= 300) { if ($httpCode < 200 || $httpCode >= 300) {
log_message('error', "RAG chat HTTP {$httpCode}: " . substr((string) $response, 0, 500)); $this->logRagStep('CHAT', 'failed', "HTTP {$httpCode}", [
'policy_id' => $policyId,
'file_id' => $fileId,
'response' => substr((string) $response, 0, 500),
]);
return ['status' => 'failed', 'message' => "RAG chat failed with HTTP {$httpCode}"]; return ['status' => 'failed', 'message' => "RAG chat failed with HTTP {$httpCode}"];
} }
$responseData = json_decode((string) $response, true); $responseData = json_decode((string) $response, true);
if (!is_array($responseData) || empty($responseData['success'])) { if (!is_array($responseData) || empty($responseData['success'])) {
$this->logRagStep('CHAT', 'failed', 'Invalid API response', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
return ['status' => 'failed', 'message' => 'RAG chat returned invalid response']; return ['status' => 'failed', 'message' => 'RAG chat returned invalid response'];
} }
$this->logRagStep('CHAT', 'success', 'RAG chat response received', [
'policy_id' => $policyId,
'file_id' => $fileId,
'answer_length' => strlen((string) ($responseData['answer'] ?? '')),
]);
return ['status' => 'success', 'data' => $responseData]; return ['status' => 'success', 'data' => $responseData];
} }
private function deleteRagFile(string $fileId): array private function deleteRagFile(string $fileId, ?int $policyId = null): array
{ {
$url = $this->ragApiBaseUrl() . '/api/files/id/' . urlencode($fileId); $url = $this->ragApiBaseUrl() . '/api/files/id/' . urlencode($fileId);
@ -489,24 +639,40 @@ class PolicyRagController extends ResourceController
curl_close($ch); curl_close($ch);
if ($curlError) { if ($curlError) {
$this->logRagStep('DELETE', 'failed', 'cURL error: ' . $curlError, [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
return ['status' => 'failed', 'message' => 'RAG delete cURL error: ' . $curlError]; return ['status' => 'failed', 'message' => 'RAG delete cURL error: ' . $curlError];
} }
if ($httpCode < 200 || $httpCode >= 300) { if ($httpCode < 200 || $httpCode >= 300) {
log_message('error', "RAG delete HTTP {$httpCode}: " . substr((string) $response, 0, 500)); $this->logRagStep('DELETE', 'failed', "HTTP {$httpCode}", [
'policy_id' => $policyId,
'file_id' => $fileId,
'response' => substr((string) $response, 0, 500),
]);
return ['status' => 'failed', 'message' => "RAG delete failed with HTTP {$httpCode}"]; return ['status' => 'failed', 'message' => "RAG delete failed with HTTP {$httpCode}"];
} }
$responseData = json_decode((string) $response, true); $responseData = json_decode((string) $response, true);
if (!is_array($responseData) || empty($responseData['success'])) { if (!is_array($responseData) || empty($responseData['success'])) {
$this->logRagStep('DELETE', 'failed', 'Invalid API response', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
return ['status' => 'failed', 'message' => 'RAG delete returned invalid response']; return ['status' => 'failed', 'message' => 'RAG delete returned invalid response'];
} }
log_message('info', 'RAG file deleted: file_id=' . $fileId); $this->logRagStep('DELETE', 'success', 'RAG file deleted', [
'policy_id' => $policyId,
'file_id' => $fileId,
]);
return ['status' => 'success', 'data' => $responseData]; return ['status' => 'success', 'data' => $responseData];
} }
private function extractJsonFromText(string $text): ?array private function extractJsonFromText(string $text, ?int $policyId = null): ?array
{ {
$jsonString = ''; $jsonString = '';
@ -525,7 +691,28 @@ class PolicyRagController extends ResourceController
} }
$decoded = json_decode($jsonString, true); $decoded = json_decode($jsonString, true);
return is_array($decoded) ? $decoded : null; if (!is_array($decoded)) {
$this->logRagStep('JSON_EXTRACT', 'failed', 'JSON decode failed', [
'policy_id' => $policyId,
'json_snippet' => substr($jsonString, 0, 500),
]);
return null;
}
$this->logRagStep('JSON_EXTRACT', 'success', 'JSON extracted from RAG answer', [
'policy_id' => $policyId,
'field_keys' => array_keys($decoded),
]);
return $decoded;
}
private function logRagStep(string $step, string $status, string $message, array $context = []): void
{
$level = in_array($status, ['failed', 'error'], true) ? 'error' : 'info';
$contextJson = $context ? ' | ' . json_encode($context, JSON_UNESCAPED_SLASHES) : '';
log_message($level, "[PolicyRag][{$step}][{$status}] {$message}{$contextJson}");
} }
private function formatReadResponse(string $status, string $message, ?array $data = null): array private function formatReadResponse(string $status, string $message, ?array $data = null): array