Filter data added in kpi api : GWM
This commit is contained in:
parent
f8662176f8
commit
dfc8152b12
2312
kpi_files/divorces.json
Normal file
2312
kpi_files/divorces.json
Normal file
File diff suppressed because it is too large
Load Diff
100
kpi_files/total.json
Normal file
100
kpi_files/total.json
Normal file
@ -0,0 +1,100 @@
|
||||
[
|
||||
{
|
||||
"ObsKey": {
|
||||
"DV_TYPE": "D_TOT",
|
||||
"FREQ": "A",
|
||||
"MEASURE": "D",
|
||||
"REF_AREA": "AE",
|
||||
"SOURCE_DETAIL": "FCSAMD",
|
||||
"TIME_PERIOD": "2015",
|
||||
"UNIT_MEASURE": "NUMBER"
|
||||
},
|
||||
"ObsValue": {
|
||||
"Value": "4913"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ObsKey": {
|
||||
"DV_TYPE": "D_TOT",
|
||||
"FREQ": "A",
|
||||
"MEASURE": "D",
|
||||
"REF_AREA": "AE",
|
||||
"SOURCE_DETAIL": "FCSAMD",
|
||||
"TIME_PERIOD": "2016",
|
||||
"UNIT_MEASURE": "NUMBER"
|
||||
},
|
||||
"ObsValue": {
|
||||
"Value": "4599"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ObsKey": {
|
||||
"DV_TYPE": "D_TOT",
|
||||
"FREQ": "A",
|
||||
"MEASURE": "D",
|
||||
"REF_AREA": "AE",
|
||||
"SOURCE_DETAIL": "FCSAMD",
|
||||
"TIME_PERIOD": "2017",
|
||||
"UNIT_MEASURE": "NUMBER"
|
||||
},
|
||||
"ObsValue": {
|
||||
"Value": "4403"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ObsKey": {
|
||||
"DV_TYPE": "D_TOT",
|
||||
"FREQ": "A",
|
||||
"MEASURE": "D",
|
||||
"REF_AREA": "AE",
|
||||
"SOURCE_DETAIL": "FCSAMD",
|
||||
"TIME_PERIOD": "2018",
|
||||
"UNIT_MEASURE": "NUMBER"
|
||||
},
|
||||
"ObsValue": {
|
||||
"Value": "4506"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ObsKey": {
|
||||
"DV_TYPE": "D_TOT",
|
||||
"FREQ": "A",
|
||||
"MEASURE": "D",
|
||||
"REF_AREA": "AE",
|
||||
"SOURCE_DETAIL": "FCSAMD",
|
||||
"TIME_PERIOD": "2019",
|
||||
"UNIT_MEASURE": "NUMBER"
|
||||
},
|
||||
"ObsValue": {
|
||||
"Value": "4554"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ObsKey": {
|
||||
"DV_TYPE": "D_TOT",
|
||||
"FREQ": "A",
|
||||
"MEASURE": "D",
|
||||
"REF_AREA": "AE",
|
||||
"SOURCE_DETAIL": "FCSAMD",
|
||||
"TIME_PERIOD": "2020",
|
||||
"UNIT_MEASURE": "NUMBER"
|
||||
},
|
||||
"ObsValue": {
|
||||
"Value": "4213"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ObsKey": {
|
||||
"DV_TYPE": "D_TOT",
|
||||
"FREQ": "A",
|
||||
"MEASURE": "D",
|
||||
"REF_AREA": "AE",
|
||||
"SOURCE_DETAIL": "FCSAMD",
|
||||
"TIME_PERIOD": "2021",
|
||||
"UNIT_MEASURE": "NUMBER"
|
||||
},
|
||||
"ObsValue": {
|
||||
"Value": "5450"
|
||||
}
|
||||
}
|
||||
]
|
||||
238
main.go
238
main.go
@ -69,7 +69,7 @@ func removeNamespace(xmlBytes []byte) string {
|
||||
}
|
||||
|
||||
// Function to fetch XML from an API and convert it to JSON
|
||||
func fetchAndConvertXMLToJSON(apiURL string, kpiName string, saveToFile bool) ([]map[string]interface{}, error) {
|
||||
func fetchAndConvertXMLToJSON(apiURL string, kpiName string, chartId string) ([]map[string]interface{}, error) {
|
||||
// Call the API and get the XML response
|
||||
resp, err := http.Get(apiURL)
|
||||
if err != nil {
|
||||
@ -110,18 +110,27 @@ func fetchAndConvertXMLToJSON(apiURL string, kpiName string, saveToFile bool) ([
|
||||
}
|
||||
|
||||
// Save the JSON data to a file if required
|
||||
if saveToFile {
|
||||
// Ensure the folder exists
|
||||
outputDir := "kpi_files"
|
||||
if err := os.MkdirAll(outputDir, os.ModePerm); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Save the file with a timestamped filename
|
||||
fileName := fmt.Sprintf("%s/%s.json", outputDir, kpiName)
|
||||
if err := ioutil.WriteFile(fileName, jsonData, 0644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Ensure the folder exists
|
||||
outputDir := "kpi_files"
|
||||
if err := os.MkdirAll(outputDir, os.ModePerm); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Save the file with a timestamped filename
|
||||
fileName := fmt.Sprintf("%s/%s.json", outputDir, kpiName)
|
||||
if err := ioutil.WriteFile(fileName, jsonData, 0644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find the charts record
|
||||
record, err := app.Dao().FindRecordById("charts", chartId)
|
||||
if err != nil {
|
||||
// return c.JSON(500, map[string]interface{}{"code": 500,"message": "Error finding user.",})
|
||||
}
|
||||
record.Set("file_status", true)
|
||||
if err := app.Dao().SaveRecord(record); err != nil {
|
||||
// return c.JSON(500, map[string]interface{}{"code": 500,"message": "Failed to verify user.",})
|
||||
}
|
||||
|
||||
return response, nil
|
||||
@ -147,6 +156,61 @@ func readJSONFromFile(kpiName string) ([]map[string]interface{}, error) {
|
||||
return jsonResponse, nil
|
||||
}
|
||||
|
||||
type ResponseItem struct {
|
||||
ObsKey map[string]string `json:"ObsKey"`
|
||||
ObsValue map[string]string `json:"ObsValue"`
|
||||
}
|
||||
|
||||
type FilterGroup struct {
|
||||
FilterKey string `json:"filter_key"`
|
||||
FilterData []interface{} `json:"filter_data"`
|
||||
}
|
||||
|
||||
func generateFilters(responseRaw []map[string]interface{}, filterKeys []string) ([]FilterGroup, error) {
|
||||
// Transform responseRaw to []ResponseItem
|
||||
var response []ResponseItem
|
||||
responseBytes, err := json.Marshal(responseRaw) // Marshal raw data to JSON
|
||||
if err != nil {
|
||||
log.Printf("Failed to marshal raw response: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(responseBytes, &response); err != nil { // Unmarshal to []ResponseItem
|
||||
log.Printf("Failed to unmarshal response: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create a map to hold unique filter data
|
||||
filterMap := make(map[string]map[string]struct{})
|
||||
for _, key := range filterKeys {
|
||||
filterMap[key] = make(map[string]struct{})
|
||||
}
|
||||
|
||||
// Populate filter data dynamically
|
||||
for _, item := range response {
|
||||
for _, key := range filterKeys {
|
||||
if value, exists := item.ObsKey[key]; exists {
|
||||
filterMap[key][value] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert filter map to the desired structure
|
||||
var filters []FilterGroup
|
||||
for key, values := range filterMap {
|
||||
filterData := make([]interface{}, 0, len(values))
|
||||
for value := range values {
|
||||
filterData = append(filterData, value)
|
||||
}
|
||||
filters = append(filters, FilterGroup{
|
||||
FilterKey: key,
|
||||
FilterData: filterData,
|
||||
})
|
||||
}
|
||||
|
||||
return filters, nil
|
||||
}
|
||||
|
||||
var app *pocketbase.PocketBase
|
||||
|
||||
func main() {
|
||||
@ -191,17 +255,17 @@ func main() {
|
||||
|
||||
// Email body
|
||||
body := fmt.Sprintf(`
|
||||
<html>
|
||||
<body>
|
||||
<p>Dear %s,</p>
|
||||
<p>Thank you for registering. Please click the link below to verify your email address:</p>
|
||||
<p>
|
||||
<a href="%s" style="padding: 10px 15px; background-color: blue; color: white; text-decoration: none; border-radius: 5px;">Verify Email</a>
|
||||
</p>
|
||||
<p>If you did not register for this account, please ignore this email.</p>
|
||||
</body>
|
||||
</html>
|
||||
`, name, verificationURL)
|
||||
<html>
|
||||
<body>
|
||||
<p>Dear %s,</p>
|
||||
<p>Thank you for registering. Please click the link below to verify your email address:</p>
|
||||
<p>
|
||||
<a href="%s" style="padding: 10px 15px; background-color: blue; color: white; text-decoration: none; border-radius: 5px;">Verify Email</a>
|
||||
</p>
|
||||
<p>If you did not register for this account, please ignore this email.</p>
|
||||
</body>
|
||||
</html>
|
||||
`, name, verificationURL)
|
||||
|
||||
// Create the email message
|
||||
message := &mailer.Message{
|
||||
@ -255,50 +319,50 @@ func main() {
|
||||
// Check if the email is already verified
|
||||
if record.GetBool("reviewed") {
|
||||
return c.HTML(http.StatusOK, `
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
background-color: #f0f8ff;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 100vh;
|
||||
margin: 0;
|
||||
}
|
||||
.message-container {
|
||||
text-align: center;
|
||||
background-color: #ffe0e0;
|
||||
padding: 20px;
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
.message-container h1 {
|
||||
font-size: 24px;
|
||||
color: #b71c1c;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.message-container p {
|
||||
font-size: 16px;
|
||||
color: #880e4f;
|
||||
}
|
||||
.icon {
|
||||
font-size: 50px;
|
||||
color: #b71c1c;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="message-container">
|
||||
<div class="icon">✖</div>
|
||||
<h1>Your Already Reviewed</h1>
|
||||
<p>You have already been reviewed. No further action is required.</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`)
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
background-color: #f0f8ff;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 100vh;
|
||||
margin: 0;
|
||||
}
|
||||
.message-container {
|
||||
text-align: center;
|
||||
background-color: #ffe0e0;
|
||||
padding: 20px;
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
.message-container h1 {
|
||||
font-size: 24px;
|
||||
color: #b71c1c;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.message-container p {
|
||||
font-size: 16px;
|
||||
color: #880e4f;
|
||||
}
|
||||
.icon {
|
||||
font-size: 50px;
|
||||
color: #b71c1c;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="message-container">
|
||||
<div class="icon">✖</div>
|
||||
<h1>Your Already Reviewed</h1>
|
||||
<p>You have already been reviewed. No further action is required.</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`)
|
||||
}
|
||||
|
||||
// Update the status
|
||||
@ -745,17 +809,11 @@ func main() {
|
||||
})
|
||||
|
||||
e.Router.GET("/api/custom/apicall", func(c echo.Context) error {
|
||||
// Get kpi_id from query parameters
|
||||
|
||||
dataset := c.QueryParam("dataset")
|
||||
if dataset == "" {
|
||||
return c.JSON(http.StatusBadRequest, map[string]string{"error": "dataset is required"})
|
||||
}
|
||||
apicall := c.QueryParam("apicall")
|
||||
// Convert apicall string to a boolean
|
||||
saveToFile := false // Default value
|
||||
if apicall == "true" {
|
||||
saveToFile = true
|
||||
}
|
||||
|
||||
// Fetch all matching records manually
|
||||
// Construct the dbx.Expression
|
||||
@ -781,10 +839,14 @@ func main() {
|
||||
}
|
||||
|
||||
// Fetch and convert XML to JSON
|
||||
_, err := fetchAndConvertXMLToJSON(apiURL, kpi, saveToFile)
|
||||
if err != nil {
|
||||
log.Printf("Failed to process URL '%s': %v", apiURL, err)
|
||||
continue // Skip this record and proceed with others
|
||||
saveToFile := record.GetBool("file_status")
|
||||
chartId := record.GetString("id")
|
||||
if !saveToFile { // Check if saveToFile is false
|
||||
_, err := fetchAndConvertXMLToJSON(apiURL, kpi, chartId)
|
||||
if err != nil {
|
||||
log.Printf("Failed to process URL '%s': %v", apiURL, err)
|
||||
continue // Skip this record and proceed with others
|
||||
}
|
||||
}
|
||||
|
||||
// Read the JSON data from the file
|
||||
@ -794,6 +856,24 @@ func main() {
|
||||
continue // Skip this record and proceed with others
|
||||
}
|
||||
|
||||
var filters []FilterGroup
|
||||
filterKeysRaw := record.GetString("filter")
|
||||
if record.GetBool("is_chart") {
|
||||
|
||||
// Split the filter keys if they come as a comma-separated string
|
||||
filterKeys := strings.Split(filterKeysRaw, ",")
|
||||
|
||||
// Generate filters
|
||||
var err error
|
||||
filters, err = generateFilters(response, filterKeys)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
} else {
|
||||
// Assign an empty filter slice
|
||||
filters = []FilterGroup{}
|
||||
}
|
||||
|
||||
// Prepare the structure with URL and its response
|
||||
recordResult := map[string]interface{}{
|
||||
"url": apiURL,
|
||||
@ -804,6 +884,8 @@ func main() {
|
||||
"is_chart": record.GetString("is_chart"),
|
||||
"chart_type": record.GetString("chart_type"),
|
||||
"response": response,
|
||||
"filters": filters,
|
||||
"group_by": record.GetString("group_by"),
|
||||
}
|
||||
|
||||
// Add the result to the aggregated response
|
||||
|
||||
BIN
pb_data/data.db
BIN
pb_data/data.db
Binary file not shown.
BIN
pb_data/logs.db
BIN
pb_data/logs.db
Binary file not shown.
Loading…
Reference in New Issue
Block a user