This commit is contained in:
Gowtham M 2025-10-01 15:52:06 +05:30
parent fe106c9365
commit 9649a0ebdc
2612 changed files with 25427 additions and 110 deletions

3
.env
View File

@ -1,3 +1,4 @@
APP_SIGNATURE = fcsc.gov.ae.X7pL9qZm2A
BASE_URL = https://pb.venbait.in
LIVE_BASE_URL = https://pb.venbait.in
BASE_URL = http://127.0.0.1:8090

289
main.go
View File

@ -1,6 +1,7 @@
package main
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
@ -36,6 +37,7 @@ import (
)
var baseUrl = os.Getenv("BASE_URL")
var liveBaseUrl = os.Getenv("LIVE_BASE_URL")
type EmailConfigurationResponse struct {
Page int `json:"page"`
@ -95,6 +97,9 @@ func fetchAndConvertXMLToJSON(apiURL string, outputDir string, fileName string,
if err != nil {
return nil, err
}
// fmt.Println("Response url:", apiURL)
// fmt.Println("Response status:", resp.Status)
// fmt.Println("Response body:", string(body))
// Clean the XML by removing namespace prefixes
cleanXML := []byte(removeNamespace(body))
@ -472,6 +477,7 @@ func main() {
}
var baseUrl = os.Getenv("BASE_URL")
var liveBaseUrl = os.Getenv("LIVE_BASE_URL")
app = pocketbase.New()
@ -1458,112 +1464,6 @@ func main() {
})
// e.Router.GET("/api/getHomePageData", func(c echo.Context) error {
// language := c.QueryParam("language")
// colorMode := c.QueryParam("color_mode")
// var colorPatternColumn string
// if colorMode == "dark" {
// colorPatternColumn = "color_pattern_dark"
// } else {
// colorPatternColumn = "color_pattern_light"
// }
// var languageSpecificColumn string
// if language == "en" {
// languageSpecificColumn = "data_set_tile_heading_en"
// } else {
// languageSpecificColumn = "data_set_tile_heading_ar"
// }
// // Define the struct for holding the query results
// mainTopics := []struct {
// MainTopicEn string `db:"main_topic_en" json:"main_topic_en"`
// MainTopicAr string `db:"main_topic_ar" json:"main_topic_ar"`
// MainTopicListOrder string `db:"main_topic_list_order" json:"main_topic_list_order"`
// ColorPattern string `db:"color_pattern" json:"color_pattern"`
// }{}
// // Execute the query
// query := fmt.Sprintf(`
// SELECT
// main_topic_en,
// main_topic_ar,
// main_topic_list_order,
// %s AS color_pattern
// FROM home_screen
// GROUP BY main_topic_en, main_topic_ar, main_topic_list_order, %s
// ORDER BY main_topic_list_order ASC
// `, colorPatternColumn, colorPatternColumn)
// err := app.DB().NewQuery(query).All(&mainTopics)
// // err := app.DB().
// // Select("main_topic_en", "main_topic_ar", "main_topic_list_order", "color_pattern").
// // From("home_screen").
// // GroupBy("main_topic_en", "main_topic_ar", "color_pattern").
// // OrderBy("main_topic_list_order ASC").
// // All(&mainTopics)
// if err != nil {
// log.Printf("Failed to fetch home_screen data: %v", err)
// return c.JSON(http.StatusInternalServerError, map[string]string{"error": "Failed to fetch records"})
// }
// // If no records are found
// if len(mainTopics) == 0 {
// return c.JSON(http.StatusNotFound, map[string]string{"error": "No records found for home screen"})
// }
// // Final result structure
// result := []map[string]interface{}{}
// // Loop through main topics and fetch sub-topic data for each
// for _, mainTopic := range mainTopics {
// dataSets := []struct {
// DataSet string `db:"data_set" json:"data_set"`
// DataSetTileHeading string `db:"data_set_tile_heading" json:"data_set_tile_heading"`
// ValueSource string `db:"value_source" json:"value_source"`
// Value string `db:"value" json:"value"`
// DataSetListOrder string `db:"data_set_list_order" json:"data_set_list_order"`
// }{}
// query := ` SELECT data_set,` + languageSpecificColumn + ` AS data_set_tile_heading,value_source,value, data_set_list_order FROM home_screen WHERE main_topic_en = {:topic}`
// err := app.DB().
// NewQuery(query).
// Bind(dbx.Params{
// "topic": mainTopic.MainTopicEn,
// }).
// All(&dataSets)
// if err != nil {
// log.Printf("Error fetching data sets: %v", err)
// continue
// }
// var MainTopic string
// if language == "en" {
// MainTopic = mainTopic.MainTopicEn
// } else {
// MainTopic = mainTopic.MainTopicAr
// }
// // Add the main topic and its sub-topics to the result
// result = append(result, map[string]interface{}{
// "main_topic": MainTopic,
// "main_topic_list_order": mainTopic.MainTopicListOrder,
// "color_pattern": mainTopic.ColorPattern,
// "tile_data": dataSets,
// })
// }
// // Send the combined result as JSON
// return c.JSON(http.StatusOK, result)
// })
e.Router.GET("/api/getUAENumbersData", func(c echo.Context) error {
apiKey := c.Request().Header.Get("APP_SIGNATURE")
@ -1801,6 +1701,183 @@ func main() {
})
type DownloadRequest struct {
Username string `json:"username"`
Password string `json:"password"`
Collection string `json:"collection"`
SyncWithLive bool `json:"syncWithLive"`
LiveUsername string `json:"liveUsername"`
LivePassword string `json:"livePassword"`
}
type AdminLoginResponse struct {
Token string `json:"token"`
Admin map[string]interface{} `json:"admin"`
}
type DownloadResponse struct {
Collection string `json:"collection"`
Records []map[string]interface{} `json:"records"`
}
e.Router.POST("/api/downloadWithLogin", func(c echo.Context) error {
reqBody := new(DownloadRequest)
if err := c.Bind(reqBody); err != nil {
return c.JSON(400, map[string]string{"error": "invalid request"})
}
if reqBody.Username == "" || reqBody.Password == "" || reqBody.Collection == "" {
return c.JSON(400, map[string]string{"error": "username, password, and collection are required"})
}
// 1⃣ Login to PocketBase
loginPayload := map[string]string{
"identity": reqBody.Username,
"password": reqBody.Password,
}
data, _ := json.Marshal(loginPayload)
loginReq, _ := http.NewRequest("POST", fmt.Sprintf("%s/api/admins/auth-with-password", baseUrl), bytes.NewBuffer(data))
loginReq.Header.Set("Content-Type", "application/json")
loginResp, err := http.DefaultClient.Do(loginReq)
if err != nil {
return c.JSON(500, map[string]string{"error": "failed to connect to PB"})
}
defer loginResp.Body.Close()
loginBody, _ := ioutil.ReadAll(loginResp.Body)
if loginResp.StatusCode != 200 {
return c.JSON(401, map[string]string{"error": "invalid admin credentials", "details": string(loginBody)})
}
var loginRes AdminLoginResponse
if err := json.Unmarshal(loginBody, &loginRes); err != nil {
return c.JSON(500, map[string]string{"error": "failed to parse login response"})
}
// 2⃣ Use token to fetch records
url := fmt.Sprintf("%s/api/collections/%s/records?perPage=500", baseUrl, reqBody.Collection)
fmt.Println("Fetching collection from URL:", url)
req, _ := http.NewRequest("GET", url, nil)
req.Header.Set("Authorization", loginRes.Token)
fmt.Println("Using token:", loginRes.Token)
resp, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Println("Error fetching collection:", err)
return c.JSON(500, map[string]string{"error": "failed to fetch collection"})
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
fmt.Println("HTTP Status Code:", resp.StatusCode)
// parse JSON
var parsed map[string]interface{}
if err := json.Unmarshal(body, &parsed); err != nil {
fmt.Println("JSON Unmarshal error:", err)
return c.JSON(500, map[string]string{"error": "invalid JSON from PocketBase"})
}
items, ok := parsed["items"].([]interface{})
if !ok {
return c.JSON(500, map[string]string{"error": "no items found"})
}
records := []map[string]interface{}{}
for _, i := range items {
rec := i.(map[string]interface{})
records = append(records, rec)
}
syncResult := map[string]interface{}{"synced": []string{}, "errors": []string{}}
// 3⃣ If syncWithLive=true, sync to live PB
if reqBody.SyncWithLive {
if reqBody.LiveUsername == "" || reqBody.LivePassword == "" {
return c.JSON(400, map[string]string{"error": "live username and password required for sync"})
}
// Login to live
livePayload := map[string]string{
"identity": reqBody.LiveUsername,
"password": reqBody.LivePassword,
}
data, _ := json.Marshal(livePayload)
liveReq, _ := http.NewRequest("POST", fmt.Sprintf("%s/api/admins/auth-with-password", liveBaseUrl), bytes.NewBuffer(data))
liveReq.Header.Set("Content-Type", "application/json")
liveResp, err := http.DefaultClient.Do(liveReq)
fmt.Println("Live url:", liveBaseUrl)
fmt.Println("Live req:", liveReq)
if err != nil {
return c.JSON(500, map[string]string{"error": "failed to connect to Live PB"})
}
defer liveResp.Body.Close()
liveBody, _ := ioutil.ReadAll(liveResp.Body)
if liveResp.StatusCode != 200 {
return c.JSON(401, map[string]string{"error": "invalid live admin credentials", "details": string(liveBody)})
}
var liveLogin AdminLoginResponse
if err := json.Unmarshal(liveBody, &liveLogin); err != nil {
return c.JSON(500, map[string]string{"error": "failed to parse live login response"})
}
// Sync each record
for _, rec := range records {
id := rec["id"].(string)
data, _ := json.Marshal(rec)
// Try PATCH first
patchURL := fmt.Sprintf("%s/api/collections/%s/records/%s", liveBaseUrl, reqBody.Collection, id)
req, _ := http.NewRequest("PATCH", patchURL, bytes.NewBuffer(data))
req.Header.Set("Authorization", liveLogin.Token)
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
syncResult["errors"] = append(syncResult["errors"].([]string), fmt.Sprintf("%s: %s", id, err.Error()))
continue
}
defer resp.Body.Close()
if resp.StatusCode == 404 {
// Create instead
postURL := fmt.Sprintf("%s/api/collections/%s/records", liveBaseUrl, reqBody.Collection)
req, _ := http.NewRequest("POST", postURL, bytes.NewBuffer(data))
req.Header.Set("Authorization", liveLogin.Token)
req.Header.Set("Content-Type", "application/json")
resp, err = http.DefaultClient.Do(req)
if err != nil {
syncResult["errors"] = append(syncResult["errors"].([]string), fmt.Sprintf("%s: %s", id, err.Error()))
continue
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
body, _ := ioutil.ReadAll(resp.Body)
syncResult["errors"] = append(syncResult["errors"].([]string), fmt.Sprintf("%s: create failed: %s", id, string(body)))
continue
}
}
syncResult["synced"] = append(syncResult["synced"].([]string), id)
}
}
// 4⃣ Return dev records + optional sync result
return c.JSON(200, map[string]interface{}{
"collection": reqBody.Collection,
"records": records,
"sync": syncResult,
})
})
return nil
})

BIN
pb.zip Normal file

Binary file not shown.

4
pb/.env Normal file
View File

@ -0,0 +1,4 @@
APP_SIGNATURE = fcsc.gov.ae.X7pL9qZm2A
LIVE_BASE_URL = https://pocket.fcsc.gov.ae
BASE_URL = https://pocket.fcsc.gov.ae

3069
pb/main.go Executable file

File diff suppressed because it is too large Load Diff

825
pb/utils/calculation.go Executable file
View File

@ -0,0 +1,825 @@
package utils
import (
"encoding/json"
"errors"
"fmt"
"math"
"regexp"
"sort"
"strconv"
"strings"
)
// Observation struct with json.Number for flexible value handling
type Observation struct {
ObsKey map[string]string `json:"ObsKey"`
ObsValue struct {
Value json.Number `json:"Value"`
} `json:"ObsValue"`
}
// ------------------------------------------------------------ card calculation---------------------------------------------------
type GroupByAndSum struct {
GroupByAndSum string `json:"groupby_and_sum"`
GetTop int `json:"get_top"`
Conversion int `json:"conversion"`
}
type CountCalculation struct {
Key string `json:"key"`
GetCountOf string `json:"get_count_of"`
}
type CardCalculationRequest struct {
Formula string `json:"formula"`
RecordCount string `json:"record_count"`
RecodeIndex string `json:"record_index"`
RecordKey string `json:"record_key"`
AdditionalRecordKey string `json:"additional_record_key"`
NumberFormat string `json:"number_format"`
Conversion string `json:"conversion"`
GroupByAndSum GroupByAndSum `json:"groupby_and_sum"`
CountCalculation CountCalculation `json:"count_calculation"`
}
func CardCalculation(data []Observation, req CardCalculationRequest) (map[string]string, error) {
// Sort the data in descending order based on TimePeriod
sort.Slice(data, func(i, j int) bool {
return data[i].ObsKey["TIME_PERIOD"] > data[j].ObsKey["TIME_PERIOD"]
})
var selectedData []Observation
if req.RecodeIndex != "" {
if req.RecodeIndex == "last" {
index := len(data)
if index < 0 || index >= len(data) {
selectedData = []Observation{}
} else {
selectedData = []Observation{data[index]}
}
} else {
// Convert RecodeIndex to an integer
index, err := strconv.Atoi(req.RecodeIndex)
if err != nil || index < 0 || index >= len(data) {
selectedData = []Observation{}
} else {
selectedData = []Observation{data[index]}
}
}
} else {
// Determine the number of records to use
if req.RecordCount == "all" {
if len(data) > 0 {
selectedData = data
} else {
selectedData = []Observation{}
}
} else if req.RecordCount == "first_pair" {
if len(data) >= 2 {
selectedData = data[:2]
} else {
selectedData = []Observation{}
}
} else if req.RecordCount == "second_pair" {
if len(data) >= 3 {
selectedData = data[1:3]
} else {
selectedData = []Observation{}
}
} else {
n := len(data) // array leangth
var err error
n, err = strconv.Atoi(req.RecordCount)
if err != nil || n >= len(data) {
// return nil, errors.New("invalid record count");
selectedData = []Observation{}
} else {
selectedData = data[:n]
}
}
}
var result float64
switch req.Formula {
case "none":
result = none(selectedData)
case "total":
result = sum(selectedData)
case "average":
result = average(selectedData)
case "highest":
result = highest(selectedData)
case "lowest":
result = lowest(selectedData)
case "different":
result = difference(selectedData)
case "groupby_and_sum":
customCalculationReq := req.GroupByAndSum
var chartCustomCalculationRequest ChartCalculationRequest
jsonData, err := json.Marshal(customCalculationReq)
if err != nil {
fmt.Println("Error:", err)
}
err = json.Unmarshal(jsonData, &chartCustomCalculationRequest)
if err != nil {
fmt.Println("Error:", err)
}
selectedData, err = CalculateGroupedSum(selectedData, chartCustomCalculationRequest)
if err != nil {
fmt.Println("Error:", err)
}
if len(selectedData) > 0 {
value, err := selectedData[0].ObsValue.Value.Float64()
if err == nil {
result = value
} else {
fmt.Println("Error converting first observation value to float64:", err)
}
} else {
fmt.Println("No observations returned from CalculateGroupedSum")
}
case "count_calculation":
countCalculationReq := req.CountCalculation
var cardCountCalculationRequest CountCalculation
jsonData, err := json.Marshal(countCalculationReq)
if err != nil {
fmt.Println("Error:", err)
}
err = json.Unmarshal(jsonData, &cardCountCalculationRequest)
if err != nil {
fmt.Println("Error:", err)
}
selectedData, err = CalculateCount(selectedData, cardCountCalculationRequest)
if err != nil {
fmt.Println("Error:", err)
}
if len(selectedData) > 0 {
result = float64(len(selectedData))
selectedData = selectedData[:1]
} else {
fmt.Println("No observations returned from CalculateGroupedSum")
}
default:
return nil, errors.New("invalid formula")
}
convertedValue := applyConversion(result, req.Conversion, req.NumberFormat)
displayValue := extractRecordKeyValue(selectedData, req.RecordKey, req.AdditionalRecordKey, req.Formula, result)
color := ""
if req.Formula == "different" || req.Conversion == "%" {
// Remove the percentage sign if present
convertedValueStr := strings.TrimSuffix(convertedValue, "%")
// Convert to float
convertedValue, err := strconv.ParseFloat(convertedValueStr, 64)
if err != nil {
return nil, err // Handle error if conversion fails
}
if convertedValue > 0 {
color = "#11AF22"
} else if convertedValue < 0 {
color = "#D83731"
}
}
return map[string]string{
"value": convertedValue,
"display_value": displayValue,
"calculation": req.Formula,
"font_color": color,
}, nil
}
// Helper function to convert json.Number to float64 safely
func getFloatValue(num json.Number) float64 {
val, err := num.Float64()
if err != nil {
fmt.Println("Error converting value:", err)
return 0
}
return val
}
func none(data []Observation) float64 {
total := 0.0
for _, obs := range data {
total += getFloatValue(obs.ObsValue.Value)
}
return total
}
func sum(data []Observation) float64 {
total := 0.0
for _, obs := range data {
total += getFloatValue(obs.ObsValue.Value)
}
return total
}
func average(data []Observation) float64 {
if len(data) == 0 {
return 0
}
return sum(data) / float64(len(data))
}
func highest(data []Observation) float64 {
if len(data) == 0 {
return 0
}
max := getFloatValue(data[0].ObsValue.Value)
for _, obs := range data[1:] {
val := getFloatValue(obs.ObsValue.Value)
if val > max {
max = val
}
}
return max
}
func lowest(data []Observation) float64 {
if len(data) == 0 {
return 0
}
min := getFloatValue(data[0].ObsValue.Value)
for _, obs := range data {
val := getFloatValue(obs.ObsValue.Value)
if val < min {
min = val
}
}
return min
}
func difference(data []Observation) float64 {
if len(data) < 2 {
return 0
}
latest := getFloatValue(data[0].ObsValue.Value)
previous := getFloatValue(data[1].ObsValue.Value)
if previous == 0 {
return 0
}
return ((latest - previous) / previous) * 100
}
// Function to add commas to a number string
func addCommas(num string, suffix string) string {
n := len(num)
if n <= 3 {
return num + suffix // Attach suffix directly if no commas needed
}
var sb strings.Builder
start := n % 3
if start == 0 {
start = 3
}
sb.WriteString(num[:start])
for i := start; i < n; i += 3 {
sb.WriteString("," + num[i:i+3])
}
return sb.String() + suffix
}
func formatNumberWithCommas(value float64, suffix string) string {
// Convert the value to a string with 2 decimal places
formatted := fmt.Sprintf("%.2f", value)
// Split integer and decimal parts
parts := strings.Split(formatted, ".")
integerPart := parts[0]
decimalPart := parts[1]
// Add commas to the integer part
var result strings.Builder
n := len(integerPart)
for i, c := range integerPart {
if i > 0 && (n-i)%3 == 0 {
result.WriteRune(',')
}
result.WriteRune(c)
}
// Append decimal part if needed
if decimalPart != "00" {
return result.String() + "." + decimalPart + suffix
}
return result.String() + suffix
}
func applyConversion(value float64, conversion string, numberFormat string) string {
if numberFormat == "T" {
value *= 1000000000000
} else if numberFormat == "B" {
value *= 1000000000
} else if numberFormat == "M" {
value *= 1000000
} else if numberFormat == "K" {
value *= 1000
}
// Helper function to format numbers with a thousands separator only in the default case
formatNumber := func(v float64, suffix string) string {
if suffix == "" || suffix == "K" || suffix == "M" || suffix == "B" || suffix == "T" {
if v == float64(int(v)) { // No decimal part
return addCommas(strconv.Itoa(int(v)), suffix)
}
return formatNumberWithCommas(v, suffix)
}
if v == float64(int(v)) { // No decimal part
return fmt.Sprintf("%d%s", int(v), suffix)
}
return fmt.Sprintf("%.2f%s", v, suffix)
}
switch conversion {
case "T":
return formatNumber(value/1000000000000, "T")
case "B":
return formatNumber(value/1000000000, "B")
case "M":
return formatNumber(value/1000000, "M")
case "K":
return formatNumber(value/1000, "K")
case "%":
return formatNumber(value, "%")
default:
return formatNumber(value, conversion)
}
}
func extractRecordKeyValue(data []Observation, key string, additionalkey string, formula string, targetValue float64) string {
if len(data) == 0 {
return ""
}
switch formula {
case "highest", "lowest", "none", "groupby_and_sum":
for _, obs := range data {
value, err := obs.ObsValue.Value.Float64()
if err == nil && value == targetValue {
if val, exists := obs.ObsKey[key]; exists {
// Check if additionalkey exists in obs.ObsKey
if val2, exists2 := obs.ObsKey[additionalkey]; exists2 {
return val + " - " + val2
} else {
return val
}
}
}
}
case "total", "average", "different":
if val1, exists1 := data[0].ObsKey[key]; exists1 {
if val2, exists2 := data[len(data)-1].ObsKey[key]; exists2 {
if val1 == val2 {
return val1
} else {
if val1 > val2 {
return val2 + " - " + val1
} else {
return val1 + " - " + val2
}
}
}
return val1
}
case "count_calculation":
for _, obs := range data {
if val, exists := obs.ObsKey[key]; exists {
// Check if additionalkey exists in obs.ObsKey
if val2, exists2 := obs.ObsKey[additionalkey]; exists2 {
return val + " - " + val2
} else {
return val
}
}
}
}
return ""
}
// ------------------------------------------------------------ chart calculation---------------------------------------------------
type ChartCalculationRequest struct {
GroupByAndSum string `json:"groupby_and_sum"`
AdditionalForGroupByAndSum string `json:"additional_for_groupby_and_sum"`
GetTop string `json:"get_top"`
SortBy string `json:"sort_by"`
SortAction string `json:"sort_action"`
Conversion string `json:"conversion"`
}
func CalculateGroupedSum(observations []Observation, request ChartCalculationRequest) ([]Observation, error) {
// Map to store the summed values
summedData := make(map[string]float64)
if request.GroupByAndSum == "none" {
// If "none", treat each observation separately
// Collect observations with their original keys
var valueList []Observation
for _, obs := range observations {
value, err := obs.ObsValue.Value.Float64()
if err != nil {
return nil, fmt.Errorf("invalid number format: %v", err)
}
// Preserve the original ObsKey
valueList = append(valueList, Observation{
ObsKey: obs.ObsKey, // Keep all keys
ObsValue: struct {
Value json.Number `json:"Value"`
}{
Value: json.Number(strconv.FormatFloat(convertValue(value, request.Conversion), 'f', 6, 64)),
},
})
}
// Sort by value in descending order or given key
sortByKey := request.SortBy
var nonDigit = regexp.MustCompile(`\D`)
if sortByKey != "" {
// Sort by given key
sort.Slice(valueList, func(i, j int) bool {
val1Str := valueList[i].ObsKey[sortByKey]
val2Str := valueList[j].ObsKey[sortByKey]
// remove non-digits
val1Digits := nonDigit.ReplaceAllString(val1Str, "")
val2Digits := nonDigit.ReplaceAllString(val2Str, "")
val1, _ := strconv.ParseInt(val1Digits, 10, 64)
val2, _ := strconv.ParseInt(val2Digits, 10, 64)
sortActionKey := request.SortAction
if sortActionKey == "DESC" {
return val1 > val2 // Descending order
}
return val1 < val2 // Ascending order
})
} else {
// Default sorting by ObsValue.Value
sort.Slice(valueList, func(i, j int) bool {
val1, _ := valueList[i].ObsValue.Value.Float64()
val2, _ := valueList[j].ObsValue.Value.Float64()
return val1 > val2 // Descending order
})
}
// Get the top N values
topN, err := strconv.Atoi(request.GetTop)
if err != nil || topN <= 0 || topN > len(valueList) {
topN = len(valueList) // If invalid input, return all
}
return valueList[:topN], nil
} else {
// Iterate through observations and sum values based on GroupBy key and Additional key
for _, obs := range observations {
groupKey, exists := obs.ObsKey[request.GroupByAndSum]
if !exists {
continue // Skip if the key does not exist in ObsKey
}
additionalKey := ""
if request.AdditionalForGroupByAndSum != "" {
additionalKey, _ = obs.ObsKey[request.AdditionalForGroupByAndSum]
}
combinedKey := groupKey
if additionalKey != "" {
combinedKey += "_" + additionalKey
}
value, err := obs.ObsValue.Value.Float64()
if err != nil {
return nil, fmt.Errorf("invalid number format: %v", err)
}
summedData[combinedKey] += value
}
}
// Convert map to slice of Observation
var result []Observation
for key, sum := range summedData {
keyParts := strings.Split(key, "_")
obsKey := map[string]string{
request.GroupByAndSum: keyParts[0],
}
if len(keyParts) > 1 {
obsKey[request.AdditionalForGroupByAndSum] = keyParts[1]
}
result = append(result, Observation{
ObsKey: obsKey,
ObsValue: struct {
Value json.Number `json:"Value"`
}{
Value: json.Number(strconv.FormatFloat(convertValue(sum, request.Conversion), 'f', 6, 64)),
},
})
}
// Sort by value in descending order or given key
sortByKey := request.SortBy
var nonDigit = regexp.MustCompile(`\D`)
if sortByKey != "" {
// Sort by given key
sort.Slice(result, func(i, j int) bool {
val1Str := result[i].ObsKey[sortByKey]
val2Str := result[j].ObsKey[sortByKey]
// remove non-digits
val1Digits := nonDigit.ReplaceAllString(val1Str, "")
val2Digits := nonDigit.ReplaceAllString(val2Str, "")
val1, _ := strconv.ParseInt(val1Digits, 10, 64)
val2, _ := strconv.ParseInt(val2Digits, 10, 64)
sortActionKey := request.SortAction
if sortActionKey == "DESC" {
return val1 > val2 // Descending order
}
return val1 < val2 // Ascending order
})
} else {
// Default sorting by ObsValue.Value
sort.Slice(result, func(i, j int) bool {
val1, _ := result[i].ObsValue.Value.Float64()
val2, _ := result[j].ObsValue.Value.Float64()
return val1 > val2 // Descending order
})
}
// Sort by summed values in descending order
// sort.Slice(result, func(i, j int) bool {
// val1, _ := result[i].ObsValue.Value.Float64()
// val2, _ := result[j].ObsValue.Value.Float64()
// return val1 > val2 // Sort descending
// })
// Get the top N values
topN, err := strconv.Atoi(request.GetTop)
if err != nil || topN <= 0 || topN > len(result) {
topN = len(result) // If invalid input, return all
}
return result[:topN], nil
}
// CalculateGroupedSum processes the observations based on the given request
// func CalculateGroupedSum(observations []Observation, request ChartCalculationRequest) ([]Observation, error) {
// // Map to store the summed values
// summedData := make(map[string]float64)
// if request.GroupByAndSum == "none" {
// // If "none", treat each observation separately
// for _, obs := range observations {
// value, err := obs.ObsValue.Value.Float64()
// if err != nil {
// return nil, fmt.Errorf("invalid number format: %v", err)
// }
// // Use a unique key (e.g., index-based) to preserve order
// summedData[strconv.Itoa(len(summedData))] = value
// }
// } else {
// // Iterate through observations and sum values based on GroupBy key
// for _, obs := range observations {
// groupKey, exists := obs.ObsKey[request.GroupByAndSum]
// if !exists {
// continue // Skip if the key does not exist in ObsKey
// }
// value, err := obs.ObsValue.Value.Float64()
// if err != nil {
// return nil, fmt.Errorf("invalid number format: %v", err)
// }
// summedData[groupKey] += value
// }
// }
// // Convert map to slice of Observation
// var result []Observation
// for key, sum := range summedData {
// result = append(result, Observation{
// ObsKey: map[string]string{
// request.GroupByAndSum: key,
// },
// ObsValue: struct {
// Value json.Number `json:"Value"`
// }{
// // Value: json.Number(fmt.Sprintf("%.0f", sum)),
// Value: json.Number(strconv.FormatFloat(convertValue(sum, request.Conversion), 'f', 6, 64)),
// },
// })
// }
// // Sort by summed values in descending order
// sort.Slice(result, func(i, j int) bool {
// val1, _ := result[i].ObsValue.Value.Float64()
// val2, _ := result[j].ObsValue.Value.Float64()
// return val1 > val2 // Sort descending
// })
// // Get the top N values
// topN, err := strconv.Atoi(request.GetTop)
// if err != nil || topN <= 0 || topN > len(result) {
// topN = len(result) // If invalid input, return all
// }
// return result[:topN], nil
// }
// Convert values based on conversion type
func convertValue(value float64, conversion string) float64 {
switch conversion {
case "T": // Trillion
return round(value/1000000000000, 2)
case "B": // Billion
return round(value/1000000000, 2)
case "M": // Million
return round(value/1000000, 2)
case "K": // Thousand
return round(value/1000, 2)
default: // No conversion
return round(value, 2)
}
}
// Function to round a float to 2 decimal places
func round(val float64, precision int) float64 {
p := math.Pow(10, float64(precision))
return math.Round(val*p) / p
}
//------------------------------------------------------------- default filter ---------------------------------------------------
func DefaultYearFilter(data []Observation, latestYearCount string) ([]Observation, error) {
// Convert latestYearCount to integer
yearCount, err := strconv.Atoi(latestYearCount)
if err != nil {
return nil, err
}
if yearCount == 0 {
yearCount = 1
}
// Sort the data in descending order based on TIME_PERIOD
sort.Slice(data, func(i, j int) bool {
return data[i].ObsKey["TIME_PERIOD"] > data[j].ObsKey["TIME_PERIOD"]
})
// Collect unique years in sorted order
uniqueYears := make([]string, 0)
yearMap := make(map[string]bool)
for _, item := range data {
year := item.ObsKey["TIME_PERIOD"]
if !yearMap[year] {
yearMap[year] = true
uniqueYears = append(uniqueYears, year)
}
}
// Adjust year count if there are fewer available years
if len(uniqueYears) < yearCount {
yearCount = len(uniqueYears) // Set to available years
}
// Create a set for only the required latest years
latestYears := make(map[string]bool)
for i := 0; i < yearCount; i++ {
latestYears[uniqueYears[i]] = true
}
// Filter the data to keep only records from the selected latest N years
var filteredData []Observation
for _, item := range data {
if latestYears[item.ObsKey["TIME_PERIOD"]] {
filteredData = append(filteredData, item)
}
}
return filteredData, nil
}
type Filter struct {
FilterKey string `json:"filter_key"`
FilterData []string `json:"filter_data"`
}
func FilterByMultipleKeys(data []Observation, filters []Filter) ([]Observation, error) {
var filteredData []Observation
// Convert filter conditions into a map for quick lookup
filterMap := make(map[string]map[string]bool)
for _, filter := range filters {
if len(filter.FilterData) == 0 {
continue // Skip empty filter_data
}
if filterMap[filter.FilterKey] == nil {
filterMap[filter.FilterKey] = make(map[string]bool)
}
for _, value := range filter.FilterData {
filterMap[filter.FilterKey][value] = true
}
}
fmt.Println("filterMap :", filterMap)
// Iterate through data and apply the filters
for _, item := range data {
matches := true
for key, allowedValues := range filterMap {
if _, exists := item.ObsKey[key]; exists {
if !allowedValues[item.ObsKey[key]] { // If value not in allowed list
matches = false
break
}
} else {
matches = false // Key not present in data
break
}
}
if matches {
filteredData = append(filteredData, item)
}
}
return filteredData, nil
}
//-------------------------------------------------------------- Count Calculation -------------------------------------------------
func CalculateCount(observations []Observation, request CountCalculation) ([]Observation, error) {
if request.GetCountOf == "highest" || request.GetCountOf == "lowest" {
var extremeValue int
var filteredObservations []Observation
first := true
isHighest := request.GetCountOf == "highest"
for _, obs := range observations {
if valStr, exists := obs.ObsKey[request.Key]; exists {
val, err := strconv.Atoi(valStr)
if err != nil {
continue // Skip if conversion fails
}
if first || (isHighest && val > extremeValue) || (!isHighest && val < extremeValue) {
extremeValue = val
first = false
}
}
}
// Filter observations that match the extreme value
for _, obs := range observations {
if valStr, exists := obs.ObsKey[request.Key]; exists {
val, err := strconv.Atoi(valStr)
if err == nil && val == extremeValue {
filteredObservations = append(filteredObservations, obs)
}
}
}
return filteredObservations, nil
}
return nil, fmt.Errorf("unsupported get_count_of value: %s", request.GetCountOf)
}

65
pb/utils/exclude_filter_data.go Executable file
View File

@ -0,0 +1,65 @@
package utils
type FilterJsonData struct {
Order int `json:"order"`
En string `json:"en"`
Ar string `json:"ar"`
}
type FilterGroup struct {
FilterKey string `json:"filter_key"`
FilterData []interface{} `json:"filter_data"`
FilterTextAndOrder FilterJsonData `json:"filter_text_and_order"`
}
// RemoveExcludedFilters removes specified filter_data values from filters.
func RemoveExcludedFilters(filters []FilterGroup, exclusions []FilterGroup) []FilterGroup {
// Create a map for quick lookup of exclusion filter data
exclusionMap := make(map[string]map[interface{}]bool)
// Populate the exclusion map
for _, excl := range exclusions {
if _, exists := exclusionMap[excl.FilterKey]; !exists {
exclusionMap[excl.FilterKey] = make(map[interface{}]bool)
}
for _, value := range excl.FilterData {
exclusionMap[excl.FilterKey][value] = true
}
}
// Debug: Print exclusionMap to verify exclusions
// log.Println("Exclusion Map: ", exclusionMap)
// Process the filters and remove excluded values
var filteredFilters []FilterGroup
for _, filter := range filters {
if excludedValues, found := exclusionMap[filter.FilterKey]; found {
// Debug: Print current filter data and exclusions
// log.Printf("Checking filter: %s with data: %v", filter.FilterKey, filter.FilterData)
// Remove excluded values
var newFilterData []interface{}
for _, value := range filter.FilterData {
// Debug: Check if value is excluded
if _, exists := excludedValues[value]; exists {
// log.Printf("Excluding value: %v", value) // Debug log for excluded value
} else {
newFilterData = append(newFilterData, value)
}
}
if len(newFilterData) > 0 {
filter.FilterData = newFilterData
filteredFilters = append(filteredFilters, filter)
}
} else {
// No exclusions, add filter as is
filteredFilters = append(filteredFilters, filter)
}
}
// Debug: Print final filtered result
// log.Println("Filtered Filters: ", filteredFilters)
return filteredFilters
}

View File

@ -0,0 +1,48 @@
package utils
// TranslateChartData processes chart data with language source data to produce the result based on the language key.
func TranslateChartData(chartData []map[string]interface{}, languageSource []map[string]interface{}, langKey string) []map[string]interface{} {
// Helper function to find translation
findTranslation := func(key string, value string) string {
for _, langEntry := range languageSource {
if langEntry["key"] == key && langEntry["value"] == value {
if translatedValue, ok := langEntry[langKey]; ok {
return translatedValue.(string)
}
}
}
return value // Return the original value if no translation is found
}
// Extract unique translation keys from languageSource
translationKeys := map[string]bool{}
for _, langEntry := range languageSource {
if key, ok := langEntry["key"].(string); ok {
translationKeys[key] = true
}
}
// Process each chart data entry
result := []map[string]interface{}{}
for _, entry := range chartData {
obsKey := entry["ObsKey"].(map[string]interface{})
translatedObsKey := map[string]interface{}{}
// Translate fields in ObsKey dynamically based on extracted translation keys
for key, value := range obsKey {
if translationKeys[key] {
translatedObsKey[key] = findTranslation(key, value.(string))
} else {
translatedObsKey[key] = value
}
}
// Append the translated entry to the result
result = append(result, map[string]interface{}{
"ObsKey": translatedObsKey,
"ObsValue": entry["ObsValue"],
})
}
return result
}

View File

@ -1 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"averageLengthOfStay.png"},"md5":"/wPtLtddRDkeFZxAQTJM/g=="}
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"FCSCLogo.png"},"md5":"64vKaJS7ySlwJ01MBaM7jA=="}

View File

@ -1 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"wm5GWTiDEeBU/0WxUDPNLQ=="}
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"+vM8FbqC90cnR9oH7giBtg=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/jpeg","user.metadata":{"original-filename":"1000287684.jpg"},"md5":"UA2VEKUvUpv91ELI1ayF2A=="}

View File

@ -1 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/jpeg","user.metadata":{"original-filename":"file.enc"},"md5":"vBw+7shsc/yBqxUxLwI37Q=="}
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/jpeg","user.metadata":null,"md5":"YpRYnPvJR9Rlra27TIX/QQ=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/jpeg","user.metadata":{"original-filename":"image_picker_F6015E16-A4C4-491D-8CFF-CE71669AF99C-65295-00000C0776ED3D83.jpg"},"md5":"+Z1kSCcjj2mBOX3zIuyk4A=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/jpeg","user.metadata":null,"md5":"G1aysWz6YE5JVQj40dEjOw=="}

Binary file not shown.

After

Width:  |  Height:  |  Size: 254 KiB

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/jpeg","user.metadata":{"original-filename":"Ammar.jpg"},"md5":"vPoShsb0FqX7ibTohKag8g=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/jpeg","user.metadata":null,"md5":"mjcg7GRHcZqGTNEVkc9nEg=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/jpeg","user.metadata":{"original-filename":"1000122419.jpg"},"md5":"qXmhDbfI1nZmlpmZJfRrBw=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/jpeg","user.metadata":null,"md5":"aKkMzCDVLRptiYrzRiV6Kw=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-OilExport.png"},"md5":"N0yA93/49HeIhAyNKrrPHA=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"QtQNpYi5me0OVZxMW9h2Zg=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"female.png"},"md5":"LHr3qNvXccxDzGNR1QwAiA=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"6qNv8XkjaWR5lPp91glyrw=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-GDP.png"},"md5":"GbjBCTBkQMjDA/ePurXQ4w=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"oR1IfXTRW05tuALKgeFtkw=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-GDP.png"},"md5":"GbjBCTBkQMjDA/ePurXQ4w=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"oR1IfXTRW05tuALKgeFtkw=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-NaturalReserves.png"},"md5":"EidU5zyMspc/3lI2BgLvkg=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"uiU2k7ZNxToo4t8GzxMZpg=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-CropsArea.png"},"md5":"6BWT6i/rBlR1hej7AA6Gkw=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"srBd6fWEm86XwUV6j55geA=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-OccupancyRate.png"},"md5":"Cs+jJNffYFWAhBqJwmKX0A=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"U+jFTo4vOOwg1q6sg8TA+A=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-Import.png"},"md5":"2BDdMENO2Qr0frmP5x6KWQ=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"zgwJqK1psQOQsoWeaubwUg=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-Arrival.png"},"md5":"McfbcfxPW54tep7w0S7YIA=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"zwEbsR6JRku8qJ19X0D3jw=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-HealthCenters (2).png"},"md5":"BSOP0npbkrCLnRX9kYzicg=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-HealthCenters.png"},"md5":"QUPb24ZJ+yVFx/cBFROKSQ=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"y9NTfS1CkEsgZhHgNHEUMQ=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"lv9iL3SiCG7IKY80k3aevQ=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"female.png"},"md5":"LHr3qNvXccxDzGNR1QwAiA=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"6qNv8XkjaWR5lPp91glyrw=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-ReExport.png"},"md5":"wuPAc6IENmbI4H4Y5iykQA=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"55tIdEz4EfKupJ7SzRKiNQ=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-GDP.png"},"md5":"GbjBCTBkQMjDA/ePurXQ4w=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"oR1IfXTRW05tuALKgeFtkw=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-OccupancyRate.png"},"md5":"Cs+jJNffYFWAhBqJwmKX0A=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"U+jFTo4vOOwg1q6sg8TA+A=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"KPI-icon-NaturalReserves.png"},"md5":"EidU5zyMspc/3lI2BgLvkg=="}

View File

@ -0,0 +1 @@
{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"uiU2k7ZNxToo4t8GzxMZpg=="}

Some files were not shown because too many files have changed in this diff Show More