diff --git a/main.go b/main.go index 0c9dabb..98339aa 100644 --- a/main.go +++ b/main.go @@ -161,12 +161,18 @@ type ResponseItem struct { ObsValue map[string]string `json:"ObsValue"` } -type FilterGroup struct { - FilterKey string `json:"filter_key"` - FilterData []interface{} `json:"filter_data"` +type FilterJsonData struct { + Order int `json:"order"` + Text string `json:"text"` } -func generateFilters(responseRaw []map[string]interface{}, filterKeys []string, unsetKey string, unsetValue string) ([]FilterGroup, error) { +type FilterGroup struct { + FilterKey string `json:"filter_key"` + FilterData []interface{} `json:"filter_data"` + FilterTextAndOrder FilterJsonData `json:"filter_text_and_order"` +} + +func generateFilters(responseRaw []map[string]interface{}, filterKeys []string, filterJsonData map[string]FilterJsonData) ([]FilterGroup, error) { // Transform responseRaw to []ResponseItem var response []ResponseItem responseBytes, err := json.Marshal(responseRaw) // Marshal raw data to JSON @@ -200,17 +206,13 @@ func generateFilters(responseRaw []map[string]interface{}, filterKeys []string, for key, values := range filterMap { filterData := make([]interface{}, 0, len(values)) for value := range values { - if key == unsetKey { - if value != unsetValue { - filterData = append(filterData, value) - } - } else { - filterData = append(filterData, value) - } + + filterData = append(filterData, value) } filters = append(filters, FilterGroup{ - FilterKey: key, - FilterData: filterData, + FilterKey: key, + FilterData: filterData, + FilterTextAndOrder: filterJsonData[key], }) } @@ -222,19 +224,19 @@ func fetchfilterJSON(kpi string) ([]FilterGroup, error) { // Define struct for the result var dataSetFilter struct { - Id string `db:"id" json:"id"` - FullDataUrl string `db:"full_data_url" json:"full_data_url"` - FilterKeys string `db:"filter_keys" json:"filter_keys"` - FileStatus bool `db:"file_status" json:"file_status"` - DataSet string `db:"dataset" json:"dataset"` - Kpi string `db:"kpi" json:"kpi"` - UnsetKey string `db:"unset_key" json:"unset_key"` - UnsetValue string `db:"unset_value" json:"unset_value"` + Id string `db:"id" json:"id"` + FullDataUrl string `db:"full_data_url" json:"full_data_url"` + FilterKeys string `db:"filter_keys" json:"filter_keys"` + FileStatus bool `db:"file_status" json:"file_status"` + DataSet string `db:"dataset" json:"dataset"` + Kpi string `db:"kpi" json:"kpi"` + ExclusionFiltersJSON string `db:"exclusion_filters_array" json:"exclusion_filters_array"` + FilterJson string `db:"filter_json" json:"filter_json"` } // Query to get one row from the data_set_filter table err := app.DB(). - Select("id", "full_data_url", "filter_keys", "file_status", "dataset", "kpi", "unset_key", "unset_value"). + Select("id", "full_data_url", "filter_keys", "file_status", "dataset", "kpi", "filter_json", "exclusion_filters_array"). From("data_set_filter"). Where(dbx.HashExp{"kpi": kpi}). One(&dataSetFilter) @@ -244,6 +246,15 @@ func fetchfilterJSON(kpi string) ([]FilterGroup, error) { return nil, err } + // Use a map to store the dynamic JSON keys + var filterJson map[string]FilterJsonData + + // Parse the JSON + err = json.Unmarshal([]byte(dataSetFilter.FilterJson), &filterJson) + if err != nil { + log.Fatalf("Error parsing JSON: %v", err) + } + // Fetch and convert XML to JSON if file is not saved if !dataSetFilter.FileStatus { // Check if FileStatus is false _, err := fetchAndConvertXMLToJSON(dataSetFilter.FullDataUrl, "filter_files", dataSetFilter.Kpi, "data_set_filter", dataSetFilter.Id) @@ -264,14 +275,72 @@ func fetchfilterJSON(kpi string) ([]FilterGroup, error) { filterKeys := strings.Split(dataSetFilter.FilterKeys, ",") // Generate filters - filters, err := generateFilters(response, filterKeys, dataSetFilter.UnsetKey, dataSetFilter.UnsetValue) + filters, err := generateFilters(response, filterKeys, filterJson) if err != nil { log.Printf("Failed to generate filters: %v", err) return nil, err } - // Return the filters directly + // Debug Log Exclusion Filters + log.Printf("Raw Exclusion Filters JSON: %s", dataSetFilter.ExclusionFiltersJSON) + + // Parse exclusion filters JSON if it's not empty + var exclusionFilters []utils.FilterGroup + if dataSetFilter.ExclusionFiltersJSON != "" { + err = json.Unmarshal([]byte(dataSetFilter.ExclusionFiltersJSON), &exclusionFilters) + if err != nil { + log.Printf("Failed to parse exclusion filters JSON: %v", err) + return nil, err + } + log.Printf("Parsed Exclusion Filters: %+v", exclusionFilters) + } + + // Convert filters from []FilterGroup to []utils.FilterGroup manually + var utilsFilters []utils.FilterGroup + for _, f := range filters { + utilsFilters = append(utilsFilters, convertToUtilsFilterGroup(f)) + } + + // Apply the filter removal logic only if exclusionFilters has values + if len(exclusionFilters) > 0 { + filteredResult := utils.RemoveExcludedFilters(utilsFilters, exclusionFilters) + + // Convert back to []FilterGroup before returning + var finalFilters []FilterGroup + for _, f := range filteredResult { + finalFilters = append(finalFilters, convertToFilterGroup(f)) + } + + return finalFilters, nil + } + + // If no exclusion filters, return the original filters return filters, nil + +} + +// Convert FilterGroup to utils.FilterGroup +func convertToUtilsFilterGroup(f FilterGroup) utils.FilterGroup { + return utils.FilterGroup{ + FilterKey: f.FilterKey, + FilterData: f.FilterData, + FilterTextAndOrder: utils.FilterJsonData{ + Order: f.FilterTextAndOrder.Order, + Text: f.FilterTextAndOrder.Text, + }, + } +} + +// Convert utils.FilterGroup to FilterGroup +func convertToFilterGroup(f utils.FilterGroup) FilterGroup { + return FilterGroup{ + FilterKey: f.FilterKey, + FilterData: f.FilterData, + FilterTextAndOrder: FilterJsonData{ + Order: f.FilterTextAndOrder.Order, + Text: f.FilterTextAndOrder.Text, + }, + } } // LanguageSource represents the source language data structure diff --git a/pb_data/data.db b/pb_data/data.db index 1375e25..330e891 100644 Binary files a/pb_data/data.db and b/pb_data/data.db differ diff --git a/pb_data/data.db-shm b/pb_data/data.db-shm new file mode 100644 index 0000000..fe9ac28 Binary files /dev/null and b/pb_data/data.db-shm differ diff --git a/pb_data/data.db-wal b/pb_data/data.db-wal new file mode 100644 index 0000000..e69de29 diff --git a/pb_data/logs.db b/pb_data/logs.db index 1cc3321..c0f66bc 100644 Binary files a/pb_data/logs.db and b/pb_data/logs.db differ diff --git a/pb_data/logs.db-shm b/pb_data/logs.db-shm new file mode 100644 index 0000000..19f38fe Binary files /dev/null and b/pb_data/logs.db-shm differ diff --git a/pb_data/logs.db-wal b/pb_data/logs.db-wal new file mode 100644 index 0000000..da8095d Binary files /dev/null and b/pb_data/logs.db-wal differ diff --git a/utils/filter_utils.go b/utils/filter_utils.go new file mode 100644 index 0000000..9c0310f --- /dev/null +++ b/utils/filter_utils.go @@ -0,0 +1,64 @@ +package utils + +type FilterJsonData struct { + Order int `json:"order"` + Text string `json:"text"` +} + +type FilterGroup struct { + FilterKey string `json:"filter_key"` + FilterData []interface{} `json:"filter_data"` + FilterTextAndOrder FilterJsonData `json:"filter_text_and_order"` +} + +// RemoveExcludedFilters removes specified filter_data values from filters. +func RemoveExcludedFilters(filters []FilterGroup, exclusions []FilterGroup) []FilterGroup { + // Create a map for quick lookup of exclusion filter data + exclusionMap := make(map[string]map[interface{}]bool) + + // Populate the exclusion map + for _, excl := range exclusions { + if _, exists := exclusionMap[excl.FilterKey]; !exists { + exclusionMap[excl.FilterKey] = make(map[interface{}]bool) + } + for _, value := range excl.FilterData { + exclusionMap[excl.FilterKey][value] = true + } + } + + // Debug: Print exclusionMap to verify exclusions + // log.Println("Exclusion Map: ", exclusionMap) + + // Process the filters and remove excluded values + var filteredFilters []FilterGroup + + for _, filter := range filters { + if excludedValues, found := exclusionMap[filter.FilterKey]; found { + // Debug: Print current filter data and exclusions + // log.Printf("Checking filter: %s with data: %v", filter.FilterKey, filter.FilterData) + + // Remove excluded values + var newFilterData []interface{} + for _, value := range filter.FilterData { + // Debug: Check if value is excluded + if _, exists := excludedValues[value]; exists { + // log.Printf("Excluding value: %v", value) // Debug log for excluded value + } else { + newFilterData = append(newFilterData, value) + } + } + if len(newFilterData) > 0 { + filter.FilterData = newFilterData + filteredFilters = append(filteredFilters, filter) + } + } else { + // No exclusions, add filter as is + filteredFilters = append(filteredFilters, filter) + } + } + + // Debug: Print final filtered result + // log.Println("Filtered Filters: ", filteredFilters) + + return filteredFilters +}