Read data from file and dynamically call kpi api and convert xml to json and stote file in server : GOWTHAM
This commit is contained in:
commit
f8662176f8
14
go.mod
14
go.mod
@ -54,20 +54,20 @@ require (
|
||||
github.com/spf13/cast v1.7.0 // indirect
|
||||
github.com/spf13/cobra v1.8.1 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/stretchr/testify v1.8.4 // indirect
|
||||
github.com/stretchr/testify v1.10.0 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasttemplate v1.2.2 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.40.0 // indirect
|
||||
golang.org/x/crypto v0.29.0 // indirect
|
||||
golang.org/x/crypto v0.31.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
|
||||
golang.org/x/image v0.22.0 // indirect
|
||||
golang.org/x/net v0.31.0 // indirect
|
||||
golang.org/x/net v0.33.0 // indirect
|
||||
golang.org/x/oauth2 v0.24.0 // indirect
|
||||
golang.org/x/sync v0.9.0 // indirect
|
||||
golang.org/x/sys v0.27.0 // indirect
|
||||
golang.org/x/term v0.26.0 // indirect
|
||||
golang.org/x/text v0.20.0 // indirect
|
||||
golang.org/x/sync v0.10.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/term v0.27.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.209.0 // indirect
|
||||
|
||||
28
go.sum
28
go.sum
@ -197,8 +197,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
|
||||
@ -221,8 +221,8 @@ gocloud.dev v0.40.0/go.mod h1:drz+VyYNBvrMTW0KZiBAYEdl8lbNZx+OQ7oQvdrFmSQ=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
|
||||
golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo=
|
||||
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
|
||||
@ -245,8 +245,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
|
||||
golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
|
||||
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
|
||||
golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
|
||||
@ -254,8 +254,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
|
||||
golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -267,19 +267,19 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
|
||||
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
|
||||
golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
|
||||
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
|
||||
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
|
||||
golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
||||
6402
kpi_files/hotel_estabhlishments.json
Normal file
6402
kpi_files/hotel_estabhlishments.json
Normal file
File diff suppressed because it is too large
Load Diff
10626
kpi_files/hotel_occupancy_rate.json
Normal file
10626
kpi_files/hotel_occupancy_rate.json
Normal file
File diff suppressed because it is too large
Load Diff
168
main.go
168
main.go
@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
@ -14,6 +15,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/labstack/echo/v5"
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase"
|
||||
"github.com/pocketbase/pocketbase/apis"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
@ -67,7 +69,7 @@ func removeNamespace(xmlBytes []byte) string {
|
||||
}
|
||||
|
||||
// Function to fetch XML from an API and convert it to JSON
|
||||
func fetchAndConvertXMLToJSON(apiURL string) ([]map[string]interface{}, error) {
|
||||
func fetchAndConvertXMLToJSON(apiURL string, kpiName string, saveToFile bool) ([]map[string]interface{}, error) {
|
||||
// Call the API and get the XML response
|
||||
resp, err := http.Get(apiURL)
|
||||
if err != nil {
|
||||
@ -101,13 +103,54 @@ func fetchAndConvertXMLToJSON(apiURL string) ([]map[string]interface{}, error) {
|
||||
response = append(response, obsMap)
|
||||
}
|
||||
|
||||
// Convert the response to JSON
|
||||
jsonData, err := json.MarshalIndent(response, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Save the JSON data to a file if required
|
||||
if saveToFile {
|
||||
// Ensure the folder exists
|
||||
outputDir := "kpi_files"
|
||||
if err := os.MkdirAll(outputDir, os.ModePerm); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Save the file with a timestamped filename
|
||||
fileName := fmt.Sprintf("%s/%s.json", outputDir, kpiName)
|
||||
if err := ioutil.WriteFile(fileName, jsonData, 0644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// Function to read JSON data from a file
|
||||
func readJSONFromFile(kpiName string) ([]map[string]interface{}, error) {
|
||||
// Construct the file path
|
||||
filePath := fmt.Sprintf("kpi_files/%s.json", kpiName)
|
||||
|
||||
// Read and return the JSON content from the file
|
||||
fileContent, err := ioutil.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file '%s': %w", filePath, err)
|
||||
}
|
||||
|
||||
// Parse the JSON content from the file
|
||||
var jsonResponse []map[string]interface{}
|
||||
if err := json.Unmarshal(fileContent, &jsonResponse); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse JSON from file '%s': %w", filePath, err)
|
||||
}
|
||||
|
||||
return jsonResponse, nil
|
||||
}
|
||||
|
||||
var app *pocketbase.PocketBase
|
||||
|
||||
func main() {
|
||||
// app := pocketbase.New()
|
||||
|
||||
app = pocketbase.New()
|
||||
|
||||
//send mail while status changed from app admin
|
||||
@ -646,7 +689,7 @@ func main() {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("OTP sent to %s expires at and saved successfully.\n", email, expiresAt)
|
||||
log.Printf("OTP sent to %s expires at and saved successfully.\n", email)
|
||||
return nil
|
||||
})
|
||||
|
||||
@ -659,6 +702,125 @@ func main() {
|
||||
|
||||
//app.Router.GET("/verify-email", verifyEmailHandler)
|
||||
|
||||
app.OnBeforeServe().Add(func(e *core.ServeEvent) error {
|
||||
|
||||
e.Router.GET("/api/custom/apicalltest", func(c echo.Context) error {
|
||||
// Get the database instance
|
||||
db := app.Dao().DB()
|
||||
|
||||
// Define the struct for the expected result
|
||||
type Chart struct {
|
||||
ID string `json:"id"`
|
||||
ChartType string `json:"chart_type"`
|
||||
Dataset string `json:"dataset"`
|
||||
Filter string `json:"filter"`
|
||||
KPI string `json:"kpi"`
|
||||
MainID string `json:"main_id"`
|
||||
SubID string `json:"sub_id"`
|
||||
URL string `json:"url"`
|
||||
IsChart bool `json:"is_chart"`
|
||||
Created string `json:"created"`
|
||||
Updated string `json:"updated"`
|
||||
}
|
||||
|
||||
// Query to fetch the data
|
||||
sqlQuery := "SELECT * FROM charts"
|
||||
var results []Chart
|
||||
|
||||
// Execute the query
|
||||
err := db.NewQuery(sqlQuery).All(&results)
|
||||
if err != nil {
|
||||
// Log the error details for better debugging
|
||||
log.Printf("Failed to execute query: %v", err)
|
||||
return c.JSON(http.StatusInternalServerError, map[string]string{"error": "Failed to execute query", "details": err.Error()})
|
||||
}
|
||||
|
||||
// Check if results are empty
|
||||
if len(results) == 0 {
|
||||
return c.JSON(http.StatusNotFound, map[string]string{"error": "No records found"})
|
||||
}
|
||||
|
||||
// Return the results
|
||||
return c.JSON(http.StatusOK, results)
|
||||
})
|
||||
|
||||
e.Router.GET("/api/custom/apicall", func(c echo.Context) error {
|
||||
// Get kpi_id from query parameters
|
||||
dataset := c.QueryParam("dataset")
|
||||
if dataset == "" {
|
||||
return c.JSON(http.StatusBadRequest, map[string]string{"error": "dataset is required"})
|
||||
}
|
||||
apicall := c.QueryParam("apicall")
|
||||
// Convert apicall string to a boolean
|
||||
saveToFile := false // Default value
|
||||
if apicall == "true" {
|
||||
saveToFile = true
|
||||
}
|
||||
|
||||
// Fetch all matching records manually
|
||||
// Construct the dbx.Expression
|
||||
filter := dbx.NewExp("dataset = {:dataset}", dbx.Params{"dataset": dataset})
|
||||
records, err := app.Dao().FindRecordsByExpr("charts", filter)
|
||||
if err != nil {
|
||||
log.Printf("Failed to fetch records for dataset '%s': %v", dataset, err)
|
||||
return c.JSON(http.StatusInternalServerError, map[string]string{"error": "Failed to fetch records"})
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
return c.JSON(http.StatusNotFound, map[string]string{"error": "No records found for the given dataset"})
|
||||
}
|
||||
|
||||
var aggregatedResponse []map[string]interface{}
|
||||
for _, record := range records {
|
||||
// Extract the URL for each record
|
||||
apiURL := record.GetString("url")
|
||||
kpi := record.GetString("kpi")
|
||||
if apiURL == "" {
|
||||
log.Printf("No URL for record '%v'", record)
|
||||
continue // Skip records with no URL
|
||||
}
|
||||
|
||||
// Fetch and convert XML to JSON
|
||||
_, err := fetchAndConvertXMLToJSON(apiURL, kpi, saveToFile)
|
||||
if err != nil {
|
||||
log.Printf("Failed to process URL '%s': %v", apiURL, err)
|
||||
continue // Skip this record and proceed with others
|
||||
}
|
||||
|
||||
// Read the JSON data from the file
|
||||
response, err := readJSONFromFile(kpi)
|
||||
if err != nil {
|
||||
log.Printf("Failed to read JSON for KPI '%s': %v", kpi, err)
|
||||
continue // Skip this record and proceed with others
|
||||
}
|
||||
|
||||
// Prepare the structure with URL and its response
|
||||
recordResult := map[string]interface{}{
|
||||
"url": apiURL,
|
||||
"dataset": record.GetString("dataset"),
|
||||
"main_id": record.GetString("main_id"),
|
||||
"sub_id": record.GetString("sub_id"),
|
||||
"kpi": record.GetString("kpi"),
|
||||
"is_chart": record.GetString("is_chart"),
|
||||
"chart_type": record.GetString("chart_type"),
|
||||
"response": response,
|
||||
}
|
||||
|
||||
// Add the result to the aggregated response
|
||||
aggregatedResponse = append(aggregatedResponse, recordResult)
|
||||
|
||||
}
|
||||
|
||||
if len(aggregatedResponse) == 0 {
|
||||
return c.JSON(http.StatusInternalServerError, map[string]string{"error": "No data could be fetched from the provided URLs"})
|
||||
}
|
||||
|
||||
return c.JSON(http.StatusOK, aggregatedResponse)
|
||||
})
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
// Start your custom HTTP server on port 8091
|
||||
log.Println("Starting custom HTTP server on :8091")
|
||||
if err := http.ListenAndServe(":8091", nil); err != nil {
|
||||
|
||||
BIN
pb_data/data.db
BIN
pb_data/data.db
Binary file not shown.
BIN
pb_data/logs.db
BIN
pb_data/logs.db
Binary file not shown.
Loading…
Reference in New Issue
Block a user