diff --git a/main.go b/main.go index 21bf6cc..bda709d 100755 --- a/main.go +++ b/main.go @@ -2749,7 +2749,7 @@ func main() { } data, _ := json.Marshal(loginPayload) - loginReq, _ := http.NewRequest("POST", fmt.Sprintf("%s/api/admins/auth-with-password", "https://pb.venbait.in"), bytes.NewBuffer(data)) + loginReq, _ := http.NewRequest("POST", fmt.Sprintf("%s/api/admins/auth-with-password", "http://127.0.0.1:8090"), bytes.NewBuffer(data)) loginReq.Header.Set("Content-Type", "application/json") loginResp, err := http.DefaultClient.Do(loginReq) @@ -2769,40 +2769,61 @@ func main() { return c.JSON(500, map[string]string{"error": "failed to parse login response"}) } - // 2️⃣ Use token to fetch records - url := fmt.Sprintf("%s/api/collections/%s/records?perPage=500", "https://pb.venbait.in", reqBody.Collection) - fmt.Println("Fetching collection from URL:", url) - - req, _ := http.NewRequest("GET", url, nil) - req.Header.Set("Authorization", loginRes.Token) - fmt.Println("Using token:", loginRes.Token) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - fmt.Println("Error fetching collection:", err) - return c.JSON(500, map[string]string{"error": "failed to fetch collection"}) - } - defer resp.Body.Close() - - body, _ := ioutil.ReadAll(resp.Body) - fmt.Println("HTTP Status Code:", resp.StatusCode) - - // parse JSON - var parsed map[string]interface{} - if err := json.Unmarshal(body, &parsed); err != nil { - fmt.Println("JSON Unmarshal error:", err) - return c.JSON(500, map[string]string{"error": "invalid JSON from PocketBase"}) - } - - items, ok := parsed["items"].([]interface{}) - if !ok { - return c.JSON(500, map[string]string{"error": "no items found"}) - } - + // 2️⃣ Fetch all records with pagination. + // PocketBase (v0.22.x) caps perPage at 500 (tools/search.MaxPerPage); larger values are ignored, + // so a single ?perPage=5000 request only returns the first 500 rows. + const recordsPerPage = 500 records := []map[string]interface{}{} - for _, i := range items { - rec := i.(map[string]interface{}) - records = append(records, rec) + for page := 1; ; page++ { + url := fmt.Sprintf("%s/api/collections/%s/records?perPage=%d&page=%d", "http://127.0.0.1:8090", reqBody.Collection, recordsPerPage, page) + fmt.Println("Fetching collection from URL:", url) + + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Authorization", loginRes.Token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + fmt.Println("Error fetching collection:", err) + return c.JSON(500, map[string]string{"error": "failed to fetch collection"}) + } + + body, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + fmt.Println("HTTP Status Code:", resp.StatusCode, "page", page) + + if resp.StatusCode != 200 { + return c.JSON(500, map[string]string{"error": fmt.Sprintf("fetch collection failed: %s", string(body))}) + } + + var parsed map[string]interface{} + if err := json.Unmarshal(body, &parsed); err != nil { + fmt.Println("JSON Unmarshal error:", err) + return c.JSON(500, map[string]string{"error": "invalid JSON from PocketBase"}) + } + + items, ok := parsed["items"].([]interface{}) + if !ok { + return c.JSON(500, map[string]string{"error": "no items found"}) + } + for _, i := range items { + rec := i.(map[string]interface{}) + records = append(records, rec) + } + + var totalPages int + if tp, ok := parsed["totalPages"].(float64); ok { + totalPages = int(tp) + } + if totalPages > 0 { + if page >= totalPages { + break + } + continue + } + // skipTotal or missing totals: stop after a short / empty page + if len(items) < recordsPerPage { + break + } } syncResult := map[string]interface{}{"synced": []string{}, "errors": []string{}} @@ -2819,10 +2840,10 @@ func main() { "password": reqBody.LivePassword, } data, _ := json.Marshal(livePayload) - liveReq, _ := http.NewRequest("POST", fmt.Sprintf("%s/api/admins/auth-with-password", "https://pocket.fcsc.gov.ae"), bytes.NewBuffer(data)) + liveReq, _ := http.NewRequest("POST", fmt.Sprintf("%s/api/admins/auth-with-password", "https://pb.venbait.in"), bytes.NewBuffer(data)) liveReq.Header.Set("Content-Type", "application/json") liveResp, err := http.DefaultClient.Do(liveReq) - fmt.Println("Live url:", "https://pocket.fcsc.gov.ae") + fmt.Println("Live url:", "https://pb.venbait.in") fmt.Println("Live req:", liveReq) if err != nil { return c.JSON(500, map[string]string{"error": "failed to connect to Live PB"}) @@ -2845,7 +2866,7 @@ func main() { data, _ := json.Marshal(rec) // Try PATCH first - patchURL := fmt.Sprintf("%s/api/collections/%s/records/%s", "https://pocket.fcsc.gov.ae", reqBody.Collection, id) + patchURL := fmt.Sprintf("%s/api/collections/%s/records/%s", "https://pb.venbait.in", reqBody.Collection, id) req, _ := http.NewRequest("PATCH", patchURL, bytes.NewBuffer(data)) req.Header.Set("Authorization", liveLogin.Token) req.Header.Set("Content-Type", "application/json") @@ -2859,7 +2880,7 @@ func main() { if resp.StatusCode == 404 { // Create instead - postURL := fmt.Sprintf("%s/api/collections/%s/records", "https://pocket.fcsc.gov.ae", reqBody.Collection) + postURL := fmt.Sprintf("%s/api/collections/%s/records", "https://pb.venbait.in", reqBody.Collection) req, _ := http.NewRequest("POST", postURL, bytes.NewBuffer(data)) req.Header.Set("Authorization", liveLogin.Token) req.Header.Set("Content-Type", "application/json") diff --git a/pb_data/data.db b/pb_data/data.db index ecce007..4365901 100644 Binary files a/pb_data/data.db and b/pb_data/data.db differ diff --git a/pb_data/logs.db b/pb_data/logs.db index 31ba063..d40b1b1 100644 Binary files a/pb_data/logs.db and b/pb_data/logs.db differ diff --git a/pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/imd_sOGbelx5FS.png b/pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/imd_m87uTzlYiI.png similarity index 100% rename from pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/imd_sOGbelx5FS.png rename to pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/imd_m87uTzlYiI.png diff --git a/pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/imd_sOGbelx5FS.png.attrs b/pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/imd_m87uTzlYiI.png.attrs similarity index 100% rename from pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/imd_sOGbelx5FS.png.attrs rename to pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/imd_m87uTzlYiI.png.attrs diff --git a/pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/thumbs_imd_sOGbelx5FS.png/100x100_imd_sOGbelx5FS.png b/pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/thumbs_imd_m87uTzlYiI.png/100x100_imd_m87uTzlYiI.png similarity index 100% rename from pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/thumbs_imd_sOGbelx5FS.png/100x100_imd_sOGbelx5FS.png rename to pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/thumbs_imd_m87uTzlYiI.png/100x100_imd_m87uTzlYiI.png diff --git a/pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/thumbs_imd_sOGbelx5FS.png/100x100_imd_sOGbelx5FS.png.attrs b/pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/thumbs_imd_m87uTzlYiI.png/100x100_imd_m87uTzlYiI.png.attrs similarity index 100% rename from pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/thumbs_imd_sOGbelx5FS.png/100x100_imd_sOGbelx5FS.png.attrs rename to pb_data/storage/1kmwbfcf9u4gk20/kgkroj0o6rni858/thumbs_imd_m87uTzlYiI.png/100x100_imd_m87uTzlYiI.png.attrs diff --git a/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/imd_2Z9tdwo1lM.png b/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/imd_2Z9tdwo1lM.png deleted file mode 100644 index b0f6df5..0000000 Binary files a/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/imd_2Z9tdwo1lM.png and /dev/null differ diff --git a/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/imd_2Z9tdwo1lM.png.attrs b/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/imd_2Z9tdwo1lM.png.attrs deleted file mode 100644 index 13532c7..0000000 --- a/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/imd_2Z9tdwo1lM.png.attrs +++ /dev/null @@ -1 +0,0 @@ -{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"IMD.png"},"md5":"8bGx6LhXqxPbgVjRi2Mdgw=="} diff --git a/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/thumbs_imd_2Z9tdwo1lM.png/100x100_imd_2Z9tdwo1lM.png b/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/thumbs_imd_2Z9tdwo1lM.png/100x100_imd_2Z9tdwo1lM.png deleted file mode 100644 index 0c454c7..0000000 Binary files a/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/thumbs_imd_2Z9tdwo1lM.png/100x100_imd_2Z9tdwo1lM.png and /dev/null differ diff --git a/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/thumbs_imd_2Z9tdwo1lM.png/100x100_imd_2Z9tdwo1lM.png.attrs b/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/thumbs_imd_2Z9tdwo1lM.png/100x100_imd_2Z9tdwo1lM.png.attrs deleted file mode 100644 index e4b22cc..0000000 --- a/pb_data/storage/1kmwbfcf9u4gk20/udnorihkqgswib7/thumbs_imd_2Z9tdwo1lM.png/100x100_imd_2Z9tdwo1lM.png.attrs +++ /dev/null @@ -1 +0,0 @@ -{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"y/a73YxDdhHm9l0Kw6Pi1Q=="} diff --git a/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/open_data_1nrNd0OGXw.png b/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/open_data_1nrNd0OGXw.png deleted file mode 100644 index 323f5b0..0000000 Binary files a/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/open_data_1nrNd0OGXw.png and /dev/null differ diff --git a/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/open_data_1nrNd0OGXw.png.attrs b/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/open_data_1nrNd0OGXw.png.attrs deleted file mode 100644 index 53010b9..0000000 --- a/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/open_data_1nrNd0OGXw.png.attrs +++ /dev/null @@ -1 +0,0 @@ -{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"Open Data.png"},"md5":"zqS1mfeZDSDnS4lCDYYVFA=="} diff --git a/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/thumbs_open_data_1nrNd0OGXw.png/100x100_open_data_1nrNd0OGXw.png b/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/thumbs_open_data_1nrNd0OGXw.png/100x100_open_data_1nrNd0OGXw.png deleted file mode 100644 index d9e4c22..0000000 Binary files a/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/thumbs_open_data_1nrNd0OGXw.png/100x100_open_data_1nrNd0OGXw.png and /dev/null differ diff --git a/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/thumbs_open_data_1nrNd0OGXw.png/100x100_open_data_1nrNd0OGXw.png.attrs b/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/thumbs_open_data_1nrNd0OGXw.png/100x100_open_data_1nrNd0OGXw.png.attrs deleted file mode 100644 index a9afd73..0000000 --- a/pb_data/storage/andh1m2lzkyu7si/pjh6a9dwrladwv3/thumbs_open_data_1nrNd0OGXw.png/100x100_open_data_1nrNd0OGXw.png.attrs +++ /dev/null @@ -1 +0,0 @@ -{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"BoezhRm3qH1p+B0LKtT2MQ=="} diff --git a/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/imd_Flt4maU1V9.png b/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/imd_Flt4maU1V9.png deleted file mode 100644 index b0f6df5..0000000 Binary files a/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/imd_Flt4maU1V9.png and /dev/null differ diff --git a/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/imd_Flt4maU1V9.png.attrs b/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/imd_Flt4maU1V9.png.attrs deleted file mode 100644 index 13532c7..0000000 --- a/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/imd_Flt4maU1V9.png.attrs +++ /dev/null @@ -1 +0,0 @@ -{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":{"original-filename":"IMD.png"},"md5":"8bGx6LhXqxPbgVjRi2Mdgw=="} diff --git a/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/thumbs_imd_Flt4maU1V9.png/100x100_imd_Flt4maU1V9.png b/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/thumbs_imd_Flt4maU1V9.png/100x100_imd_Flt4maU1V9.png deleted file mode 100644 index 0c454c7..0000000 Binary files a/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/thumbs_imd_Flt4maU1V9.png/100x100_imd_Flt4maU1V9.png and /dev/null differ diff --git a/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/thumbs_imd_Flt4maU1V9.png/100x100_imd_Flt4maU1V9.png.attrs b/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/thumbs_imd_Flt4maU1V9.png/100x100_imd_Flt4maU1V9.png.attrs deleted file mode 100644 index e4b22cc..0000000 --- a/pb_data/storage/andh1m2lzkyu7si/zjoj410rl5nxt5v/thumbs_imd_Flt4maU1V9.png/100x100_imd_Flt4maU1V9.png.attrs +++ /dev/null @@ -1 +0,0 @@ -{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"image/png","user.metadata":null,"md5":"y/a73YxDdhHm9l0Kw6Pi1Q=="}