Validate data before commit

datasources/kmb/query.go

@@ -1,8 +1,9 @@
 package kmb
 
 import (
-    "strings"
 	query "github.com/tgckpg/golifehk/query"
+	"log"
+	"strings"
 )
 
 func Query(lang string, message string) query.IQueryResult {
@@ -43,6 +44,9 @@ func Query( lang string, message string ) query.IQueryResult {
 	}
 
 qrReturn:
+	if qr.Error != nil {
+		log.Println(qr.Error)
+	}
 	var iqr query.IQueryResult
 	iqr = &qr
 	return iqr

datasources/kmb/routestops.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"path/filepath"
 
 	query "github.com/tgckpg/golifehk/query"
@@ -99,33 +98,46 @@ func readBusStopsData(buff *bytes.Buffer) (*map[string]*BusStop, error) {
 }
 
 func getRouteStops() (*[]query.ISearchable, error) {
+	var busStopMap *map[string]*BusStop
 
 	QUERY_FUNC := func() (io.ReadCloser, error) {
 		return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/stop")
 	}
 
-	buff, err := utils.CacheStream(JSON_BUSSTOPS, QUERY_FUNC, 7*24*3600)
+	PARSE_FUNC := func(buff *bytes.Buffer) error {
+		var err error
+		busStopMap, err = readBusStopsData(buff)
+		return err
+	}
+
+	cs, err := utils.CacheStreamEx(JSON_BUSSTOPS, QUERY_FUNC)
 	if err != nil {
 		return nil, err
 	}
 
-	busStopMap, err := readBusStopsData(buff)
+	err = cs.Try(PARSE_FUNC, 7*24*3600, 3)
 	if err != nil {
 		return nil, err
 	}
 
+	var routeStops *[]*RouteStop
 	QUERY_FUNC = func() (io.ReadCloser, error) {
 		return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/route-stop")
 	}
 
-	buff, err = utils.CacheStream(JSON_ROUTESTOPS, QUERY_FUNC, 7*24*3600)
+	PARSE_FUNC = func(buff *bytes.Buffer) error {
+		var err error
+		routeStops, err = readRouteStopsData(busStopMap, buff)
+		return err
+	}
+
+	cs, err = utils.CacheStreamEx(JSON_ROUTESTOPS, QUERY_FUNC)
 	if err != nil {
 		return nil, err
 	}
 
-	routeStops, err := readRouteStopsData(busStopMap, buff)
+	err = cs.Try(PARSE_FUNC, 7*24*3600, 3)
 	if err != nil {
-		log.Printf("Unable to parse RouteStopsData: %s", err)
 		return nil, err
 	}
 

utils/cache.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"log"
 	"os"
@@ -30,6 +31,35 @@ func ( cs *CacheStreams ) Commit() error {
 	return err
 }
 
+func (cs *CacheStreams) Try(Parser func(*bytes.Buffer) error, expires time.Duration, retries int) error {
+	var err error
+	for i := 0; i < retries; i++ {
+		if cs.NotExpired(expires) {
+			err = Parser(cs.Local)
+			if err == nil {
+				log.Printf("Using cache: %s", cs.Path)
+				return nil
+			}
+		}
+
+		err = cs.Reload()
+		log.Printf("Reloading (%d): %s", i+1, cs.Path)
+		if err != nil {
+			log.Printf("Error retrieving data(%s): %s", i+1, err)
+		}
+
+		if cs.Remote != nil {
+			err = Parser(cs.Remote)
+			if err == nil {
+				cs.Commit()
+				return nil
+			}
+		}
+	}
+
+	return fmt.Errorf("Failed to get data for: %s", cs.Path)
+}
+
 func (cs *CacheStreams) Reload() error {
 
 	s, err := cs.Query()
@@ -76,7 +106,6 @@ func CacheStreamEx( path string, readStream func() ( io.ReadCloser, error ) ) (
 	return &cs, nil
 }
 
-
 func CacheStream(path string, readStream func() (io.ReadCloser, error), expires time.Duration) (*bytes.Buffer, error) {
 
 	cache, err := os.Stat(path)
@@ -100,7 +129,8 @@ func CacheStream( path string, readStream func() ( io.ReadCloser, error ), expir
 
 	err = os.MkdirAll(filepath.Dir(path), 0750)
 	if err != nil {
-        return nil, err }
+		return nil, err
+	}
 
 	writeBuff := bytes.NewBuffer([]byte{})