penguin/golifehk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Validate data before commit

Browse Source
This commit is contained in:
斟酌 鵬兄
2026-03-01 19:22:15 +08:00
parent 6245ef6290
commit 8bd60a58bb
3 changed files with 231 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
datasources/kmb/query.go
View File

@@ -1,23 +1,24 @@
package kmb
import (
"strings"
query "github.com/tgckpg/golifehk/query"
"log"
"strings"
)
func Query( lang string, message string ) query.IQueryResult {
func Query(lang string, message string) query.IQueryResult {
var qo *query.QueryObject
var err error
qr := QueryResult{ Lang: lang }
qr := QueryResult{Lang: lang}
routeStops, err := getRouteStops()
if err != nil {
qr.Error = err
goto qrReturn
}
qo, err = query.Parse( strings.ToUpper( message ), routeStops )
qo, err = query.Parse(strings.ToUpper(message), routeStops)
if err != nil {
qr.Error = err
goto qrReturn
@@ -25,13 +26,13 @@ func Query( lang string, message string ) query.IQueryResult {
qr.Query = qo
if 0 < len( *qo.Results ) && 1 < len( *qo.SearchTerms ) {
if 0 < len(*qo.Results) && 1 < len(*qo.SearchTerms) {
rSchedules := map[*RouteStop] *[] *Schedule{}
rSchedules := map[*RouteStop]*[]*Schedule{}
for _, item := range *qo.Results {
var r *RouteStop
r = any( item ).( *RouteStop )
schedules, err := getSchedule( r )
r = any(item).(*RouteStop)
schedules, err := getSchedule(r)
if err != nil {
qr.Error = err
break
@@ -42,7 +43,10 @@ func Query( lang string, message string ) query.IQueryResult {
qr.Schedules = &rSchedules
}
qrReturn:
qrReturn:
if qr.Error != nil {
log.Println(qr.Error)
}
var iqr query.IQueryResult
iqr = &qr
return iqr

24
datasources/kmb/routestops.go
View File

@@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"path/filepath"
query "github.com/tgckpg/golifehk/query"
@@ -99,33 +98,46 @@ func readBusStopsData(buff *bytes.Buffer) (*map[string]*BusStop, error) {
}
func getRouteStops() (*[]query.ISearchable, error) {
var busStopMap *map[string]*BusStop
QUERY_FUNC := func() (io.ReadCloser, error) {
return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/stop")
}
buff, err := utils.CacheStream(JSON_BUSSTOPS, QUERY_FUNC, 7*24*3600)
PARSE_FUNC := func(buff *bytes.Buffer) error {
var err error
busStopMap, err = readBusStopsData(buff)
return err
}
cs, err := utils.CacheStreamEx(JSON_BUSSTOPS, QUERY_FUNC)
if err != nil {
return nil, err
}
busStopMap, err := readBusStopsData(buff)
err = cs.Try(PARSE_FUNC, 7*24*3600, 3)
if err != nil {
return nil, err
}
var routeStops *[]*RouteStop
QUERY_FUNC = func() (io.ReadCloser, error) {
return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/route-stop")
}
buff, err = utils.CacheStream(JSON_ROUTESTOPS, QUERY_FUNC, 7*24*3600)
PARSE_FUNC = func(buff *bytes.Buffer) error {
var err error
routeStops, err = readRouteStopsData(busStopMap, buff)
return err
}
cs, err = utils.CacheStreamEx(JSON_ROUTESTOPS, QUERY_FUNC)
if err != nil {
return nil, err
}
routeStops, err := readRouteStopsData(busStopMap, buff)
err = cs.Try(PARSE_FUNC, 7*24*3600, 3)
if err != nil {
log.Printf("Unable to parse RouteStopsData: %s", err)
return nil, err
}

116
utils/cache.go
View File

@@ -2,6 +2,7 @@ package utils
import (
"bytes"
"fmt"
"io"
"log"
"os"
@@ -13,24 +14,53 @@ type CacheStreams struct {
Local *bytes.Buffer
Remote *bytes.Buffer
Path string
Query func() ( io.ReadCloser, error )
Query func() (io.ReadCloser, error)
}
func ( cs *CacheStreams ) Commit() error {
func (cs *CacheStreams) Commit() error {
f, err := os.OpenFile( cs.Path, os.O_CREATE | os.O_WRONLY | os.O_TRUNC, 0644 )
f, err := os.OpenFile(cs.Path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy( f, bytes.NewReader( cs.Remote.Bytes() ) )
log.Printf( "Commit: %s", cs.Path )
_, err = io.Copy(f, bytes.NewReader(cs.Remote.Bytes()))
log.Printf("Commit: %s", cs.Path)
return err
}
func ( cs *CacheStreams ) Reload() error {
func (cs *CacheStreams) Try(Parser func(*bytes.Buffer) error, expires time.Duration, retries int) error {
var err error
for i := 0; i < retries; i++ {
if cs.NotExpired(expires) {
err = Parser(cs.Local)
if err == nil {
log.Printf("Using cache: %s", cs.Path)
return nil
}
}
err = cs.Reload()
log.Printf("Reloading (%d): %s", i+1, cs.Path)
if err != nil {
log.Printf("Error retrieving data(%s): %s", i+1, err)
}
if cs.Remote != nil {
err = Parser(cs.Remote)
if err == nil {
cs.Commit()
return nil
}
}
}
return fmt.Errorf("Failed to get data for: %s", cs.Path)
}
func (cs *CacheStreams) Reload() error {
s, err := cs.Query()
if err != nil {
@@ -39,35 +69,35 @@ func ( cs *CacheStreams ) Reload() error {
defer s.Close()
cs.Remote = bytes.NewBuffer( []byte{} )
_, err = io.Copy( cs.Remote, s )
cs.Remote = bytes.NewBuffer([]byte{})
_, err = io.Copy(cs.Remote, s)
return err
}
func ( cs *CacheStreams ) NotExpired( expires time.Duration ) bool {
func (cs *CacheStreams) NotExpired(expires time.Duration) bool {
cache, err := os.Stat( cs.Path )
cache, err := os.Stat(cs.Path)
if err == nil {
expired := cache.ModTime().Add( expires * 1e9 )
return time.Now().Before( expired )
expired := cache.ModTime().Add(expires * 1e9)
return time.Now().Before(expired)
}
return false
}
func CacheStreamEx( path string, readStream func() ( io.ReadCloser, error ) ) ( *CacheStreams, error ) {
func CacheStreamEx(path string, readStream func() (io.ReadCloser, error)) (*CacheStreams, error) {
cs := CacheStreams{}
cs.Path = path
cs.Query = readStream
f, err := os.Open( path )
f, err := os.Open(path)
if err == nil {
defer f.Close()
cs.Local = bytes.NewBuffer( []byte{} )
_, err = io.Copy( cs.Local, f )
cs.Local = bytes.NewBuffer([]byte{})
_, err = io.Copy(cs.Local, f)
if err != nil {
return nil, err
}
@@ -76,21 +106,20 @@ func CacheStreamEx( path string, readStream func() ( io.ReadCloser, error ) ) (
return &cs, nil
}
func CacheStream(path string, readStream func() (io.ReadCloser, error), expires time.Duration) (*bytes.Buffer, error) {
func CacheStream( path string, readStream func() ( io.ReadCloser, error ), expires time.Duration ) ( *bytes.Buffer, error ) {
cache, err := os.Stat( path )
cache, err := os.Stat(path)
// Check if cache exists and not expired
if err == nil {
expired := cache.ModTime().Add( expires * 1e9 )
if time.Now().Before( expired ) {
f, err := os.Open( path )
expired := cache.ModTime().Add(expires * 1e9)
if time.Now().Before(expired) {
f, err := os.Open(path)
if err == nil {
defer f.Close()
log.Printf( "Using cache: %s", path )
writeBuff := bytes.NewBuffer( []byte{} )
_, err = io.Copy( writeBuff, f )
log.Printf("Using cache: %s", path)
writeBuff := bytes.NewBuffer([]byte{})
_, err = io.Copy(writeBuff, f)
if err == nil {
return writeBuff, nil
}
@@ -98,11 +127,12 @@ func CacheStream( path string, readStream func() ( io.ReadCloser, error ), expir
}
}
err = os.MkdirAll( filepath.Dir( path ), 0750 )
err = os.MkdirAll(filepath.Dir(path), 0750)
if err != nil {
return nil, err }
return nil, err
}
writeBuff := bytes.NewBuffer( []byte{} )
writeBuff := bytes.NewBuffer([]byte{})
// Get the reader that return new data
s, err := readStream()
@@ -112,12 +142,12 @@ func CacheStream( path string, readStream func() ( io.ReadCloser, error ), expir
defer s.Close()
_, err = io.Copy( writeBuff, s )
_, err = io.Copy(writeBuff, s)
if err != nil {
return nil, err
}
f, err := os.OpenFile( path, os.O_CREATE | os.O_WRONLY | os.O_TRUNC, 0644 )
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, err
}
@@ -125,7 +155,7 @@ func CacheStream( path string, readStream func() ( io.ReadCloser, error ), expir
defer f.Close()
data := writeBuff.Bytes()
_, err = io.Copy( f, bytes.NewReader( data ) )
_, err = io.Copy(f, bytes.NewReader(data))
if err != nil {
return nil, err
}
@@ -133,19 +163,19 @@ func CacheStream( path string, readStream func() ( io.ReadCloser, error ), expir
return writeBuff, nil
}
func ChangedStream( path string, readStream func() ( io.Reader, error ), dataModTime time.Time ) ( *bytes.Buffer, error ) {
func ChangedStream(path string, readStream func() (io.Reader, error), dataModTime time.Time) (*bytes.Buffer, error) {
cache, err := os.Stat( path )
cache, err := os.Stat(path)
// Check if cache exists and not expired
if err == nil {
if dataModTime.Before( cache.ModTime() ) {
f, err := os.Open( path )
if dataModTime.Before(cache.ModTime()) {
f, err := os.Open(path)
if err == nil {
defer f.Close()
log.Printf( "Reading from file: %s", path )
writeBuff := bytes.NewBuffer( []byte{} )
_, err = io.Copy( writeBuff, f )
log.Printf("Reading from file: %s", path)
writeBuff := bytes.NewBuffer([]byte{})
_, err = io.Copy(writeBuff, f)
if err == nil {
return writeBuff, nil
}
@@ -153,12 +183,12 @@ func ChangedStream( path string, readStream func() ( io.Reader, error ), dataMod
}
}
err = os.MkdirAll( filepath.Dir( path ), 0750 )
err = os.MkdirAll(filepath.Dir(path), 0750)
if err != nil {
return nil, err
}
writeBuff := bytes.NewBuffer( []byte{} )
writeBuff := bytes.NewBuffer([]byte{})
// Get the reader that return new data
s, err := readStream()
@@ -166,12 +196,12 @@ func ChangedStream( path string, readStream func() ( io.Reader, error ), dataMod
return nil, err
}
_, err = io.Copy( writeBuff, s )
_, err = io.Copy(writeBuff, s)
if err != nil {
return nil, err
}
f, err := os.OpenFile( path, os.O_CREATE | os.O_WRONLY | os.O_TRUNC, 0644 )
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, err
}
@@ -179,7 +209,7 @@ func ChangedStream( path string, readStream func() ( io.Reader, error ), dataMod
defer f.Close()
data := writeBuff.Bytes()
_, err = io.Copy( f, bytes.NewReader( data ) )
_, err = io.Copy(f, bytes.NewReader(data))
if err != nil {
return nil, err
}