penguin/golifehk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Validate data before commit

Browse Source
This commit is contained in:
斟酌 鵬兄
2026-03-01 19:22:15 +08:00
parent 6245ef6290
commit 8bd60a58bb
3 changed files with 231 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
datasources/kmb/query.go
View File

@@ -1,49 +1,53 @@
package kmb package kmb
import ( import (
"strings" query "github.com/tgckpg/golifehk/query"
query "github.com/tgckpg/golifehk/query" "log"
"strings"
) )
func Query( lang string, message string ) query.IQueryResult { func Query(lang string, message string) query.IQueryResult {
var qo *query.QueryObject var qo *query.QueryObject
var err error var err error
qr := QueryResult{ Lang: lang } qr := QueryResult{Lang: lang}
routeStops, err := getRouteStops() routeStops, err := getRouteStops()
if err != nil { if err != nil {
qr.Error = err qr.Error = err
goto qrReturn goto qrReturn
} }
qo, err = query.Parse( strings.ToUpper( message ), routeStops ) qo, err = query.Parse(strings.ToUpper(message), routeStops)
if err != nil { if err != nil {
qr.Error = err qr.Error = err
goto qrReturn goto qrReturn
} }
qr.Query = qo qr.Query = qo
if 0 < len( *qo.Results ) && 1 < len( *qo.SearchTerms ) { if 0 < len(*qo.Results) && 1 < len(*qo.SearchTerms) {
rSchedules := map[*RouteStop] *[] *Schedule{} rSchedules := map[*RouteStop]*[]*Schedule{}
for _, item := range *qo.Results { for _, item := range *qo.Results {
var r *RouteStop var r *RouteStop
r = any( item ).( *RouteStop ) r = any(item).(*RouteStop)
schedules, err := getSchedule( r ) schedules, err := getSchedule(r)
if err != nil { if err != nil {
qr.Error = err qr.Error = err
break break
} }
rSchedules[r] = schedules rSchedules[r] = schedules
} }
qr.Schedules = &rSchedules qr.Schedules = &rSchedules
} }
qrReturn: qrReturn:
var iqr query.IQueryResult if qr.Error != nil {
iqr = &qr log.Println(qr.Error)
return iqr }
var iqr query.IQueryResult
iqr = &qr
return iqr
} }

24
datasources/kmb/routestops.go
View File

@@ -5,7 +5,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"log"
"path/filepath" "path/filepath"
query "github.com/tgckpg/golifehk/query" query "github.com/tgckpg/golifehk/query"
@@ -99,33 +98,46 @@ func readBusStopsData(buff *bytes.Buffer) (*map[string]*BusStop, error) {
} }
func getRouteStops() (*[]query.ISearchable, error) { func getRouteStops() (*[]query.ISearchable, error) {
var busStopMap *map[string]*BusStop
QUERY_FUNC := func() (io.ReadCloser, error) { QUERY_FUNC := func() (io.ReadCloser, error) {
return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/stop") return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/stop")
} }
buff, err := utils.CacheStream(JSON_BUSSTOPS, QUERY_FUNC, 7*24*3600) PARSE_FUNC := func(buff *bytes.Buffer) error {
var err error
busStopMap, err = readBusStopsData(buff)
return err
}
cs, err := utils.CacheStreamEx(JSON_BUSSTOPS, QUERY_FUNC)
if err != nil { if err != nil {
return nil, err return nil, err
} }
busStopMap, err := readBusStopsData(buff) err = cs.Try(PARSE_FUNC, 7*24*3600, 3)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var routeStops *[]*RouteStop
QUERY_FUNC = func() (io.ReadCloser, error) { QUERY_FUNC = func() (io.ReadCloser, error) {
return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/route-stop") return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/route-stop")
} }
buff, err = utils.CacheStream(JSON_ROUTESTOPS, QUERY_FUNC, 7*24*3600) PARSE_FUNC = func(buff *bytes.Buffer) error {
var err error
routeStops, err = readRouteStopsData(busStopMap, buff)
return err
}
cs, err = utils.CacheStreamEx(JSON_ROUTESTOPS, QUERY_FUNC)
if err != nil { if err != nil {
return nil, err return nil, err
} }
routeStops, err := readRouteStopsData(busStopMap, buff) err = cs.Try(PARSE_FUNC, 7*24*3600, 3)
if err != nil { if err != nil {
log.Printf("Unable to parse RouteStopsData: %s", err)
return nil, err return nil, err
} }

318
utils/cache.go
View File

@@ -1,188 +1,218 @@
package utils package utils
import ( import (
"bytes" "bytes"
"io" "fmt"
"log" "io"
"os" "log"
"path/filepath" "os"
"time" "path/filepath"
"time"
) )
type CacheStreams struct { type CacheStreams struct {
Local *bytes.Buffer Local *bytes.Buffer
Remote *bytes.Buffer Remote *bytes.Buffer
Path string Path string
Query func() ( io.ReadCloser, error ) Query func() (io.ReadCloser, error)
} }
func ( cs *CacheStreams ) Commit() error { func (cs *CacheStreams) Commit() error {
f, err := os.OpenFile( cs.Path, os.O_CREATE | os.O_WRONLY | os.O_TRUNC, 0644 ) f, err := os.OpenFile(cs.Path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil { if err != nil {
return err return err
} }
defer f.Close() defer f.Close()
_, err = io.Copy( f, bytes.NewReader( cs.Remote.Bytes() ) ) _, err = io.Copy(f, bytes.NewReader(cs.Remote.Bytes()))
log.Printf( "Commit: %s", cs.Path ) log.Printf("Commit: %s", cs.Path)
return err return err
} }
func ( cs *CacheStreams ) Reload() error { func (cs *CacheStreams) Try(Parser func(*bytes.Buffer) error, expires time.Duration, retries int) error {
var err error
for i := 0; i < retries; i++ {
if cs.NotExpired(expires) {
err = Parser(cs.Local)
if err == nil {
log.Printf("Using cache: %s", cs.Path)
return nil
}
}
s, err := cs.Query() err = cs.Reload()
if err != nil { log.Printf("Reloading (%d): %s", i+1, cs.Path)
return err if err != nil {
} log.Printf("Error retrieving data(%s): %s", i+1, err)
}
defer s.Close() if cs.Remote != nil {
err = Parser(cs.Remote)
if err == nil {
cs.Commit()
return nil
}
}
}
cs.Remote = bytes.NewBuffer( []byte{} ) return fmt.Errorf("Failed to get data for: %s", cs.Path)
_, err = io.Copy( cs.Remote, s )
return err
} }
func ( cs *CacheStreams ) NotExpired( expires time.Duration ) bool { func (cs *CacheStreams) Reload() error {
cache, err := os.Stat( cs.Path ) s, err := cs.Query()
if err != nil {
return err
}
if err == nil { defer s.Close()
expired := cache.ModTime().Add( expires * 1e9 )
return time.Now().Before( expired )
}
return false cs.Remote = bytes.NewBuffer([]byte{})
_, err = io.Copy(cs.Remote, s)
return err
} }
func CacheStreamEx( path string, readStream func() ( io.ReadCloser, error ) ) ( *CacheStreams, error ) { func (cs *CacheStreams) NotExpired(expires time.Duration) bool {
cs := CacheStreams{} cache, err := os.Stat(cs.Path)
cs.Path = path
cs.Query = readStream
f, err := os.Open( path ) if err == nil {
if err == nil { expired := cache.ModTime().Add(expires * 1e9)
defer f.Close() return time.Now().Before(expired)
cs.Local = bytes.NewBuffer( []byte{} ) }
_, err = io.Copy( cs.Local, f )
if err != nil {
return nil, err
}
}
return &cs, nil return false
} }
func CacheStreamEx(path string, readStream func() (io.ReadCloser, error)) (*CacheStreams, error) {
func CacheStream( path string, readStream func() ( io.ReadCloser, error ), expires time.Duration ) ( *bytes.Buffer, error ) { cs := CacheStreams{}
cs.Path = path
cs.Query = readStream
cache, err := os.Stat( path ) f, err := os.Open(path)
if err == nil {
defer f.Close()
cs.Local = bytes.NewBuffer([]byte{})
_, err = io.Copy(cs.Local, f)
if err != nil {
return nil, err
}
}
// Check if cache exists and not expired return &cs, nil
if err == nil {
expired := cache.ModTime().Add( expires * 1e9 )
if time.Now().Before( expired ) {
f, err := os.Open( path )
if err == nil {
defer f.Close()
log.Printf( "Using cache: %s", path )
writeBuff := bytes.NewBuffer( []byte{} )
_, err = io.Copy( writeBuff, f )
if err == nil {
return writeBuff, nil
}
}
}
}
err = os.MkdirAll( filepath.Dir( path ), 0750 )
if err != nil {
return nil, err }
writeBuff := bytes.NewBuffer( []byte{} )
// Get the reader that return new data
s, err := readStream()
if err != nil {
return nil, err
}
defer s.Close()
_, err = io.Copy( writeBuff, s )
if err != nil {
return nil, err
}
f, err := os.OpenFile( path, os.O_CREATE | os.O_WRONLY | os.O_TRUNC, 0644 )
if err != nil {
return nil, err
}
defer f.Close()
data := writeBuff.Bytes()
_, err = io.Copy( f, bytes.NewReader( data ) )
if err != nil {
return nil, err
}
return writeBuff, nil
} }
func ChangedStream( path string, readStream func() ( io.Reader, error ), dataModTime time.Time ) ( *bytes.Buffer, error ) { func CacheStream(path string, readStream func() (io.ReadCloser, error), expires time.Duration) (*bytes.Buffer, error) {
cache, err := os.Stat( path ) cache, err := os.Stat(path)
// Check if cache exists and not expired // Check if cache exists and not expired
if err == nil { if err == nil {
if dataModTime.Before( cache.ModTime() ) { expired := cache.ModTime().Add(expires * 1e9)
f, err := os.Open( path ) if time.Now().Before(expired) {
if err == nil { f, err := os.Open(path)
defer f.Close() if err == nil {
log.Printf( "Reading from file: %s", path ) defer f.Close()
writeBuff := bytes.NewBuffer( []byte{} ) log.Printf("Using cache: %s", path)
_, err = io.Copy( writeBuff, f ) writeBuff := bytes.NewBuffer([]byte{})
if err == nil { _, err = io.Copy(writeBuff, f)
return writeBuff, nil if err == nil {
} return writeBuff, nil
} }
} }
} }
}
err = os.MkdirAll( filepath.Dir( path ), 0750 ) err = os.MkdirAll(filepath.Dir(path), 0750)
if err != nil { if err != nil {
return nil, err return nil, err
} }
writeBuff := bytes.NewBuffer( []byte{} ) writeBuff := bytes.NewBuffer([]byte{})
// Get the reader that return new data // Get the reader that return new data
s, err := readStream() s, err := readStream()
if err != nil { if err != nil {
return nil, err return nil, err
} }
_, err = io.Copy( writeBuff, s ) defer s.Close()
if err != nil {
return nil, err
}
f, err := os.OpenFile( path, os.O_CREATE | os.O_WRONLY | os.O_TRUNC, 0644 ) _, err = io.Copy(writeBuff, s)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer f.Close() f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, err
}
data := writeBuff.Bytes() defer f.Close()
_, err = io.Copy( f, bytes.NewReader( data ) )
if err != nil {
return nil, err
}
return writeBuff, nil data := writeBuff.Bytes()
_, err = io.Copy(f, bytes.NewReader(data))
if err != nil {
return nil, err
}
return writeBuff, nil
}
func ChangedStream(path string, readStream func() (io.Reader, error), dataModTime time.Time) (*bytes.Buffer, error) {
cache, err := os.Stat(path)
// Check if cache exists and not expired
if err == nil {
if dataModTime.Before(cache.ModTime()) {
f, err := os.Open(path)
if err == nil {
defer f.Close()
log.Printf("Reading from file: %s", path)
writeBuff := bytes.NewBuffer([]byte{})
_, err = io.Copy(writeBuff, f)
if err == nil {
return writeBuff, nil
}
}
}
}
err = os.MkdirAll(filepath.Dir(path), 0750)
if err != nil {
return nil, err
}
writeBuff := bytes.NewBuffer([]byte{})
// Get the reader that return new data
s, err := readStream()
if err != nil {
return nil, err
}
_, err = io.Copy(writeBuff, s)
if err != nil {
return nil, err
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, err
}
defer f.Close()
data := writeBuff.Bytes()
_, err = io.Copy(f, bytes.NewReader(data))
if err != nil {
return nil, err
}
return writeBuff, nil
} }