Generalize cache get with expire param

.gitignore

@@ -1,2 +1,3 @@
 *.csv
 *.json
+*.swp

businfo/mtr/busstops_test.go

@@ -1,22 +0,0 @@
-package mtr
-
-import (
-    "fmt"
-	"testing"
-)
-
-func TestAll(t *testing.T) {
-    /*
-    entries, err := pullRemote()
-    if err != nil {
-        t.Error( err )
-    }
-    fmt.Println( entries )
-    */
-
-    entries, err := pullLocal()
-    if err != nil {
-        t.Error( err )
-    }
-    fmt.Println( entries )
-}

datasources/mtr/bus/busschedule.go

@@ -1,12 +1,13 @@
-package mtr
+package bus
 
 import (
-    "fmt"
+    "bytes"
-    // "byte"
-    // "net/http"
-    "io"
-    "os"
     "encoding/json"
+    "io"
+    "net/http"
+    "path/filepath"
+
+    "github.com/tgckpg/golifehk/utils"
 )
 
 type Location struct {
@@ -38,41 +39,38 @@ type BusSchedule struct {
 
 func getSchedule( lang string, routeName string ) ( *BusSchedule, error ) {
 
-    /*
+    CACHE_PATH := filepath.Join(
-    values := map[string]string { "language": lang , "routeName": routeName }
+        utils.WORKDIR,
-
+        "mtr_bsch" + "-" + lang + "-" + routeName + ".json",
-    jsonValue, _ := json.Marshal(values)
-
-    resp, err := http.Post(
-        "https://rt.data.gov.hk/v1/transport/mtr/bus/getSchedule",
-        "application/json",
-        bytes.NewBuffer( jsonValue ),
     )
 
-    if err != nil {
+    QUERY_FUNC := func() ( io.ReadCloser, error ) {
-        return nil, err
+        // Query Remote
+        values := map[string]string { "language": lang , "routeName": routeName }
+        jsonValue, _ := json.Marshal(values)
+        resp, err := http.Post(
+            "https://rt.data.gov.hk/v1/transport/mtr/bus/getSchedule",
+            "application/json",
+            bytes.NewBuffer( jsonValue ),
+        )
+
+        if err != nil {
+            return nil, err
+        }
+
+        return resp.Body, nil
     }
 
-    defer resp.Body.Close()
+    buff, err := utils.CacheStream( CACHE_PATH, QUERY_FUNC, 60 )
-
-    data, err := io.ReadAll( resp.Body )
-    /*/
-    f, err := os.Open( "abc.json" )
-    defer f.Close()
-
-    data, err := io.ReadAll( f )
-    //*/
     if err != nil {
         return nil, err
     }
 
     schedules := BusSchedule{}
-
+    err = json.Unmarshal( buff.Bytes(), &schedules )
-    err = json.Unmarshal( data, &schedules )
     if err != nil {
         return nil, err
     }
 
-    fmt.Printf( "%+v", schedules )
     return &schedules, nil
 }

datasources/mtr/bus/busschedule_test.go

@@ -1,4 +1,4 @@
-package mtr
+package bus
 
 import (
 	"testing"

datasources/mtr/bus/busstops.go

@@ -1,11 +1,10 @@
-package mtr
+package bus
 
 import (
     "encoding/csv"
     "fmt"
     "io"
     "net/http"
-    "os"
     "path/filepath"
     "strconv"
     "strings"
@@ -26,7 +25,7 @@ type BusStop struct {
 
 var mBusStops *map[string]BusStop
 
-var DEF_CSV string = filepath.Join( utils.WORKDIR, "mtr_bus_stops.csv" )
+var CSV_BUSSTOPS string = filepath.Join( utils.WORKDIR, "mtr_bus_stops.csv" )
 
 func readBusStopData( r io.Reader ) ( *map[string]BusStop, error ) {
 
@@ -82,60 +81,18 @@ func readBusStopData( r io.Reader ) ( *map[string]BusStop, error ) {
 
 func getBusStops() (*map[string]BusStop, error) {
 
-    if mBusStops != nil {
+    QUERY_FUNC := func() ( io.ReadCloser, error ) {
-        return mBusStops, nil
+        resp, err := http.Get( "https://opendata.mtr.com.hk/data/mtr_bus_stops.csv" )
+        if err != nil {
+            return nil, err
+        }
+        return resp.Body, nil
     }
 
-    mBusStops, err := pullLocal()
+    buff, err := utils.CacheStream( CSV_BUSSTOPS, QUERY_FUNC, 7 * 24 * 3600 )
-    if mBusStops != nil {
+    if err != nil {
-        return mBusStops, err
+        return nil, err
     }
 
-    mBusStops, err = pullRemote()
+    return readBusStopData( buff )
-    if mBusStops != nil {
-        return mBusStops, err
-    }
-
-    return nil, err
-}
-
-func pullRemote() (*map[string]BusStop, error) {
-
-    err := os.MkdirAll( filepath.Dir( DEF_CSV ), 0750 )
-    if err != nil {
-        return nil, err
-    }
-
-    resp, err := http.Get( "https://opendata.mtr.com.hk/data/mtr_bus_stops.csv" )
-    if err != nil {
-        return nil, err
-    }
-
-    defer resp.Body.Close()
-
-    f, err := os.OpenFile( DEF_CSV, os.O_CREATE | os.O_RDWR, 0644 )
-    if err != nil {
-        return nil, err
-    }
-
-    defer f.Close()
-
-    _, err = io.Copy( f, resp.Body )
-    if err != nil {
-        return nil, err
-    }
-
-    f.Seek( 0, 0 )
-    return readBusStopData( f )
-}
-
-func pullLocal() (*map[string]BusStop, error) {
-    f, err := os.Open( DEF_CSV )
-    if err != nil {
-        return nil, err
-    }
-
-    defer f.Close()
-
-    return readBusStopData( f )
 }

datasources/mtr/bus/busstops_test.go

@@ -0,0 +1,8 @@
+package bus
+
+import (
+	"testing"
+)
+
+func TestAll(t *testing.T) {
+}

datasources/mtr/bus/query.go

@@ -1,4 +1,4 @@
-package mtr
+package bus
 
 import (
     "fmt"
@@ -8,7 +8,12 @@ import (
 
 type QueryObject struct {
     Route string
+    BusStops *[]BusStop
+}
+
+type QueryResult struct {
     BusStops []BusStop
+    err string
 }
 
 type Term struct {
@@ -16,6 +21,10 @@ type Term struct {
     ProblyRoute bool
 }
 
+func Query( message string ) *QueryResult {
+    return &QueryResult{ err: "No Result" }
+}
+
 func test( entry BusStop, val string ) bool {
     switch true {
     case strings.Contains( entry.Name_zhant, val ):
@@ -37,6 +46,7 @@ func parse( line string ) ( *QueryObject, error ) {
     var searches = []string{}
     matches := []BusStop{}
 
+    // Sanitize and assume properties for each of the keywords
     terms := []Term{}
     for _, val := range strings.Split( line, " " ) {
         val = strings.ToUpper( strings.Trim( val, " " ) )
@@ -47,6 +57,7 @@ func parse( line string ) ( *QueryObject, error ) {
         terms = append( terms, term )
     }
 
+    // Search for route name first, otherwise search in other props
     for _, entry := range *busStops {
 
         // Search for RouteId
@@ -69,6 +80,8 @@ func parse( line string ) ( *QueryObject, error ) {
         }
     }
 
+    // If route found, filter out all other route
+    // then search within that route
     if route != "" && 0 < len( searches ) {
         matches_in := []BusStop{}
         for _, entry := range matches {
@@ -86,6 +99,5 @@ func parse( line string ) ( *QueryObject, error ) {
         matches = matches_in
     }
 
-
+    return &QueryObject{ Route: route, BusStops: &matches }, err
-    return &QueryObject{ Route: route, BusStops: matches }, err
 }

datasources/mtr/bus/query_test.go

@@ -1,4 +1,4 @@
-package mtr
+package bus
 
 import (
     "fmt"
@@ -11,6 +11,6 @@ func TestQuerySchedule(t *testing.T) {
         t.Error( err )
     }
 
-    fmt.Println( qo )
+    fmt.Printf( "%+v", qo )
 }
 

utils/cache.go

@@ -0,0 +1,66 @@
+package utils
+
+import (
+    "bytes"
+    "io"
+    "os"
+    "path/filepath"
+    "time"
+)
+
+
+func CacheStream( path string, readStream func() ( io.ReadCloser, error ), expires int ) ( *bytes.Buffer, error ) {
+
+    cache, err := os.Stat( path )
+
+    // Check if cache exists and not expired
+    if err == nil {
+        expired := cache.ModTime().Add( 60 * 1e9 )
+        if time.Now().Before( expired ) {
+            f, err := os.Open( path )
+            if err == nil {
+                defer f.Close()
+                writeBuff := bytes.NewBuffer( []byte{} )
+                _, err = io.Copy( writeBuff, f )
+                if err == nil {
+                    return writeBuff, nil
+                }
+            }
+        }
+    }
+
+    err = os.MkdirAll( filepath.Dir( path ), 0750 )
+    if err != nil {
+        return nil, err
+    }
+
+    writeBuff := bytes.NewBuffer( []byte{} )
+
+    // Get the reader that return new data
+    s, err := readStream()
+    if err != nil {
+        return nil, err
+    }
+
+    defer s.Close()
+
+    _, err = io.Copy( writeBuff, s )
+    if err != nil {
+        return nil, err
+    }
+
+    f, err := os.OpenFile( path, os.O_CREATE | os.O_WRONLY | os.O_TRUNC, 0644 )
+    if err != nil {
+        return nil, err
+    }
+
+    defer f.Close()
+
+    data := writeBuff.Bytes()
+    _, err = io.Copy( f, bytes.NewReader( data ) )
+    if err != nil {
+        return nil, err
+    }
+
+    return writeBuff, nil
+}