Mimic browser requests because kmb blocked Go httpGet clients
This commit is contained in:
@@ -1,147 +1,139 @@
|
|||||||
package kmb
|
package kmb
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"log"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
// "strings"
|
|
||||||
|
|
||||||
query "github.com/tgckpg/golifehk/query"
|
query "github.com/tgckpg/golifehk/query"
|
||||||
"github.com/tgckpg/golifehk/utils"
|
"github.com/tgckpg/golifehk/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
var JSON_ROUTESTOPS string = filepath.Join( utils.WORKDIR, "kmb-routestops.json" )
|
var JSON_ROUTESTOPS string = filepath.Join(utils.WORKDIR, "kmb-routestops.json")
|
||||||
var JSON_BUSSTOPS string = filepath.Join( utils.WORKDIR, "kmb-busstops.json" )
|
var JSON_BUSSTOPS string = filepath.Join(utils.WORKDIR, "kmb-busstops.json")
|
||||||
|
|
||||||
func readRouteStopsData( busStops *map[string] *BusStop, buff *bytes.Buffer ) ( *[] *RouteStop, error ) {
|
func readRouteStopsData(busStops *map[string]*BusStop, buff *bytes.Buffer) (*[]*RouteStop, error) {
|
||||||
|
|
||||||
routeStopsData := RouteStops{}
|
routeStopsData := RouteStops{}
|
||||||
err := json.Unmarshal( buff.Bytes(), &routeStopsData )
|
err := json.Unmarshal(buff.Bytes(), &routeStopsData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// routeStops[ Route ][ ServiceType ][ Direction ][ Seq ] = RouteStop
|
// routeStops[ Route ][ ServiceType ][ Direction ][ Seq ] = RouteStop
|
||||||
routeStops := map[string] *map[string] *map[ string ] *map[ int ] *RouteStop{}
|
routeStops := map[string]*map[string]*map[string]*map[int]*RouteStop{}
|
||||||
allRouteStops := [] *RouteStop{}
|
allRouteStops := []*RouteStop{}
|
||||||
|
|
||||||
for _, entry := range routeStopsData.RouteStops {
|
for _, entry := range routeStopsData.RouteStops {
|
||||||
|
|
||||||
busStop := (*busStops)[ entry.StationId ]
|
busStop := (*busStops)[entry.StationId]
|
||||||
if busStop == nil {
|
if busStop == nil {
|
||||||
busStop = &BusStop {
|
busStop = &BusStop{
|
||||||
BusStopId: entry.StationId,
|
BusStopId: entry.StationId,
|
||||||
Name_en: "???", Name_tc: "???", Name_sc: "???",
|
Name_en: "???", Name_tc: "???", Name_sc: "???",
|
||||||
}
|
}
|
||||||
busStop.Reload()
|
busStop.Reload()
|
||||||
|
|
||||||
(*busStops)[ entry.StationId ] = busStop
|
(*busStops)[entry.StationId] = busStop
|
||||||
}
|
}
|
||||||
|
|
||||||
if busStop.Routes == nil {
|
if busStop.Routes == nil {
|
||||||
busStopRoutes := [] *RouteStop{}
|
busStopRoutes := []*RouteStop{}
|
||||||
busStop.Routes = &busStopRoutes
|
busStop.Routes = &busStopRoutes
|
||||||
}
|
}
|
||||||
|
|
||||||
(*busStop.Routes) = append( (*busStop.Routes), entry )
|
(*busStop.Routes) = append((*busStop.Routes), entry)
|
||||||
entry.BusStop = busStop
|
entry.BusStop = busStop
|
||||||
|
|
||||||
route := routeStops[ entry.RouteId ]
|
route := routeStops[entry.RouteId]
|
||||||
if route == nil {
|
if route == nil {
|
||||||
route = &map[string] *map[ string ] *map[ int ] *RouteStop{}
|
route = &map[string]*map[string]*map[int]*RouteStop{}
|
||||||
routeStops[ entry.RouteId ] = route
|
routeStops[entry.RouteId] = route
|
||||||
}
|
}
|
||||||
|
|
||||||
service := (*route)[ entry.ServiceType ]
|
service := (*route)[entry.ServiceType]
|
||||||
if service == nil {
|
if service == nil {
|
||||||
service = &map[ string ] *map[ int ] *RouteStop{}
|
service = &map[string]*map[int]*RouteStop{}
|
||||||
(*route)[ entry.ServiceType ] = service
|
(*route)[entry.ServiceType] = service
|
||||||
}
|
}
|
||||||
|
|
||||||
direction := (*service)[ entry.Direction ]
|
direction := (*service)[entry.Direction]
|
||||||
if direction == nil {
|
if direction == nil {
|
||||||
direction = &map[ int ] *RouteStop{}
|
direction = &map[int]*RouteStop{}
|
||||||
(*service)[ entry.Direction ] = direction
|
(*service)[entry.Direction] = direction
|
||||||
}
|
}
|
||||||
entry.RouteStops = direction
|
entry.RouteStops = direction
|
||||||
|
|
||||||
seq := (*direction)[ entry.StationSeq ]
|
seq := (*direction)[entry.StationSeq]
|
||||||
if seq == nil {
|
if seq == nil {
|
||||||
(*direction)[ entry.StationSeq ] = entry
|
(*direction)[entry.StationSeq] = entry
|
||||||
}
|
}
|
||||||
allRouteStops = append( allRouteStops, entry )
|
allRouteStops = append(allRouteStops, entry)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &allRouteStops, nil
|
return &allRouteStops, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func readBusStopsData( buff *bytes.Buffer ) ( *map[string]*BusStop, error ) {
|
func readBusStopsData(buff *bytes.Buffer) (*map[string]*BusStop, error) {
|
||||||
busStopsData := BusStops{}
|
busStopsData := BusStops{}
|
||||||
err := json.Unmarshal( buff.Bytes(), &busStopsData )
|
err := json.Unmarshal(buff.Bytes(), &busStopsData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
busStopMap := map[string] *BusStop{}
|
busStopMap := map[string]*BusStop{}
|
||||||
for _, entry := range busStopsData.BusStops {
|
for _, entry := range busStopsData.BusStops {
|
||||||
|
|
||||||
entry.Reload()
|
entry.Reload()
|
||||||
|
|
||||||
if _, ok := busStopMap[ entry.BusStopId ]; ok {
|
if _, ok := busStopMap[entry.BusStopId]; ok {
|
||||||
return nil, fmt.Errorf( "Duplicated BusStop: %s", entry.BusStopId )
|
return nil, fmt.Errorf("Duplicated BusStop: %s", entry.BusStopId)
|
||||||
}
|
}
|
||||||
|
|
||||||
busStopMap[ entry.BusStopId ] = entry
|
busStopMap[entry.BusStopId] = entry
|
||||||
}
|
}
|
||||||
return &busStopMap, nil
|
return &busStopMap, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getRouteStops() (*[] query.ISearchable, error) {
|
func getRouteStops() (*[]query.ISearchable, error) {
|
||||||
|
|
||||||
QUERY_FUNC := func() ( io.ReadCloser, error ) {
|
QUERY_FUNC := func() (io.ReadCloser, error) {
|
||||||
resp, err := http.Get( "https://data.etabus.gov.hk/v1/transport/kmb/stop" )
|
return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/stop")
|
||||||
if err != nil {
|
}
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return resp.Body, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
buff, err := utils.CacheStream( JSON_BUSSTOPS, QUERY_FUNC, 7 * 24 * 3600 )
|
buff, err := utils.CacheStream(JSON_BUSSTOPS, QUERY_FUNC, 7*24*3600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
busStopMap, err := readBusStopsData( buff )
|
busStopMap, err := readBusStopsData(buff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
QUERY_FUNC = func() ( io.ReadCloser, error ) {
|
QUERY_FUNC = func() (io.ReadCloser, error) {
|
||||||
resp, err := http.Get( "https://data.etabus.gov.hk/v1/transport/kmb/route-stop" )
|
return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/route-stop")
|
||||||
if err != nil {
|
}
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return resp.Body, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
buff, err = utils.CacheStream( JSON_ROUTESTOPS, QUERY_FUNC, 7 * 24 * 3600 )
|
buff, err = utils.CacheStream(JSON_ROUTESTOPS, QUERY_FUNC, 7*24*3600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
routeStops, err := readRouteStopsData( busStopMap, buff )
|
routeStops, err := readRouteStopsData(busStopMap, buff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
log.Printf("Unable to parse RouteStopsData: %s", err)
|
||||||
}
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
searchables := [] query.ISearchable{}
|
searchables := []query.ISearchable{}
|
||||||
for _, routeStop := range *routeStops {
|
for _, routeStop := range *routeStops {
|
||||||
searchables = append( searchables, routeStop )
|
searchables = append(searchables, routeStop)
|
||||||
routeStop.Reload()
|
routeStop.Reload()
|
||||||
}
|
}
|
||||||
|
|
||||||
return &searchables, nil
|
return &searchables, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,9 +2,12 @@ package utils
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var MARKDOWN_ESC []string = []string{"_", "*", "[", "]", "(", ")", "~", "`", ">", "#", "+", "-", "=", "|", "{", "}", ".", "!"}
|
var MARKDOWN_ESC []string = []string{"_", "*", "[", "]", "(", ")", "~", "`", ">", "#", "+", "-", "=", "|", "{", "}", ".", "!"}
|
||||||
@@ -117,3 +120,27 @@ func TryGetEnv[T any](name string, fallback T) T {
|
|||||||
|
|
||||||
return fallback
|
return fallback
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func HttpGet(url string) (io.ReadCloser, error) {
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: 15 * time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequest("GET", url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
ua := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
|
||||||
|
req.Header.Set("User-Agent", ua)
|
||||||
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
|
||||||
|
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||||
|
req.Header.Set("Connection", "keep-alive")
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp.Body, nil
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user