penguin/golifehk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Mimic browser requests because kmb blocked Go httpGet clients

Browse Source
This commit is contained in:
斟酌 鵬兄
2026-02-27 14:43:02 +08:00
parent d54f106dee
commit cda11cef2e
2 changed files with 130 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
datasources/kmb/routestops.go
View File

@@ -5,141 +5,133 @@ import (
"encoding/json"
"fmt"
"io"
"net/http"
"log"
"path/filepath"
// "strings"
query "github.com/tgckpg/golifehk/query"
"github.com/tgckpg/golifehk/utils"
)
var JSON_ROUTESTOPS string = filepath.Join( utils.WORKDIR, "kmb-routestops.json" )
var JSON_BUSSTOPS string = filepath.Join( utils.WORKDIR, "kmb-busstops.json" )
var JSON_ROUTESTOPS string = filepath.Join(utils.WORKDIR, "kmb-routestops.json")
var JSON_BUSSTOPS string = filepath.Join(utils.WORKDIR, "kmb-busstops.json")
func readRouteStopsData( busStops *map[string] *BusStop, buff *bytes.Buffer ) ( *[] *RouteStop, error ) {
func readRouteStopsData(busStops *map[string]*BusStop, buff *bytes.Buffer) (*[]*RouteStop, error) {
routeStopsData := RouteStops{}
err := json.Unmarshal( buff.Bytes(), &routeStopsData )
err := json.Unmarshal(buff.Bytes(), &routeStopsData)
if err != nil {
return nil, err
}
// routeStops[ Route ][ ServiceType ][ Direction ][ Seq ] = RouteStop
routeStops := map[string] *map[string] *map[ string ] *map[ int ] *RouteStop{}
allRouteStops := [] *RouteStop{}
routeStops := map[string]*map[string]*map[string]*map[int]*RouteStop{}
allRouteStops := []*RouteStop{}
for _, entry := range routeStopsData.RouteStops {
busStop := (*busStops)[ entry.StationId ]
busStop := (*busStops)[entry.StationId]
if busStop == nil {
busStop = &BusStop {
busStop = &BusStop{
BusStopId: entry.StationId,
Name_en: "???", Name_tc: "???", Name_sc: "???",
}
busStop.Reload()
(*busStops)[ entry.StationId ] = busStop
(*busStops)[entry.StationId] = busStop
}
if busStop.Routes == nil {
busStopRoutes := [] *RouteStop{}
busStopRoutes := []*RouteStop{}
busStop.Routes = &busStopRoutes
}
(*busStop.Routes) = append( (*busStop.Routes), entry )
(*busStop.Routes) = append((*busStop.Routes), entry)
entry.BusStop = busStop
route := routeStops[ entry.RouteId ]
route := routeStops[entry.RouteId]
if route == nil {
route = &map[string] *map[ string ] *map[ int ] *RouteStop{}
routeStops[ entry.RouteId ] = route
route = &map[string]*map[string]*map[int]*RouteStop{}
routeStops[entry.RouteId] = route
}
service := (*route)[ entry.ServiceType ]
service := (*route)[entry.ServiceType]
if service == nil {
service = &map[ string ] *map[ int ] *RouteStop{}
(*route)[ entry.ServiceType ] = service
service = &map[string]*map[int]*RouteStop{}
(*route)[entry.ServiceType] = service
}
direction := (*service)[ entry.Direction ]
direction := (*service)[entry.Direction]
if direction == nil {
direction = &map[ int ] *RouteStop{}
(*service)[ entry.Direction ] = direction
direction = &map[int]*RouteStop{}
(*service)[entry.Direction] = direction
}
entry.RouteStops = direction
seq := (*direction)[ entry.StationSeq ]
seq := (*direction)[entry.StationSeq]
if seq == nil {
(*direction)[ entry.StationSeq ] = entry
(*direction)[entry.StationSeq] = entry
}
allRouteStops = append( allRouteStops, entry )
allRouteStops = append(allRouteStops, entry)
}
return &allRouteStops, nil
}
func readBusStopsData( buff *bytes.Buffer ) ( *map[string]*BusStop, error ) {
func readBusStopsData(buff *bytes.Buffer) (*map[string]*BusStop, error) {
busStopsData := BusStops{}
err := json.Unmarshal( buff.Bytes(), &busStopsData )
err := json.Unmarshal(buff.Bytes(), &busStopsData)
if err != nil {
return nil, err
}
busStopMap := map[string] *BusStop{}
busStopMap := map[string]*BusStop{}
for _, entry := range busStopsData.BusStops {
entry.Reload()
if _, ok := busStopMap[ entry.BusStopId ]; ok {
return nil, fmt.Errorf( "Duplicated BusStop: %s", entry.BusStopId )
if _, ok := busStopMap[entry.BusStopId]; ok {
return nil, fmt.Errorf("Duplicated BusStop: %s", entry.BusStopId)
}
busStopMap[ entry.BusStopId ] = entry
busStopMap[entry.BusStopId] = entry
}
return &busStopMap, nil
}
func getRouteStops() (*[] query.ISearchable, error) {
func getRouteStops() (*[]query.ISearchable, error) {
QUERY_FUNC := func() ( io.ReadCloser, error ) {
resp, err := http.Get( "https://data.etabus.gov.hk/v1/transport/kmb/stop" )
if err != nil {
return nil, err
}
return resp.Body, nil
QUERY_FUNC := func() (io.ReadCloser, error) {
return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/stop")
}
buff, err := utils.CacheStream( JSON_BUSSTOPS, QUERY_FUNC, 7 * 24 * 3600 )
buff, err := utils.CacheStream(JSON_BUSSTOPS, QUERY_FUNC, 7*24*3600)
if err != nil {
return nil, err
}
busStopMap, err := readBusStopsData( buff )
busStopMap, err := readBusStopsData(buff)
if err != nil {
return nil, err
}
QUERY_FUNC = func() ( io.ReadCloser, error ) {
resp, err := http.Get( "https://data.etabus.gov.hk/v1/transport/kmb/route-stop" )
if err != nil {
return nil, err
}
return resp.Body, nil
QUERY_FUNC = func() (io.ReadCloser, error) {
return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/route-stop")
}
buff, err = utils.CacheStream( JSON_ROUTESTOPS, QUERY_FUNC, 7 * 24 * 3600 )
buff, err = utils.CacheStream(JSON_ROUTESTOPS, QUERY_FUNC, 7*24*3600)
if err != nil {
return nil, err
}
routeStops, err := readRouteStopsData( busStopMap, buff )
routeStops, err := readRouteStopsData(busStopMap, buff)
if err != nil {
log.Printf("Unable to parse RouteStopsData: %s", err)
return nil, err
}
searchables := [] query.ISearchable{}
searchables := []query.ISearchable{}
for _, routeStop := range *routeStops {
searchables = append( searchables, routeStop )
searchables = append(searchables, routeStop)
routeStop.Reload()
}

27
utils/shortcuts.go
View File

@@ -2,9 +2,12 @@ package utils
import (
"bytes"
"io"
"net/http"
"os"
"strconv"
"strings"
"time"
)
var MARKDOWN_ESC []string = []string{"_", "*", "[", "]", "(", ")", "~", "`", ">", "#", "+", "-", "=", "|", "{", "}", ".", "!"}
@@ -117,3 +120,27 @@ func TryGetEnv[T any](name string, fallback T) T {
return fallback
}
func HttpGet(url string) (io.ReadCloser, error) {
client := &http.Client{
Timeout: 15 * time.Second,
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
ua := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
req.Header.Set("User-Agent", ua)
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
req.Header.Set("Connection", "keep-alive")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
return resp.Body, nil
}