Mimic browser requests because kmb blocked Go httpGet clients

datasources/kmb/routestops.go

@@ -5,9 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-    "net/http"
+	"log"
 	"path/filepath"
-    // "strings"
 
 	query "github.com/tgckpg/golifehk/query"
 	"github.com/tgckpg/golifehk/utils"
@@ -102,11 +101,7 @@ func readBusStopsData( buff *bytes.Buffer ) ( *map[string]*BusStop, error ) {
 func getRouteStops() (*[]query.ISearchable, error) {
 
 	QUERY_FUNC := func() (io.ReadCloser, error) {
-        resp, err := http.Get( "https://data.etabus.gov.hk/v1/transport/kmb/stop" )
+		return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/stop")
-        if err != nil {
-            return nil, err
-        }
-        return resp.Body, nil
 	}
 
 	buff, err := utils.CacheStream(JSON_BUSSTOPS, QUERY_FUNC, 7*24*3600)
@@ -120,11 +115,7 @@ func getRouteStops() (*[] query.ISearchable, error) {
 	}
 
 	QUERY_FUNC = func() (io.ReadCloser, error) {
-        resp, err := http.Get( "https://data.etabus.gov.hk/v1/transport/kmb/route-stop" )
+		return utils.HttpGet("https://data.etabus.gov.hk/v1/transport/kmb/route-stop")
-        if err != nil {
-            return nil, err
-        }
-        return resp.Body, nil
 	}
 
 	buff, err = utils.CacheStream(JSON_ROUTESTOPS, QUERY_FUNC, 7*24*3600)
@@ -134,6 +125,7 @@ func getRouteStops() (*[] query.ISearchable, error) {
 
 	routeStops, err := readRouteStopsData(busStopMap, buff)
 	if err != nil {
+		log.Printf("Unable to parse RouteStopsData: %s", err)
 		return nil, err
 	}
 

utils/shortcuts.go

@@ -2,9 +2,12 @@ package utils
 
 import (
 	"bytes"
+	"io"
+	"net/http"
 	"os"
 	"strconv"
 	"strings"
+	"time"
 )
 
 var MARKDOWN_ESC []string = []string{"_", "*", "[", "]", "(", ")", "~", "`", ">", "#", "+", "-", "=", "|", "{", "}", ".", "!"}
@@ -117,3 +120,27 @@ func TryGetEnv[T any](name string, fallback T) T {
 
 	return fallback
 }
+
+func HttpGet(url string) (io.ReadCloser, error) {
+	client := &http.Client{
+		Timeout: 15 * time.Second,
+	}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	ua := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
+	req.Header.Set("User-Agent", ua)
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	req.Header.Set("Connection", "keep-alive")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return resp.Body, nil
+}