diff --git a/resolver-go/cmd/api-server/main.go b/resolver-go/cmd/api-server/main.go index bf03610..6a9e144 100644 --- a/resolver-go/cmd/api-server/main.go +++ b/resolver-go/cmd/api-server/main.go @@ -8,7 +8,9 @@ import ( "time" "github.com/tgckpg/resolver-go/internal/closure" + "github.com/tgckpg/resolver-go/internal/compilecache" "github.com/tgckpg/resolver-go/internal/generated" + "github.com/tgckpg/resolver-go/internal/minifier/css" "github.com/tgckpg/resolver-go/internal/resolver" ) @@ -23,8 +25,13 @@ func main() { } h := handler{ - r: r, - closure: closure.NewCompileCache(2), + r: r, + jsCache: compilecache.New( + closure.NewCompiler(), 2, 128, + ), + cssCache: compilecache.New( + css.NewEsbuildCompiler(), 2, 128, + ), } http.HandleFunc("/", h.index) log.Printf("botan-api listening on %s", *addr) @@ -32,8 +39,9 @@ func main() { } type handler struct { - r *resolver.Resolver - closure *closure.CompileCache + r *resolver.Resolver + jsCache *compilecache.Cache + cssCache *compilecache.Cache } func (h handler) index(w http.ResponseWriter, req *http.Request) { @@ -71,7 +79,8 @@ func (h handler) index(w http.ResponseWriter, req *http.Request) { } if outMode == resolver.ModeJS { - if compiled, ok := h.closure.Get(res.Hash); ok { + state, compiled, err := h.jsCache.Get(res.Hash) + if state == compilecache.Ready { w.Header().Set("Content-Type", "application/javascript") w.Header().Set("X-Botan-Compiled", "hit") elapsed := time.Since(timerStart) @@ -86,31 +95,43 @@ func (h handler) index(w http.ResponseWriter, req *http.Request) { return } - h.closure.Enqueue(closure.CompileJob{ - Hash: res.Hash, - Mode: "js", - ExternSources: jsExterns, - JSSources: []closure.SourceInput{ - { - Name: "botanjs-" + res.Hash + ".js", - Source: string(res.Content), + h.jsCache.Enqueue(compilecache.Job{ + Hash: res.Hash, + Mode: "js", + Payload: closure.CompilePayload{ + ExternSources: jsExterns, + JSSources: []closure.SourceInput{ + { + Name: "botanjs-" + res.Hash + ".js", + Source: string(res.Content), + }, + }, + Defines: map[string]any{ + "DEBUG": false, }, }, - Defines: map[string]any{ - "DEBUG": false, + }) + } else if outMode == resolver.ModeCSS { + state, compiled, _ := h.cssCache.Get(res.Hash) + if state == compilecache.Ready { + w.Header().Set("Content-Type", "text/css") + w.Header().Set("X-Botan-Compiled", "hit") + elapsed := time.Since(timerStart) + w.Header().Set("X-Botan-Resolve-Time", elapsed.String()) + w.Write(compiled) + return + } + + h.cssCache.Enqueue(compilecache.Job{ + Hash: res.Hash, + Mode: "css", + Payload: css.CompilePayload{ + Name: "style.css", + Source: string(res.Content), }, }) } - // Compatibility flags: - // rjs/rcss/ojs/ocss => content - // hjs/hcss => hash filename only - // js/css => currently content; cluster compiler can switch this later. - if strings.HasPrefix(modeText, "h") { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - _, _ = w.Write([]byte(res.Hash)) - return - } w.Header().Set("Content-Type", res.ContentType) w.Header().Set("X-Botan-Compiled", "miss") diff --git a/resolver-go/go.mod b/resolver-go/go.mod index ad1c6f3..4e0a19f 100644 --- a/resolver-go/go.mod +++ b/resolver-go/go.mod @@ -1,3 +1,7 @@ module github.com/tgckpg/resolver-go go 1.26 + +require github.com/evanw/esbuild v0.28.1 + +require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/resolver-go/go.sum b/resolver-go/go.sum index e69de29..82e4e7e 100644 --- a/resolver-go/go.sum +++ b/resolver-go/go.sum @@ -0,0 +1,4 @@ +github.com/evanw/esbuild v0.28.1 h1:ds+yuRyUaZGx++GR56CrCeuXh8PVhVM4xq8v7PNELFc= +github.com/evanw/esbuild v0.28.1/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/resolver-go/internal/closure/cache.go b/resolver-go/internal/closure/cache.go deleted file mode 100644 index 547bdf4..0000000 --- a/resolver-go/internal/closure/cache.go +++ /dev/null @@ -1,87 +0,0 @@ -package closure - -import ( - "context" - "errors" - "log" - "time" -) - -func NewCompileCache(workers int) *CompileCache { - c := &CompileCache{ - client: NewClientFromEnv(), - states: make(map[string]CompileState), - results: make(map[string][]byte), - errors: make(map[string]error), - jobs: make(chan CompileJob, 128), - } - - for i := 0; i < workers; i++ { - go c.worker() - } - - return c -} - -func (c *CompileCache) Get(hash string) ([]byte, bool) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.states[hash] != CompileReady { - return nil, false - } - - return c.results[hash], true -} - -func (c *CompileCache) Enqueue(job CompileJob) { - c.mu.Lock() - - switch c.states[job.Hash] { - case CompilePending, CompileReady: - c.mu.Unlock() - return - } - - c.states[job.Hash] = CompilePending - c.mu.Unlock() - - select { - case c.jobs <- job: - default: - // Queue full. Don't block request path. - c.mu.Lock() - c.states[job.Hash] = CompileMissing - c.errors[job.Hash] = errors.New("compile queue full") - c.mu.Unlock() - } -} - -func (c *CompileCache) worker() { - for job := range c.jobs { - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - - req := CompileRequest{ - ExternSources: job.ExternSources, - JSSources: job.JSSources, - Defines: job.Defines, - } - - c.client.DebugPrintCurl(ctx, req) - - out, err := c.client.Compile(ctx, req) - cancel() - - c.mu.Lock() - if err == nil { - c.states[job.Hash] = CompileReady - c.results[job.Hash] = out - delete(c.errors, job.Hash) - } else { - log.Printf("compile failed: %v", err) - c.states[job.Hash] = CompileFailed - c.errors[job.Hash] = err - } - c.mu.Unlock() - } -} diff --git a/resolver-go/internal/closure/compiler.go b/resolver-go/internal/closure/compiler.go new file mode 100644 index 0000000..c0e2432 --- /dev/null +++ b/resolver-go/internal/closure/compiler.go @@ -0,0 +1,31 @@ +package closure + +import ( + "context" + "fmt" + + "github.com/tgckpg/resolver-go/internal/compilecache" +) + +type Compiler struct { + client *Client +} + +func NewCompiler() *Compiler { + return &Compiler{client: NewClientFromEnv()} +} + +func (c *Compiler) Compile(ctx context.Context, job compilecache.Job) ([]byte, error) { + payload, ok := job.Payload.(CompilePayload) + if !ok { + return nil, fmt.Errorf("closure compiler got invalid payload type %T", job.Payload) + } + + req := CompileRequest{ + ExternSources: payload.ExternSources, + JSSources: payload.JSSources, + Defines: payload.Defines, + } + + return c.client.Compile(ctx, req) +} diff --git a/resolver-go/internal/closure/types.go b/resolver-go/internal/closure/types.go index 064cd92..ea7a464 100644 --- a/resolver-go/internal/closure/types.go +++ b/resolver-go/internal/closure/types.go @@ -1,20 +1,9 @@ package closure -import "sync" - -type CompileState int - -const ( - CompileMissing CompileState = iota - CompilePending - CompileReady - CompileFailed -) - -type CompileRequest struct { - ExternSources []SourceInput `json:"externSources,omitempty"` - JSSources []SourceInput `json:"jsSources"` - Defines map[string]any `json:"defines,omitempty"` +type CompilePayload struct { + ExternSources []SourceInput + JSSources []SourceInput + Defines map[string]any } type SourceInput struct { @@ -22,22 +11,10 @@ type SourceInput struct { Source string `json:"source"` } -type CompileJob struct { - Hash string - Mode string - - ExternSources []SourceInput - JSSources []SourceInput - Defines map[string]any -} - -type CompileCache struct { - client *Client - mu sync.Mutex - states map[string]CompileState - results map[string][]byte - errors map[string]error - jobs chan CompileJob +type CompileRequest struct { + ExternSources []SourceInput `json:"externSources,omitempty"` + JSSources []SourceInput `json:"jsSources"` + Defines map[string]any `json:"defines,omitempty"` } type CompileResponse struct { diff --git a/resolver-go/internal/compilecache/cache.go b/resolver-go/internal/compilecache/cache.go new file mode 100644 index 0000000..85b8903 --- /dev/null +++ b/resolver-go/internal/compilecache/cache.go @@ -0,0 +1,89 @@ +package compilecache + +import ( + "context" +) + +func New(compiler Compiler, workers int, queueSize int) *Cache { + if workers <= 0 { + workers = 1 + } + if queueSize <= 0 { + queueSize = 128 + } + + c := &Cache{ + compiler: compiler, + states: make(map[string]State), + results: make(map[string][]byte), + errors: make(map[string]error), + jobs: make(chan Job, queueSize), + } + + for i := 0; i < workers; i++ { + go c.worker() + } + + return c +} + +func (c *Cache) Get(hash string) (State, []byte, error) { + c.mu.Lock() + defer c.mu.Unlock() + + state, ok := c.states[hash] + if !ok { + return Missing, nil, nil + } + + switch state { + case Ready: + return Ready, c.results[hash], nil + case Failed: + return Failed, nil, c.errors[hash] + default: + return state, nil, nil + } +} + +func (c *Cache) Enqueue(job Job) State { + c.mu.Lock() + + state, ok := c.states[job.Hash] + if ok { + c.mu.Unlock() + return state + } + + c.states[job.Hash] = Pending + c.mu.Unlock() + + // Do not hold the lock while sending. + c.jobs <- job + + return Pending +} + +func (c *Cache) worker() { + for job := range c.jobs { + c.run(job) + } +} + +func (c *Cache) run(job Job) { + result, err := c.compiler.Compile(context.Background(), job) + + c.mu.Lock() + defer c.mu.Unlock() + + if err != nil { + c.states[job.Hash] = Failed + c.errors[job.Hash] = err + delete(c.results, job.Hash) + return + } + + c.states[job.Hash] = Ready + c.results[job.Hash] = result + delete(c.errors, job.Hash) +} diff --git a/resolver-go/internal/compilecache/types.go b/resolver-go/internal/compilecache/types.go new file mode 100644 index 0000000..1fa6ab2 --- /dev/null +++ b/resolver-go/internal/compilecache/types.go @@ -0,0 +1,36 @@ +package compilecache + +import ( + "context" + "sync" +) + +type State int + +const ( + Missing State = iota + Pending + Ready + Failed +) + +type Job struct { + Hash string + Mode string + + Payload any +} + +type Compiler interface { + Compile(ctx context.Context, job Job) ([]byte, error) +} + +type Cache struct { + compiler Compiler + + mu sync.Mutex + states map[string]State + results map[string][]byte + errors map[string]error + jobs chan Job +} diff --git a/resolver-go/internal/minifier/css/compiler.go b/resolver-go/internal/minifier/css/compiler.go new file mode 100644 index 0000000..ad0e63e --- /dev/null +++ b/resolver-go/internal/minifier/css/compiler.go @@ -0,0 +1,34 @@ +package css + +import ( + "context" + "fmt" + + "github.com/evanw/esbuild/pkg/api" + "github.com/tgckpg/resolver-go/internal/compilecache" +) + +func NewEsbuildCompiler() *EsbuildCompiler { + return &EsbuildCompiler{} +} + +func (c *EsbuildCompiler) Compile(ctx context.Context, job compilecache.Job) ([]byte, error) { + payload, ok := job.Payload.(CompilePayload) + if !ok { + return nil, fmt.Errorf("esbuild css compiler got invalid payload type %T", job.Payload) + } + + result := api.Transform(payload.Source, api.TransformOptions{ + Loader: api.LoaderCSS, + Sourcefile: payload.Name, + MinifyWhitespace: true, + MinifyIdentifiers: true, + MinifySyntax: true, + }) + + if len(result.Errors) > 0 { + return nil, fmt.Errorf("css compile failed: %s", result.Errors[0].Text) + } + + return result.Code, nil +} diff --git a/resolver-go/internal/minifier/css/types.go b/resolver-go/internal/minifier/css/types.go new file mode 100644 index 0000000..5191dee --- /dev/null +++ b/resolver-go/internal/minifier/css/types.go @@ -0,0 +1,8 @@ +package css + +type CompilePayload struct { + Name string + Source string +} + +type EsbuildCompiler struct{}