package utils import ( "bytes" "fmt" "io" "log" "os" "path/filepath" "time" ) type CacheStreams struct { Local *bytes.Buffer Remote *bytes.Buffer Path string Query func() (io.ReadCloser, error) } func (cs *CacheStreams) Commit() error { f, err := os.OpenFile(cs.Path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return err } defer f.Close() _, err = io.Copy(f, bytes.NewReader(cs.Remote.Bytes())) log.Printf("Commit: %s", cs.Path) return err } func (cs *CacheStreams) Try(Parser func(*bytes.Buffer) error, expires time.Duration, retries int) error { var err error for i := 0; i < retries; i++ { if cs.NotExpired(expires) { err = Parser(cs.Local) if err == nil { log.Printf("Using cache: %s", cs.Path) return nil } } err = cs.Reload() log.Printf("Reloading (%d): %s", i+1, cs.Path) if err != nil { log.Printf("Error retrieving data(%s): %s", i+1, err) } if cs.Remote != nil { err = Parser(cs.Remote) if err == nil { cs.Commit() return nil } } } return fmt.Errorf("Failed to get data for: %s", cs.Path) } func (cs *CacheStreams) Reload() error { s, err := cs.Query() if err != nil { return err } defer s.Close() cs.Remote = bytes.NewBuffer([]byte{}) _, err = io.Copy(cs.Remote, s) return err } func (cs *CacheStreams) NotExpired(expires time.Duration) bool { cache, err := os.Stat(cs.Path) if err == nil { expired := cache.ModTime().Add(expires * 1e9) return time.Now().Before(expired) } return false } func CacheStreamEx(path string, readStream func() (io.ReadCloser, error)) (*CacheStreams, error) { cs := CacheStreams{} cs.Path = path cs.Query = readStream f, err := os.Open(path) if err == nil { defer f.Close() cs.Local = bytes.NewBuffer([]byte{}) _, err = io.Copy(cs.Local, f) if err != nil { return nil, err } } return &cs, nil } func CacheStream(path string, readStream func() (io.ReadCloser, error), expires time.Duration) (*bytes.Buffer, error) { cache, err := os.Stat(path) // Check if cache exists and not expired if err == nil { expired := cache.ModTime().Add(expires * 1e9) if time.Now().Before(expired) { f, err := os.Open(path) if err == nil { defer f.Close() log.Printf("Using cache: %s", path) writeBuff := bytes.NewBuffer([]byte{}) _, err = io.Copy(writeBuff, f) if err == nil { return writeBuff, nil } } } } err = os.MkdirAll(filepath.Dir(path), 0750) if err != nil { return nil, err } writeBuff := bytes.NewBuffer([]byte{}) // Get the reader that return new data s, err := readStream() if err != nil { return nil, err } defer s.Close() _, err = io.Copy(writeBuff, s) if err != nil { return nil, err } f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return nil, err } defer f.Close() data := writeBuff.Bytes() _, err = io.Copy(f, bytes.NewReader(data)) if err != nil { return nil, err } return writeBuff, nil } func ChangedStream(path string, readStream func() (io.Reader, error), dataModTime time.Time) (*bytes.Buffer, error) { cache, err := os.Stat(path) // Check if cache exists and not expired if err == nil { if dataModTime.Before(cache.ModTime()) { f, err := os.Open(path) if err == nil { defer f.Close() log.Printf("Reading from file: %s", path) writeBuff := bytes.NewBuffer([]byte{}) _, err = io.Copy(writeBuff, f) if err == nil { return writeBuff, nil } } } } err = os.MkdirAll(filepath.Dir(path), 0750) if err != nil { return nil, err } writeBuff := bytes.NewBuffer([]byte{}) // Get the reader that return new data s, err := readStream() if err != nil { return nil, err } _, err = io.Copy(writeBuff, s) if err != nil { return nil, err } f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return nil, err } defer f.Close() data := writeBuff.Bytes() _, err = io.Copy(f, bytes.NewReader(data)) if err != nil { return nil, err } return writeBuff, nil }