ref: 3466884e364d0df39394742697d6fa0b37d49e64
dir: /cache/filecache/filecache.go/
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package filecache
import (
"bytes"
"errors"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/helpers"
"github.com/BurntSushi/locker"
"github.com/spf13/afero"
)
// ErrFatal can be used to signal an unrecoverable error.
var ErrFatal = errors.New("fatal filecache error")
const (
filecacheRootDirname = "filecache"
)
// Cache caches a set of files in a directory. This is usually a file on
// disk, but since this is backed by an Afero file system, it can be anything.
type Cache struct {
Fs afero.Fs
// Max age for items in this cache. Negative duration means forever,
// 0 is effectively turning this cache off.
maxAge time.Duration
// When set, we just remove this entire root directory on expiration.
pruneAllRootDir string
nlocker *lockTracker
}
type lockTracker struct {
seenMu sync.RWMutex
seen map[string]struct{}
*locker.Locker
}
// Lock tracks the ids in use. We use this information to do garbage collection
// after a Hugo build.
func (l *lockTracker) Lock(id string) {
l.seenMu.RLock()
if _, seen := l.seen[id]; !seen {
l.seenMu.RUnlock()
l.seenMu.Lock()
l.seen[id] = struct{}{}
l.seenMu.Unlock()
} else {
l.seenMu.RUnlock()
}
l.Locker.Lock(id)
}
// ItemInfo contains info about a cached file.
type ItemInfo struct {
// This is the file's name relative to the cache's filesystem.
Name string
}
// NewCache creates a new file cache with the given filesystem and max age.
func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache {
return &Cache{
Fs: fs,
nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
maxAge: maxAge,
pruneAllRootDir: pruneAllRootDir,
}
}
// lockedFile is a file with a lock that is released on Close.
type lockedFile struct {
afero.File
unlock func()
}
func (l *lockedFile) Close() error {
defer l.unlock()
return l.File.Close()
}
// WriteCloser returns a transactional writer into the cache.
// It's important that it's closed when done.
func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) {
id = cleanID(id)
c.nlocker.Lock(id)
info := ItemInfo{Name: id}
f, err := helpers.OpenFileForWriting(c.Fs, id)
if err != nil {
c.nlocker.Unlock(id)
return info, nil, err
}
return info, &lockedFile{
File: f,
unlock: func() { c.nlocker.Unlock(id) },
}, nil
}
// ReadOrCreate tries to lookup the file in cache.
// If found, it is passed to read and then closed.
// If not found a new file is created and passed to create, which should close
// it when done.
func (c *Cache) ReadOrCreate(id string,
read func(info ItemInfo, r io.ReadSeeker) error,
create func(info ItemInfo, w io.WriteCloser) error) (info ItemInfo, err error) {
id = cleanID(id)
c.nlocker.Lock(id)
defer c.nlocker.Unlock(id)
info = ItemInfo{Name: id}
if r := c.getOrRemove(id); r != nil {
err = read(info, r)
defer r.Close()
if err == nil || err == ErrFatal {
// See https://github.com/gohugoio/hugo/issues/6401
// To recover from file corruption we handle read errors
// as the cache item was not found.
// Any file permission issue will also fail in the next step.
return
}
}
f, err := helpers.OpenFileForWriting(c.Fs, id)
if err != nil {
return
}
err = create(info, f)
return
}
// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
// be invoked and the result cached.
// This method is protected by a named lock using the given id as identifier.
func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (ItemInfo, io.ReadCloser, error) {
id = cleanID(id)
c.nlocker.Lock(id)
defer c.nlocker.Unlock(id)
info := ItemInfo{Name: id}
if r := c.getOrRemove(id); r != nil {
return info, r, nil
}
r, err := create()
if err != nil {
return info, nil, err
}
if c.maxAge == 0 {
// No caching.
return info, hugio.ToReadCloser(r), nil
}
var buff bytes.Buffer
return info,
hugio.ToReadCloser(&buff),
afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff))
}
// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (ItemInfo, []byte, error) {
id = cleanID(id)
c.nlocker.Lock(id)
defer c.nlocker.Unlock(id)
info := ItemInfo{Name: id}
if r := c.getOrRemove(id); r != nil {
defer r.Close()
b, err := ioutil.ReadAll(r)
return info, b, err
}
b, err := create()
if err != nil {
return info, nil, err
}
if c.maxAge == 0 {
return info, b, nil
}
if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil {
return info, nil, err
}
return info, b, nil
}
// GetBytes gets the file content with the given id from the cahce, nil if none found.
func (c *Cache) GetBytes(id string) (ItemInfo, []byte, error) {
id = cleanID(id)
c.nlocker.Lock(id)
defer c.nlocker.Unlock(id)
info := ItemInfo{Name: id}
if r := c.getOrRemove(id); r != nil {
defer r.Close()
b, err := ioutil.ReadAll(r)
return info, b, err
}
return info, nil, nil
}
// Get gets the file with the given id from the cahce, nil if none found.
func (c *Cache) Get(id string) (ItemInfo, io.ReadCloser, error) {
id = cleanID(id)
c.nlocker.Lock(id)
defer c.nlocker.Unlock(id)
info := ItemInfo{Name: id}
r := c.getOrRemove(id)
return info, r, nil
}
// getOrRemove gets the file with the given id. If it's expired, it will
// be removed.
func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
if c.maxAge == 0 {
// No caching.
return nil
}
if c.maxAge > 0 {
fi, err := c.Fs.Stat(id)
if err != nil {
return nil
}
if c.isExpired(fi.ModTime()) {
c.Fs.Remove(id)
return nil
}
}
f, err := c.Fs.Open(id)
if err != nil {
return nil
}
return f
}
func (c *Cache) isExpired(modTime time.Time) bool {
if c.maxAge < 0 {
return false
}
return c.maxAge == 0 || time.Since(modTime) > c.maxAge
}
// For testing
func (c *Cache) getString(id string) string {
id = cleanID(id)
c.nlocker.Lock(id)
defer c.nlocker.Unlock(id)
f, err := c.Fs.Open(id)
if err != nil {
return ""
}
defer f.Close()
b, _ := ioutil.ReadAll(f)
return string(b)
}
// Caches is a named set of caches.
type Caches map[string]*Cache
// Get gets a named cache, nil if none found.
func (f Caches) Get(name string) *Cache {
return f[strings.ToLower(name)]
}
// NewCaches creates a new set of file caches from the given
// configuration.
func NewCaches(p *helpers.PathSpec) (Caches, error) {
var dcfg Configs
if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok {
dcfg = c
} else {
var err error
dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg)
if err != nil {
return nil, err
}
}
fs := p.Fs.Source
m := make(Caches)
for k, v := range dcfg {
var cfs afero.Fs
if v.isResourceDir {
cfs = p.BaseFs.ResourcesCache
} else {
cfs = fs
}
if cfs == nil {
// TODO(bep) we still have some places that do not initialize the
// full dependencies of a site, e.g. the import Jekyll command.
// That command does not need these caches, so let us just continue
// for now.
continue
}
baseDir := v.Dir
if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
return nil, err
}
bfs := afero.NewBasePathFs(cfs, baseDir)
var pruneAllRootDir string
if k == cacheKeyModules {
pruneAllRootDir = "pkg"
}
m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir)
}
return m, nil
}
func cleanID(name string) string {
return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator)
}