Archives/gophi
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

file caching now more threadsafe + added file monitor

Browse Source 
Signed-off-by: kim (grufwub) <grufwub@gmail.com>
This commit is contained in:
kim (grufwub) 2020-04-13 22:48:49 +01:00
parent c408727ba9
commit e1c74ae819
4 changed files with 245 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

213
cache.go
View File

@ -1,12 +1,83 @@
package main
import (
"os"
"sync"
"time"
)
const (
FileMonitorSleepTimeMs = 30
FileMonitorSleepTime = time.Duration(FileMonitorSleepTimeMs) * time.Second
)
func StartFileMonitor(regularCache *RegularFileCache, gophermapCache *GophermapFileCache) {
go func() {
for {
/* Check regular file cache is fresh */
regularCache.Mutex.RLock()
for path := range regularCache.CacheMap {
/* Check if file on disk has changed. */
stat, err := os.Stat(path)
if err != nil {
/* Gotta be speed, skip on error */
continue
}
timeModified := stat.ModTime().UnixNano()
file := regularCache.CacheMap[path]
file.Lock()
if file.IsFresh() && file.LastRefresh() < timeModified {
file.SetUnfresh()
}
file.Unlock()
}
regularCache.Mutex.RUnlock()
/* Check gophermap file cache is fresh */
gophermapCache.Mutex.RLock()
for path := range gophermapCache.CacheMap {
/* Check if file on disk has changed. */
stat, err := os.Stat(path)
if err != nil {
/* Gotta be speed, skip on error */
continue
}
timeModified := stat.ModTime().UnixNano()
file := gophermapCache.CacheMap[path]
file.Lock()
if file.IsFresh() && file.LastRefresh() < timeModified {
file.SetUnfresh()
}
file.Unlock()
}
gophermapCache.Mutex.RUnlock()
/* Sleep so we don't take up all the precious CPU time :) */
time.Sleep(FileMonitorSleepTime)
}
/* We shouldn't have reached here */
logSystemFatal("FileCache monitor crashed!\n")
}()
}
type File interface {
/* File contents */
Contents() []byte
LoadContents() *GophorError
/* Cache state */
IsFresh() bool
SetUnfresh()
LastRefresh() int64
/* Mutex */
Lock()
Unlock()
RLock()
RUnlock()
}
type RegularFileCache struct {
@ -14,51 +85,57 @@ type RegularFileCache struct {
Mutex sync.RWMutex
}
func (fc *RegularFileCache) Init() {
func (fc *RegularFileCache) Init(size int) {
fc.CacheMap = make(map[string]*RegularFile)
fc.Mutex = sync.RWMutex{}
}
func (fc *RegularFileCache) Fetch(path string) ([]byte, *GophorError) {
/* Try get file */
file, ok := fc.GetFile(path)
if !ok {
/* File not in cache, we need to load the file */
var gophorErr *GophorError
file, gophorErr = fc.Load(path)
if gophorErr != nil {
return nil, gophorErr
}
}
return file.Contents(), nil
}
func (fc *RegularFileCache) GetFile(path string) (*RegularFile, bool) {
/* Get read lock, try get file from the cache */
/* Get read lock, try get file and defer read unlock */
fc.Mutex.RLock()
file, ok := fc.CacheMap[path]
fc.Mutex.RUnlock()
return file, ok
}
func (fc *RegularFileCache) Load(path string) (*RegularFile, *GophorError) {
/* Create new file object for path, load contents */
file := new(RegularFile)
file.path = path
if ok {
/* File in cache, check is fresh */
fc.CacheMap[path].Lock()
if !file.IsFresh() {
/* File not fresh, get write lock and update cache */
gophorErr := fc.CacheMap[path].LoadContents()
if gophorErr != nil {
/* Error loading contents, unlock write, return error */
fc.CacheMap[path].Unlock()
fc.Mutex.RUnlock()
return nil, gophorErr
}
}
fc.CacheMap[path].Unlock()
} else {
/* File not in cache, get write lock then we need to load the file */
fc.Mutex.RUnlock()
fc.Mutex.Lock()
gophorErr := file.LoadContents()
if gophorErr != nil {
return nil, gophorErr
file = NewRegularFile(path)
/* We don't need to lock the file here before loading contents as we have a map lock */
gophorErr := file.LoadContents()
if gophorErr != nil {
/* Error loading contents, unlock write, return error */
fc.Mutex.Unlock()
return nil, gophorErr
}
/* Place the file in the cache then unlock :) */
fc.CacheMap[path] = file
fc.Mutex.Unlock()
fc.Mutex.RLock()
}
/* Get lock, add file object to cache */
fc.Mutex.Lock()
fc.CacheMap[path] = file
fc.Mutex.Unlock()
file.RLock()
b := file.Contents()
file.RUnlock()
fc.Mutex.RUnlock()
return file, nil
return b, nil
}
type GophermapFileCache struct {
@ -72,43 +149,49 @@ func (fc *GophermapFileCache) Init() {
}
func (fc *GophermapFileCache) Fetch(path string) ([]byte, *GophorError) {
/* Try get file */
file, ok := fc.GetFile(path)
if !ok {
/* File not in cache, we need to load the file */
var gophorErr *GophorError
file, gophorErr = fc.Load(path)
if gophorErr != nil {
return nil, gophorErr
}
}
return file.Contents(), nil
}
func (fc *GophermapFileCache) GetFile(path string) (*GophermapFile, bool) {
/* Get read lock, try get file from the cache */
/* Get read lock, try get file and defer read unlock */
fc.Mutex.RLock()
file, ok := fc.CacheMap[path]
fc.Mutex.RUnlock()
return file, ok
}
func (fc *GophermapFileCache) Load(path string) (*GophermapFile, *GophorError) {
/* Create new file object for path, load contents */
file := new(GophermapFile)
file.path = path
if ok {
/* File in cache, check is fresh */
fc.CacheMap[path].Lock()
if !file.IsFresh() {
/* File not fresh, get write lock and update cache */
gophorErr := fc.CacheMap[path].LoadContents()
if gophorErr != nil {
/* Error loading contents, unlock write, return error */
fc.CacheMap[path].Unlock()
fc.Mutex.RUnlock()
return nil, gophorErr
}
}
fc.CacheMap[path].Unlock()
} else {
/* File not in cache, get write lock then we need to load the file */
fc.Mutex.RUnlock()
fc.Mutex.Lock()
gophorErr := file.LoadContents()
if gophorErr != nil {
return nil, gophorErr
file = NewGophermapFile(path)
/* We don't need to lock the file here before loading contents as we have a map lock */
gophorErr := file.LoadContents()
if gophorErr != nil {
/* Error loading contents, unlock write, return error */
fc.Mutex.Unlock()
return nil, gophorErr
}
/* Place the file in the cache then unlock :) */
fc.CacheMap[path] = file
fc.Mutex.Unlock()
fc.Mutex.RLock()
}
/* Get lock, add file object to cache. */
fc.Mutex.Lock()
fc.CacheMap[path] = file
fc.Mutex.Unlock()
file.RLock()
b := file.Contents()
file.RUnlock()
fc.Mutex.RUnlock()
return file, nil
return b, nil
}

49
file.go
View File

@ -4,16 +4,28 @@ import (
"os"
"io"
"bufio"
"sync"
"time"
)
type RegularFile struct {
path string
contents []byte
path string
contents []byte
mutex *sync.RWMutex
isFresh bool
lastRefresh int64
/* Implements */
File
}
func NewRegularFile(path string) *RegularFile {
f := new(RegularFile)
f.path = path
f.mutex = new(sync.RWMutex)
return f
}
func (f *RegularFile) Contents() []byte {
return f.contents
}
@ -25,9 +37,42 @@ func (f *RegularFile) LoadContents() *GophorError {
/* Reload the file */
var gophorErr *GophorError
f.contents, gophorErr = bufferedRead(f.path)
/* Update lastRefresh time + set fresh */
f.lastRefresh = time.Now().UnixNano()
f.isFresh = true
return gophorErr
}
func (f *RegularFile) IsFresh() bool {
return f.isFresh
}
func (f *RegularFile) SetUnfresh() {
f.isFresh = false
}
func (f *RegularFile) LastRefresh() int64 {
return f.lastRefresh
}
func (f *RegularFile) Lock() {
f.mutex.Lock()
}
func (f *RegularFile) Unlock() {
f.mutex.Unlock()
}
func (f *RegularFile) RLock() {
f.mutex.RLock()
}
func (f *RegularFile) RUnlock() {
f.mutex.RUnlock()
}
func bufferedRead(path string) ([]byte, *GophorError) {
/* Open file */
fd, err := os.Open(path)

49
file_gophermap.go
View File

@ -6,6 +6,8 @@ import (
"bufio"
"bytes"
"strings"
"sync"
"time"
)
const GophermapFileStr = "gophermap"
@ -50,13 +52,23 @@ func (s *GophermapDirListing) Render() ([]byte, *GophorError) {
}
type GophermapFile struct {
path string
lines []GophermapSection
path string
lines []GophermapSection
mutex *sync.RWMutex
isFresh bool
lastRefresh int64
/* Implements */
File
}
func NewGophermapFile(path string) *GophermapFile {
f := new(GophermapFile)
f.path = path
f.mutex = new(sync.RWMutex)
return f
}
func (f *GophermapFile) Contents() []byte {
/* We don't just want to read the contents,
* but also execute any included gophermap
@ -81,9 +93,42 @@ func (f *GophermapFile) LoadContents() *GophorError {
/* Reload the file */
var gophorErr *GophorError
f.lines, gophorErr = f.readGophermap(f.path)
/* Update lastRefresh + set fresh */
f.lastRefresh = time.Now().UnixNano()
f.isFresh = true
return gophorErr
}
func (f *GophermapFile) IsFresh() bool {
return f.isFresh
}
func (f *GophermapFile) SetUnfresh() {
f.isFresh = false
}
func (f *GophermapFile) LastRefresh() int64 {
return f.lastRefresh
}
func (f *GophermapFile) Lock() {
f.mutex.Lock()
}
func (f *GophermapFile) Unlock() {
f.mutex.Unlock()
}
func (f *GophermapFile) RLock() {
f.mutex.RLock()
}
func (f *GophermapFile) RUnlock() {
f.mutex.RUnlock()
}
func (f *GophermapFile) readGophermap(path string) ([]GophermapSection, *GophorError) {
/* First, read raw file contents */
contents, gophorErr := bufferedRead(path)

3
gophor.go
View File

@ -88,6 +88,9 @@ func main() {
RegularCache = new(RegularFileCache)
RegularCache.Init()
/* Start file cache monitor */
StartFileMonitor(RegularCache, GophermapCache)
/* Serve unencrypted traffic */
go func() {
for {