separate out into separate files, first work towards file caching system

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

cache.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+    "sync"
+    "fmt"
+)
+
+type File interface {
+    Contents()     []byte
+    LoadContents() *GophorError
+}
+
+type RegularFileCache struct {
+    CacheMap map[string]*RegularFile
+    Mutex    sync.RWMutex
+}
+
+func (fc *RegularFileCache) Init() {
+    fc.CacheMap = make(map[string]*RegularFile)
+    fc.Mutex = sync.RWMutex{}
+}
+
+func (fc *RegularFileCache) Fetch(path string) ([]byte, *GophorError) {
+    /* Try get file */
+    file, ok := fc.GetFile(path)
+    
+    if !ok {
+        /* File not in cache, we need to load the file */
+        var gophorErr *GophorError
+        file, gophorErr = fc.Load(path)
+        if gophorErr != nil {
+            return nil, gophorErr
+        }
+    }
+
+    fmt.Println("fetching file:", path)
+    return file.Contents(), nil
+}
+
+func (fc *RegularFileCache) GetFile(path string) (*RegularFile, bool) {
+    /* Get read lock, try get file from the cache */
+    fc.Mutex.RLock()
+    file, ok := fc.CacheMap[path]
+    fc.Mutex.RUnlock()
+    return file, ok
+}
+
+func (fc *RegularFileCache) Load(path string) (*RegularFile, *GophorError) {
+    /* Create new file object for path, load contents */
+    file := new(RegularFile)
+    file.path = path
+
+    gophorErr := file.LoadContents()
+    if gophorErr != nil {
+        return nil, gophorErr
+    }
+
+    /* Get lock, add file object to cache */
+    fc.Mutex.Lock()
+    fc.CacheMap[path] = file
+    fc.Mutex.Unlock()
+
+    return file, nil
+}
+
+type GophermapFileCache struct {
+    CacheMap map[string]*GophermapFile
+    Mutex    sync.RWMutex
+}
+
+func (fc *GophermapFileCache) Init() {
+    fc.CacheMap = make(map[string]*GophermapFile)
+    fc.Mutex = sync.RWMutex{}
+}
+
+func (fc *GophermapFileCache) Fetch(path string) ([]byte, *GophorError) {
+    /* Try get file */
+    file, ok := fc.GetFile(path)
+    
+    if !ok {
+        /* File not in cache, we need to load the file */
+        var gophorErr *GophorError
+        file, gophorErr = fc.Load(path)
+        if gophorErr != nil {
+            return nil, gophorErr
+        }
+    }
+
+    return file.Contents(), nil
+}
+
+func (fc *GophermapFileCache) GetFile(path string) (*GophermapFile, bool) {
+    /* Get read lock, try get file from the cache */
+    fc.Mutex.RLock()
+    file, ok := fc.CacheMap[path]
+    fc.Mutex.RUnlock()
+    return file, ok
+}
+
+func (fc *GophermapFileCache) Load(path string) (*GophermapFile, *GophorError) {
+    /* Create new file object for path, load contents */
+    file := new(GophermapFile)
+    file.path = path
+
+    gophorErr := file.LoadContents()
+    if gophorErr != nil {
+        return nil, gophorErr
+    }
+
+    /* Get lock, add file object to cache. */
+    fc.Mutex.Lock()
+    fc.CacheMap[path] = file
+    fc.Mutex.Unlock()
+
+    return file, nil
+}

dir.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+    "os"
+    "path"
+    "strings"
+)
+
+func listDir(dirPath string, hidden map[string]bool) ([]byte, *GophorError) {
+    /* Open directory file descriptor */
+    fd, err := os.Open(dirPath)
+    if err != nil {
+        logSystemError("failed to open %s: %s\n", dirPath, err.Error())
+        return nil, &GophorError{ FileOpenErr, err }
+    }
+
+    /* Open directory stream for reading */
+    files, err := fd.Readdir(-1)
+    if err != nil {
+        logSystemError("failed to enumerate dir %s: %s\n", dirPath, err.Error())
+        return nil, &GophorError{ DirListErr, err }
+    }
+
+    var entity *DirEntity
+    dirContents := make([]byte, 0)
+
+    /* Walk through directory */
+    for _, file := range files {
+        /* Skip dotfiles + gophermap file + requested hidden */
+        if file.Name()[0] == '.' || strings.HasSuffix(file.Name(), GophermapFileStr) {
+            continue
+        } else if _, ok := hidden[file.Name()]; ok {
+            continue
+        }
+
+        /* Handle file, directory or ignore others */
+        switch {
+            case file.Mode() & os.ModeDir != 0:
+                /* Directory -- create directory listing */
+                itemPath := path.Join(fd.Name(), file.Name())
+                entity = newDirEntity(TypeDirectory, file.Name(), "/"+itemPath, *ServerHostname, *ServerPort)
+                dirContents = append(dirContents, entity.Bytes()...)
+
+            case file.Mode() & os.ModeType == 0:
+                /* Regular file -- find item type and creating listing */
+                itemPath := path.Join(fd.Name(), file.Name())
+                itemType := getItemType(itemPath)
+                entity = newDirEntity(itemType, file.Name(), "/"+itemPath, *ServerHostname, *ServerPort)
+                dirContents = append(dirContents, entity.Bytes()...)
+
+            default:
+                /* Ignore */
+        }
+    }
+
+    return dirContents, nil
+}

file.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+    "os"
+    "io"
+    "bufio"
+)
+
+type RegularFile struct {
+    path     string
+    contents []byte
+
+    /* Implements */
+    File
+}
+
+func (f RegularFile) Contents() []byte {
+    return f.contents
+}
+
+func (f RegularFile) LoadContents() *GophorError {
+    /* Clear current cache */
+    f.contents = nil
+
+    /* Reload the file */
+    var gophorErr *GophorError
+    f.contents, gophorErr = bufferedRead(f.path)
+    return gophorErr
+}
+
+func bufferedRead(path string) ([]byte, *GophorError) {
+    /* Open file */
+    fd, err := os.Open(path)
+    if err != nil {
+        logSystemError("failed to open %s: %s\n", path, err.Error())
+        return nil, &GophorError{ FileOpenErr, err }
+    }
+    defer fd.Close()
+
+    /* Setup buffers */
+    var count int
+    contents := make([]byte, 0)
+    buf := make([]byte, FileReadBufSize)
+
+    /* Setup reader */
+    reader := bufio.NewReader(fd)
+
+    /* Read through buffer until error or null bytes! */
+    for {
+        count, err = reader.Read(buf)
+        if err != nil {
+            if err == io.EOF {
+                break
+            }
+
+            logSystemError("failed to read %s: %s\n", path, err.Error())
+            return nil, &GophorError{ FileReadErr, err }
+        }
+
+        contents = append(contents, buf[:count]...)
+
+        if count < FileReadBufSize {
+            break
+        }
+    }
+
+    return contents, nil
+}

file_gophermap.go

@@ -0,0 +1,233 @@
+package main
+
+import (
+    "bufio"
+    "bytes"
+    "strings"
+)
+
+const GophermapFileStr = "gophermap"
+
+type GophermapSection interface {
+    Render() ([]byte, *GophorError)
+}
+
+type GophermapText struct {
+    contents []byte
+
+    /* Implements */
+    GophermapSection
+}
+
+func NewGophermapText(contents string) *GophermapText {
+    s := new(GophermapText)
+    s.contents = []byte(contents)
+    return s
+}
+
+func (s GophermapText) Render() ([]byte, *GophorError) {
+    return s.contents, nil
+}
+
+type GophermapDirListing struct {
+    path   string
+    Hidden map[string]bool
+
+    /* Implements */
+    GophermapSection
+}
+
+func NewGophermapDirListing(path string) *GophermapDirListing {
+    s := new(GophermapDirListing)
+    s.path = path
+    return s
+}
+
+func (s GophermapDirListing) Render() ([]byte, *GophorError) {
+    return listDir(s.path, s.Hidden)
+}
+
+type GophermapFile struct {
+    path     string
+    lines    []GophermapSection
+
+    /* Implements */
+    File
+}
+
+func (f GophermapFile) Contents() []byte {
+    /* We don't just want to read the contents,
+     * but also execute any included gophermap
+     * execute lines.
+     */
+    logSystem("Sections: %s\n", f.lines)
+    contents := make([]byte, 0)
+    for _, line := range f.lines {
+        content, gophorErr := line.Render()
+        if gophorErr != nil {
+            content = []byte(string(TypeInfo)+"Error rendering gophermap section."+CrLf)
+        }
+        contents = append(contents, content...)
+    }
+
+    return contents
+}
+
+func (f GophermapFile) LoadContents() *GophorError {
+    /* Clear the current cache */
+    f.lines = nil
+
+    logSystem("Loading gophermap...\n")
+
+    /* Reload the file */
+    f.lines = make([]GophermapSection, 0)
+    lines, gophorErr := f.readGophermap(f.path)
+    f.lines = append(f.lines, lines...)
+    for _, line := range f.lines {
+        renderStr, _ := line.Render()
+        logSystem("%s\n", renderStr)
+    }
+
+    return gophorErr
+}
+
+func (f *GophermapFile) readGophermap(path string) ([]GophermapSection, *GophorError) {
+    /* First, read raw file contents */
+    contents, gophorErr := bufferedRead(path)
+    if gophorErr != nil {
+        return nil, gophorErr
+    }
+
+    /* Create reader and scanner from this */
+    reader := bytes.NewReader(contents)
+    scanner := bufio.NewScanner(reader)
+
+    /* Setup scanner to split on CrLf */
+    scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
+        if atEOF && len(data) == 0  {
+            /* At EOF, no more data */
+            return 0, nil, nil
+        }
+
+        if i := bytes.Index(data, []byte{ '\r', '\n' }); i >= 0 {
+            /* We have a full new-line terminate line */
+            return i+2, data[0:i], nil
+        }
+
+        /* Request more data */
+        return 0, nil, nil
+    })
+
+    /* Create return slice + hidden files map in case dir listing requested */
+    sections := make([]GophermapSection, 0)
+    hidden := make(map[string]bool)
+    var dirListing *GophermapDirListing
+
+    /* Scan, format each token and add to parsedContents */
+    doEnd := false
+    for scanner.Scan() {
+        line := scanner.Text()
+
+        /* Parse the line item type and handle */
+        lineType := parseLineType(line)
+        switch lineType {
+            case TypeInfoNotStated:
+                /* Append TypeInfo to the beginning of line */
+                sections = append(sections, NewGophermapText(string(TypeInfo)+line+CrLf))
+
+            case TypeComment:
+                /* We ignore this line */
+                continue
+
+            case TypeHiddenFile:
+                /* Add to hidden files map */
+                hidden[line[1:]] = true
+
+            case TypeSubGophermap:
+                /* Check if we've been supplied subgophermap or regular file */
+                if strings.HasSuffix(line[1:], GophermapFileStr) {
+                    /* Ensure we haven't been passed the current gophermap. Recursion bad! */
+                    if line[1:] == path {
+                        continue
+                    }
+
+                    /* Treat as any other gopher map! */
+                    submapSections, gophorErr := f.readGophermap(line[1:])
+                    if gophorErr != nil {
+                        /* Failed to read subgophermap, insert error line */
+                        sections = append(sections, NewGophermapText(string(TypeInfo)+"Error reading subgophermap: "+line[1:]+CrLf))
+                    } else {
+                        sections = append(sections, submapSections...)
+                    }
+                } else {
+                    /* Treat as regular file, but we need to replace Unix line endings
+                     * with gophermap line endings
+                     */
+                    fileContents, gophorErr := bufferedRead(line[1:])
+                    if gophorErr != nil {
+                        /* Failed to read file, insert error line */
+                        sections = append(sections, NewGophermapText(string(TypeInfo)+"Error reading subgophermap: "+line[1:]+CrLf))
+                    } else {
+                        /* Replace line endings with CrLf */
+                        fileContents = bytes.Replace(fileContents, []byte("\n"), []byte(CrLf), -1)
+                        if !strings.HasSuffix(line, CrLf) {
+                            /* Ensure we end on CrLf */
+                            line += CrLf
+                        }
+                        sections = append(sections, NewGophermapText(string(fileContents)))
+                    }
+                }
+
+            case TypeExec:
+                /* Try executing supplied line */
+                sections = append(sections, NewGophermapText(string(TypeInfo)+"Error: inline shell commands not yet supported"+CrLf))
+/*
+                err := exec.Command(line[1:]).Run()
+                if err != nil {
+                    line = fmt.Sprintf(string(TypeInfo)+"Error executing command: %s"+CrLf, line[1:])
+                } else {
+                    line = strings.Replace(string(""), "\n", CrLf, -1)
+                    if !strings.HasSuffix(line, CrLf) {
+                        line += CrLf
+                    }
+                }
+*/
+
+            case TypeEnd:
+                /* Lastline, break out at end of loop. Interface method Contents()
+                 * will append a last line at the end so we don't have to worry about
+                 * that here, only stopping the loop.
+                 */
+                doEnd = true
+
+            case TypeEndBeginList:
+                /* Create GophermapDirListing object then break out at end of loop */
+                doEnd = true
+                dirListing = NewGophermapDirListing(line[1:])
+
+            default:
+                sections = append(sections, NewGophermapText(line+CrLf))
+        }
+
+        /* Break out of read loop if requested */
+        if doEnd {
+            break
+        }
+    }
+
+    /* If scanner didn't finish cleanly, return nil and error */
+    if scanner.Err() != nil {
+        return nil, &GophorError{ FileReadErr, scanner.Err() }
+    }
+
+    /* If dir listing requested, append the hidden files map then add
+     * to sections slice. We can do this here as the TypeEndBeginList item
+     * type ALWAYS comes last, at least in the gophermap handled by this context.
+     */
+    if dirListing != nil {
+        dirListing.Hidden = hidden
+        sections = append(sections, dirListing)
+    }
+
+    return sections, nil
+}

gophor.go

@@ -42,6 +42,10 @@ var (
     SystemLog      = flag.String("system-log", "", "Change server system log file (blank outputs to stderr).")
     AccessLog      = flag.String("access-log", "", "Change server access log file (blank outputs to stderr).")
     LoggingType    = flag.Int("log-type", 0, "Change server log file handling -- 0:default 1:disable")
+
+    /* FileCaches */
+    GophermapCache *GophermapFileCache
+    RegularCache   *RegularFileCache
 )
 
 func main() {
@@ -78,6 +82,12 @@ func main() {
     signals := make(chan os.Signal)
     signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
 
+    /* Create file caches */
+    GophermapCache = new(GophermapFileCache)
+    GophermapCache.Init()
+    RegularCache = new(RegularFileCache)
+    RegularCache.Init()
+
     /* Serve unencrypted traffic */
     go func() {
         for {

worker.go

@@ -3,20 +3,15 @@ package main
 import (
     "fmt"
     "os"
-    "io"
     "net"
-    "bufio"
     "path"
     "strings"
-    "bytes"
 )
 
 const (
-    ShowHidden          = false
     SocketReadBufSize   = 512
     MaxSocketReadChunks = 4
     FileReadBufSize     = 512
-    GopherMapFile       = "/gophermap"
     DefaultShell        = "/bin/sh"
 )
 
@@ -152,7 +147,6 @@ func (worker *Worker) Respond(data []byte) *GophorError {
         worker.SendErrorType("read fail\n") /* Purposely vague errors */
         return &GophorError{ FileOpenErr, err }
     }
-    defer file.Close()
 
     /* Leads to some more concise code below */
     type FileType int
@@ -181,6 +175,9 @@ func (worker *Worker) Respond(data []byte) *GophorError {
         }
     }
 
+    /* Don't need the file handle anymore */
+    file.Close()
+
     /* Handle file type */
     var response []byte
     var gophorErr *GophorError
@@ -188,43 +185,33 @@ func (worker *Worker) Respond(data []byte) *GophorError {
         /* Directory */
         case Dir:
             /* First try to serve gopher map */
-            requestPath = path.Join(requestPath, GopherMapFile)
-            mapFile, err := os.Open(requestPath)
-            defer mapFile.Close()
-
-            if err == nil {
-                /* Read GopherMapFile contents */
-                worker.Log("serve gophermap: /%s\n", requestPath)
-
-                response, gophorErr = worker.ReadGophermap(file, mapFile)
+            requestPath = path.Join(requestPath, "/"+GophermapFileStr)
+            response, gophorErr := GophermapCache.Fetch(requestPath)
+            if gophorErr != nil {
+                /* Get directory listing instead */
+                response, gophorErr = listDir(requestPath, map[string]bool{})
                 if gophorErr != nil {
-                    worker.SendErrorType("gophermap read fail\n")
+                    worker.SendErrorType("dir list failed\n")
                     return gophorErr
                 }
-            } else {
-                /* Get directory listing */
                 worker.Log("serve dir: /%s\n", requestPath)
 
-                response, gophorErr = worker.ListDir(file)
-                if gophorErr != nil {
-                    worker.SendErrorType("dir list fail\n")
-                    return gophorErr
-                }
-
                 /* Finish directory listing with LastLine */
                 response = append(response, []byte(LastLine)...)
+            } else {
+                /* Successfully loaded gophermap, log */
+                worker.Log("server gophermap: /%s\n", requestPath)
             }
 
         /* Regular file */
         case File:
             /* Read file contents */
-            worker.Log("%s serve file: /%s\n", requestPath)
-
-            response, gophorErr = worker.ReadFile(file)
+            response, gophorErr = RegularCache.Fetch(requestPath)
             if gophorErr != nil {
                 worker.SendErrorText("file read fail\n")
                 return gophorErr
             }
+            worker.Log("%s serve file: /%s\n", requestPath)
 
         /* Unsupport file type */
         default:
@@ -234,195 +221,3 @@ func (worker *Worker) Respond(data []byte) *GophorError {
     /* Serve response */
     return worker.SendRaw(response)
 }
-
-func (worker *Worker) ReadGophermap(dir, mapFile *os.File) ([]byte, *GophorError) {
-    fileContents := make([]byte, 0)
-
-    /* Create reader and scanner from this */
-    reader := bufio.NewReader(mapFile)
-    scanner := bufio.NewScanner(reader)
-
-    /* Setup scanner to split on CrLf */
-    scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
-        if atEOF && len(data) == 0  {
-            /* At EOF, no more data */
-            return 0, nil, nil
-        }
-
-        if i := bytes.Index(data, []byte{ '\r', '\n' }); i >= 0 {
-            /* We have a full new-line terminate line */
-            return i+2, data[0:i], nil
-        }
-
-        /* Request more data */
-        return 0, nil, nil
-    })
-
-    /* Scan, format each token and add to fileContents */
-    doEnd := false
-    for scanner.Scan() {
-        line := scanner.Text()
-
-        /* Parse the line item type and handle */
-        lineType := parseLineType(line)
-        switch lineType {
-            case TypeInfoNotStated:
-                /* Append TypeInfo to the beginning of line */
-                line = string(TypeInfo)+line+CrLf
-
-            case TypeComment:
-                /* We ignore this line */
-                continue
-
-            case TypeHiddenFile:
-                /* Add to hidden files map */
-                worker.Hidden[line[1:]] = true
-
-            case TypeSubGophermap:
-                /* Try to read subgophermap of file name */
-                line = string(TypeInfo)+"Error: subgophermaps not supported"+CrLf
-
-/*
-                subMapFile, err := os.Open(line[1:])
-                if err != nil {
-                    worker.LogError("error opening subgophermap: /%s --> %s\n", mapFile.Name(), line[1:])
-                    line = fmt.Sprintf(string(TypeInfo)+"Error reading subgophermap: %s"+CrLf, line[1:])
-                } else {
-                    subMapContent, gophorError := worker.ReadFile(subMapFile)
-                    if gophorError != nil {
-                        worker.LogError("error reading subgophermap: /%s --> %s\n", mapFile.Name(), line[1:])
-                        line = fmt.Sprintf(string(TypeInfo)+"Error reading subgophermap: %s"+CrLf, line[1:])
-                    } else {
-                        line = strings.Replace(string(subMapContent), "\n", CrLf, -1)
-                        if !strings.HasSuffix(line, CrLf) {
-                            line += CrLf
-                        }
-                    }
-                }
-*/
-
-            case TypeExec:
-                /* Try executing supplied line */
-                line = string(TypeInfo)+"Error: inline shell commands not support"+CrLf
-
-/*
-                err := exec.Command(line[1:]).Run()
-                if err != nil {
-                    line = fmt.Sprintf(string(TypeInfo)+"Error executing command: %s"+CrLf, line[1:])
-                } else {
-                    line = strings.Replace(string(""), "\n", CrLf, -1)
-                    if !strings.HasSuffix(line, CrLf) {
-                        line += CrLf
-                    }
-                }
-*/
-
-            case TypeEnd:
-                /* Lastline, break out at end of loop */
-                doEnd = true
-                line = LastLine
-
-            case TypeEndBeginList:
-                /* Read current directory listing then break out at end of loop */
-                doEnd = true
-                dirListing, gophorErr := worker.ListDir(dir)
-                if gophorErr != nil {
-                    return nil, gophorErr
-                }
-                line = string(dirListing) + LastLine
-
-            default:
-                line += CrLf
-        }
-
-        /* Append generated line to total fileContents */
-        fileContents = append(fileContents, []byte(line)...)
-
-        /* Break out of read loop if requested */
-        if doEnd {
-            break
-        }
-    }
-
-    /* If scanner didn't finish cleanly, return nil and error */
-    if scanner.Err() != nil {
-        return nil, &GophorError{ FileReadErr, scanner.Err() }
-    }
-
-    /* If we never hit doEnd, append a LastLine ourselves */
-    if !doEnd {
-        fileContents = append(fileContents, []byte(LastLine)...)
-    }
-    
-    return fileContents, nil
-}
-
-func (worker *Worker) ReadFile(file *os.File) ([]byte, *GophorError) {
-    var count int
-    fileContents := make([]byte, 0)
-    buf := make([]byte, FileReadBufSize)
-
-    var err error
-    reader := bufio.NewReader(file)
-
-    for {
-        count, err = reader.Read(buf)
-        if err != nil && err != io.EOF {
-            return nil, &GophorError{ FileReadErr, err }
-        }
-
-        for i := 0; i < count; i += 1 {
-            if buf[i] == 0 {
-                break
-            }
-
-            fileContents = append(fileContents, buf[i])
-        }
-
-        if count < FileReadBufSize {
-            break
-        }
-    }
-
-    return fileContents, nil
-}
-
-func (worker *Worker) ListDir(dir *os.File) ([]byte, *GophorError) {
-    files, err := dir.Readdir(-1)
-    if err != nil {
-        return nil, &GophorError{ DirListErr, err }
-    }
-
-    var entity *DirEntity
-    dirContents := make([]byte, 0)
-
-    for _, file := range files {
-        /* Skip dotfiles + gophermap file + requested hidden */
-        if file.Name()[0] == '.' || file.Name() == "gophermap" {
-            continue
-        } else if _, ok := worker.Hidden[file.Name()]; ok {
-            continue
-        }
-
-        /* Handle file, directory or ignore others */
-        switch {
-            case file.Mode() & os.ModeDir != 0:
-                /* Directory -- create directory listing */
-                itemPath := path.Join(dir.Name(), file.Name())
-                entity = newDirEntity(TypeDirectory, file.Name(), "/"+itemPath, *ServerHostname, *ServerPort)
-                dirContents = append(dirContents, entity.Bytes()...)
-
-            case file.Mode() & os.ModeType == 0:
-                /* Regular file -- find item type and creating listing */
-                itemPath := path.Join(dir.Name(), file.Name())
-                itemType := getItemType(itemPath)
-                entity = newDirEntity(itemType, file.Name(), "/"+itemPath, *ServerHostname, *ServerPort)
-                dirContents = append(dirContents, entity.Bytes()...)
-
-            default:
-                /* Ignore */
-        }
-    }
-
-    return dirContents, nil
-}