refactor sync.Pool usages, improve file reading perf, trim leading '/' in request
Signed-off-by: kim (grufwub) <grufwub@gmail.com>
This commit is contained in:
parent
ac38ecce80
commit
529d28e62e
112
core/buffer.go
112
core/buffer.go
@ -1,112 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
connBufferedReaderPool sync.Pool
|
||||
connBufferedWriterPool sync.Pool
|
||||
fileBufferedReaderPool sync.Pool
|
||||
|
||||
fileReadBufferPool sync.Pool
|
||||
)
|
||||
|
||||
func newConnBufferedReaderPool(size int) sync.Pool {
|
||||
return sync.Pool{
|
||||
New: func() interface{} {
|
||||
return bufio.NewReaderSize(nil, size)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newConnBufferedWriterPool(size int) sync.Pool {
|
||||
return sync.Pool{
|
||||
New: func() interface{} {
|
||||
return bufio.NewWriterSize(nil, size)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newFileBufferedReaderPool(size int) sync.Pool {
|
||||
return sync.Pool{
|
||||
New: func() interface{} {
|
||||
return bufio.NewReaderSize(nil, size)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newFileReadBufferPool(size int) sync.Pool {
|
||||
return sync.Pool{
|
||||
New: func() interface{} {
|
||||
return make([]byte, size)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getConnBufferedReader(r io.Reader) *bufio.Reader {
|
||||
// Get buffered reader
|
||||
br := connBufferedReaderPool.Get().(*bufio.Reader)
|
||||
|
||||
// Reset to new reader
|
||||
br.Reset(r)
|
||||
|
||||
// Return!
|
||||
return br
|
||||
}
|
||||
|
||||
func putConnBufferedReader(br *bufio.Reader) {
|
||||
// Reset to ensure not hanging onto old reader
|
||||
br.Reset(nil)
|
||||
|
||||
// Put back in pool
|
||||
connBufferedReaderPool.Put(br)
|
||||
}
|
||||
|
||||
func getConnBufferedWriter(w io.Writer) *bufio.Writer {
|
||||
// Get buffered writer
|
||||
bw := connBufferedWriterPool.Get().(*bufio.Writer)
|
||||
|
||||
// Reset to new writer
|
||||
bw.Reset(w)
|
||||
|
||||
// Return!
|
||||
return bw
|
||||
}
|
||||
|
||||
func putConnBufferedWriter(bw *bufio.Writer) {
|
||||
// Reset to ensure not hanging onto old writer
|
||||
bw.Reset(nil)
|
||||
|
||||
// Put back in pool
|
||||
connBufferedWriterPool.Put(bw)
|
||||
}
|
||||
|
||||
func getFileBufferedReader(r io.Reader) *bufio.Reader {
|
||||
// Get buffered reader
|
||||
br := fileBufferedReaderPool.Get().(*bufio.Reader)
|
||||
|
||||
// Reset to new reader
|
||||
br.Reset(r)
|
||||
|
||||
// Return!
|
||||
return br
|
||||
}
|
||||
|
||||
func putFileBufferedReader(br *bufio.Reader) {
|
||||
// Reset to ensure not hanging onto old reader
|
||||
br.Reset(nil)
|
||||
|
||||
// Put back in pool
|
||||
fileBufferedReaderPool.Put(br)
|
||||
}
|
||||
|
||||
func getFileReadBuffer() []byte {
|
||||
return fileReadBufferPool.Get().([]byte)
|
||||
}
|
||||
|
||||
func putFileReadBuffer(b []byte) {
|
||||
fileReadBufferPool.Put(b)
|
||||
}
|
14
core/conn.go
14
core/conn.go
@ -51,8 +51,8 @@ type conn struct {
|
||||
func wrapConn(c net.Conn) *conn {
|
||||
deadlineConn := &deadlineConn{c}
|
||||
return &conn{
|
||||
br: getConnBufferedReader(deadlineConn),
|
||||
bw: getConnBufferedWriter(deadlineConn),
|
||||
br: connBufferedReaderPool.Get(deadlineConn),
|
||||
bw: connBufferedWriterPool.Get(deadlineConn),
|
||||
cl: deadlineConn,
|
||||
}
|
||||
}
|
||||
@ -60,9 +60,11 @@ func wrapConn(c net.Conn) *conn {
|
||||
// ReadLine reads a single line and returns the result, or nil and error
|
||||
func (c *conn) ReadLine() ([]byte, Error) {
|
||||
// return slice
|
||||
b := make([]byte, 0)
|
||||
var b []byte
|
||||
|
||||
// Read!
|
||||
// Read! Use this method so we can
|
||||
// ensure we don't perform some insanely
|
||||
// long read
|
||||
for len(b) < connReadMax {
|
||||
// read the line
|
||||
line, isPrefix, err := c.br.ReadLine()
|
||||
@ -117,8 +119,8 @@ func (c *conn) Close() Error {
|
||||
err2 := c.cl.Close()
|
||||
|
||||
// Put buffers back
|
||||
putConnBufferedReader(c.br)
|
||||
putConnBufferedWriter(c.bw)
|
||||
connBufferedReaderPool.Put(c.br)
|
||||
connBufferedWriterPool.Put(c.bw)
|
||||
|
||||
// If either errors, wrap. Else return none
|
||||
if err2 != nil {
|
||||
|
@ -102,79 +102,64 @@ func (fs *FileSystemObject) ReadFile(fd *os.File) ([]byte, Error) {
|
||||
// Return slice
|
||||
ret := make([]byte, 0)
|
||||
|
||||
// Read buffers
|
||||
buf := getFileReadBuffer()
|
||||
rd := getFileBufferedReader(fd)
|
||||
// Get read buffers, defer putting back
|
||||
br := fileBufferedReaderPool.Get(fd)
|
||||
defer fileBufferedReaderPool.Put(br)
|
||||
|
||||
// Read through file until null bytes / error
|
||||
for {
|
||||
count, err := rd.Read(buf)
|
||||
// Read line
|
||||
line, err := br.ReadBytes('\n')
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
// EOF, add current to return slice and
|
||||
// break-out. WIll not have hit delim
|
||||
ret = append(ret, line...)
|
||||
break
|
||||
} else {
|
||||
// Bad error, return
|
||||
return nil, WrapError(FileReadErr, err)
|
||||
}
|
||||
return nil, WrapError(FileReadErr, err)
|
||||
}
|
||||
|
||||
ret = append(ret, buf[:count]...)
|
||||
|
||||
if count < fileReadBufSize {
|
||||
break
|
||||
}
|
||||
// Add current line to return slice, skip
|
||||
// final byte which is '\n'
|
||||
ret = append(ret, line[:len(line)-1]...)
|
||||
}
|
||||
|
||||
// Put back buffers
|
||||
putFileReadBuffer(buf)
|
||||
putFileBufferedReader(rd)
|
||||
|
||||
// Return!
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// ScanFile scans a supplied file at file descriptor, using iterator function
|
||||
func (fs *FileSystemObject) ScanFile(fd *os.File, iterator func(string) bool) Error {
|
||||
// Read buffers
|
||||
rd := getFileBufferedReader(fd)
|
||||
// Get read buffer, defer putting back
|
||||
br := fileBufferedReaderPool.Get(fd)
|
||||
defer fileBufferedReaderPool.Put(br)
|
||||
|
||||
// Iterate through file!
|
||||
for {
|
||||
// Line buffer
|
||||
b := make([]byte, 0)
|
||||
|
||||
// Read until line-end, or file end!
|
||||
done := false
|
||||
for {
|
||||
// Read a line
|
||||
line, isPrefix, err := rd.ReadLine()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
done = true
|
||||
break
|
||||
}
|
||||
// Read a line
|
||||
line, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
// Reached end of file, perform final iteration
|
||||
// and break-out. Will not have hit delim
|
||||
iterator(line)
|
||||
break
|
||||
} else {
|
||||
// Bad error, return
|
||||
return WrapError(FileReadErr, err)
|
||||
}
|
||||
|
||||
// Append to line buffer
|
||||
b = append(b, line...)
|
||||
|
||||
// If not isPrefix, we can break-out
|
||||
if !isPrefix {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Run scan iterator on this line, break-out if requested
|
||||
if !iterator(string(b)) || done {
|
||||
// Run scan iterator on this line, breaking out if requested,
|
||||
// skipping final byte which is '\n'
|
||||
if !iterator(line[:len(line)-1]) {
|
||||
break
|
||||
}
|
||||
|
||||
// Empty the slice!
|
||||
b = nil
|
||||
}
|
||||
|
||||
// Put back buffers
|
||||
putFileBufferedReader(rd)
|
||||
|
||||
// Return no errors :)
|
||||
return nil
|
||||
}
|
||||
|
@ -4,8 +4,8 @@ var (
|
||||
// Root stores the server's root directory
|
||||
Root string
|
||||
|
||||
// BindAddr stores the server's bound IP
|
||||
BindAddr string
|
||||
// Bind stores the server's bound IP
|
||||
Bind string
|
||||
|
||||
// Hostname stores the host's outward hostname
|
||||
Hostname string
|
||||
|
108
core/pool.go
Normal file
108
core/pool.go
Normal file
@ -0,0 +1,108 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
connBufferedReaderPool *bufferedReaderPool
|
||||
connBufferedWriterPool *bufferedWriterPool
|
||||
fileBufferedReaderPool *bufferedReaderPool
|
||||
fileBufferPool *bufferPool
|
||||
)
|
||||
|
||||
type bufferPool struct {
|
||||
pool sync.Pool
|
||||
}
|
||||
|
||||
func newBufferPool(size int) *bufferPool {
|
||||
return &bufferPool{
|
||||
pool: sync.Pool{
|
||||
New: func() interface{} {
|
||||
return make([]byte, size)
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (bp *bufferPool) Get() []byte {
|
||||
// Just return and cast a buffer
|
||||
return bp.pool.Get().([]byte)
|
||||
}
|
||||
|
||||
func (bp *bufferPool) Put(b []byte) {
|
||||
// Just put back in pool
|
||||
bp.pool.Put(b)
|
||||
}
|
||||
|
||||
type bufferedReaderPool struct {
|
||||
pool sync.Pool
|
||||
}
|
||||
|
||||
func newBufferedReaderPool(size int) *bufferedReaderPool {
|
||||
return &bufferedReaderPool{
|
||||
pool: sync.Pool{
|
||||
New: func() interface{} {
|
||||
return bufio.NewReaderSize(nil, size)
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (bp *bufferedReaderPool) Get(r io.Reader) *bufio.Reader {
|
||||
// Get a buffered reader from the pool
|
||||
br := bp.pool.Get().(*bufio.Reader)
|
||||
|
||||
// Reset to use our new reader!
|
||||
br.Reset(r)
|
||||
|
||||
// Return
|
||||
return br
|
||||
}
|
||||
|
||||
func (bp *bufferedReaderPool) Put(br *bufio.Reader) {
|
||||
// We must reset again here to ensure
|
||||
// we don't mess with GC with unused underlying
|
||||
// reader.
|
||||
br.Reset(nil)
|
||||
|
||||
// Put back in the pool
|
||||
bp.pool.Put(br)
|
||||
}
|
||||
|
||||
type bufferedWriterPool struct {
|
||||
pool sync.Pool
|
||||
}
|
||||
|
||||
func newBufferedWriterPool(size int) *bufferedWriterPool {
|
||||
return &bufferedWriterPool{
|
||||
pool: sync.Pool{
|
||||
New: func() interface{} {
|
||||
return bufio.NewWriterSize(nil, size)
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (bp *bufferedWriterPool) Get(w io.Writer) *bufio.Writer {
|
||||
// Get a buffered writer from the pool
|
||||
bw := bp.pool.Get().(*bufio.Writer)
|
||||
|
||||
// Reset to user our new writer
|
||||
bw.Reset(w)
|
||||
|
||||
// Return
|
||||
return bw
|
||||
}
|
||||
|
||||
func (bp *bufferedWriterPool) Put(bw *bufio.Writer) {
|
||||
// We must reset again here to ensure
|
||||
// we don't mess with GC with unused underlying
|
||||
// writer.
|
||||
bw.Reset(nil)
|
||||
|
||||
// Put back in the pool
|
||||
bp.pool.Put(bw)
|
||||
}
|
@ -30,7 +30,7 @@ func ParseFlagsAndSetup(proto string, errorMessageFunc func(ErrorCode) string) {
|
||||
sysLog := flag.String(sysLogFlagStr, "stdout", sysLogDescStr)
|
||||
accLog := flag.String(accLogFlagStr, "stdout", accLogDescStr)
|
||||
flag.StringVar(&Root, rootFlagStr, "/var/gopher", rootDescStr)
|
||||
flag.StringVar(&BindAddr, bindAddrFlagStr, "", bindAddrDescStr)
|
||||
flag.StringVar(&Bind, bindFlagStr, "", bindDescStr)
|
||||
flag.StringVar(&Hostname, hostnameFlagStr, "localhost", hostnameDescStr)
|
||||
port := flag.Uint(portFlagStr, 70, portDescStr)
|
||||
fwdPort := flag.Uint(fwdPortFlagStr, 0, fwdPortDescStr)
|
||||
@ -73,10 +73,10 @@ func ParseFlagsAndSetup(proto string, errorMessageFunc func(ErrorCode) string) {
|
||||
|
||||
// Check valid values for BindAddr and Hostname
|
||||
if Hostname == "" {
|
||||
if BindAddr == "" {
|
||||
SystemLog.Fatal(hostnameBindAddrEmptyStr)
|
||||
if Bind == "" {
|
||||
SystemLog.Fatal(hostnameBindEmptyStr)
|
||||
}
|
||||
Hostname = BindAddr
|
||||
Hostname = Bind
|
||||
}
|
||||
|
||||
// Change to server directory
|
||||
@ -97,16 +97,16 @@ func ParseFlagsAndSetup(proto string, errorMessageFunc func(ErrorCode) string) {
|
||||
|
||||
// Setup listener
|
||||
var err Error
|
||||
serverListener, err = newListener(BindAddr, Port)
|
||||
serverListener, err = newListener(Bind, Port)
|
||||
if err != nil {
|
||||
SystemLog.Fatal(listenerBeginFailStr, protocol, Hostname, FwdPort, BindAddr, Port, err.Error())
|
||||
SystemLog.Fatal(listenerBeginFailStr, protocol, Hostname, FwdPort, Bind, Port, err.Error())
|
||||
}
|
||||
|
||||
// Setup the sync pools
|
||||
connBufferedReaderPool = newConnBufferedReaderPool(int(*cReadBuf))
|
||||
connBufferedWriterPool = newConnBufferedWriterPool(int(*cWriteBuf))
|
||||
fileBufferedReaderPool = newFileBufferedReaderPool(int(*fReadBuf))
|
||||
fileReadBufferPool = newFileReadBufferPool(int(*fReadBuf))
|
||||
connBufferedReaderPool = newBufferedReaderPool(int(*cReadBuf))
|
||||
connBufferedWriterPool = newBufferedWriterPool(int(*cWriteBuf))
|
||||
fileBufferedReaderPool = newBufferedReaderPool(int(*fReadBuf))
|
||||
fileBufferPool = newBufferPool(int(*fReadBuf))
|
||||
|
||||
// Conn read max
|
||||
connReadMax = int(*cReadMax)
|
||||
@ -197,7 +197,7 @@ func Start(serve func(*Client)) {
|
||||
go FileSystem.StartMonitor()
|
||||
|
||||
// Start the listener
|
||||
SystemLog.Info(listeningOnStr, protocol, Hostname, FwdPort, BindAddr, Port)
|
||||
SystemLog.Info(listeningOnStr, protocol, Hostname, FwdPort, Bind, Port)
|
||||
go func() {
|
||||
for {
|
||||
client, err := serverListener.Accept()
|
||||
|
@ -11,8 +11,8 @@ const (
|
||||
rootFlagStr = "root"
|
||||
rootDescStr = "Server root directory"
|
||||
|
||||
bindAddrFlagStr = "bind"
|
||||
bindAddrDescStr = "IP address to bind to"
|
||||
bindFlagStr = "bind"
|
||||
bindDescStr = "IP address to bind to"
|
||||
|
||||
hostnameFlagStr = "hostname"
|
||||
hostnameDescStr = "Server hostname (FQDN)"
|
||||
@ -83,7 +83,7 @@ const (
|
||||
|
||||
// Log string constants
|
||||
const (
|
||||
hostnameBindAddrEmptyStr = "At least one of hostname or bind-addr must be non-empty!"
|
||||
hostnameBindEmptyStr = "At least one of hostname or bind-addr must be non-empty!"
|
||||
|
||||
chDirStr = "Entered server dir: %s"
|
||||
chDirErrStr = "Error entering server directory: %s"
|
||||
|
@ -16,8 +16,8 @@ func serve(client *core.Client) {
|
||||
return
|
||||
}
|
||||
|
||||
// Convert to string
|
||||
line := string(received)
|
||||
// Convert to string + remove leading '/'
|
||||
line := strings.TrimPrefix(string(received), "/")
|
||||
|
||||
// If prefixed by 'URL:' send a redirect
|
||||
lenBefore := len(line)
|
||||
|
Loading…
Reference in New Issue
Block a user