working mariadb access + redis

This commit is contained in:
Chakib Benziane 2019-03-24 22:36:28 +01:00
parent 0145558cc8
commit f880881b9b
6 changed files with 117 additions and 61 deletions

View File

@ -3,28 +3,23 @@ package db
import (
"fmt"
"log"
"net/url"
"os"
"path/filepath"
"git.sp4ke.com/sp4ke/bit4sat/utils"
_ "github.com/go-sql-driver/mysql"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
"github.com/mediocregopher/radix/v3"
)
const (
DBName = "bit4sat.sqlite"
DBPath = "db-storage"
DBPathEnv = "BIT4SAT_DB_PATH"
DBPragma = ` PRAGMA foreign_keys = ON; `
//DBName = "bit4sat.sqlite"
DBName = "bit4sat"
DBHostEnv = "DB_HOST"
DBUserEnv = "DB_USER"
DBPassEnv = "DB_PASS"
//DBPragma = ` PRAGMA foreign_keys = ON; `
)
var (
DBOptions = map[string]string{
"_journal_mode": "WAL",
}
DB *Database
)
@ -34,38 +29,38 @@ type Database struct {
}
func (d *Database) Open() error {
var err error
dsnOptions := &url.Values{}
for k, v := range DBOptions {
dsnOptions.Set(k, v)
}
// Get db base path
path, set := os.LookupEnv(DBPathEnv)
// Get env vars
host, set := os.LookupEnv(DBHostEnv)
if !set {
path = DBPath
log.Fatal("undefined DB_HOST env")
}
// Create path if not exists
err := utils.Mkdir(path)
if err != nil {
log.Fatal(err)
user, set := os.LookupEnv(DBUserEnv)
if !set {
log.Fatal("undefined DB_USER env")
}
path = filepath.Join(path, DBName)
//path = fmt.Sprintf("%s/%s", path, DBName)
dsn := fmt.Sprintf("file:%s?%s", path, dsnOptions.Encode())
log.Printf("Opening sqlite db %s\n", dsn)
d.Sql, err = sqlx.Open("sqlite3", dsn)
if err != nil {
log.Fatal(err)
pass, set := os.LookupEnv(DBPassEnv)
if !set {
log.Fatal("undefined DB_PASS env")
}
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s", user, pass, host, DBName)
log.Printf("Opening SQL at %s\n", dsn)
d.Sql = sqlx.MustConnect("mysql", dsn)
// Execute Pragmas
d.Sql.MustExec(DBPragma)
//d.Sql.MustExec(DBPragma)
// Opend redis
d.Redis, err = radix.NewPool("tcp", "redis:6379", 10)
if err != nil {
log.Fatal(err)
}
return nil
}
@ -86,9 +81,4 @@ func init() {
DB = &Database{}
DB.Open()
var err error
DB.Redis, err = radix.NewPool("tcp", "redis:6379", 10)
if err != nil {
log.Fatal(err)
}
}

View File

@ -2,8 +2,9 @@ version: "3.4"
volumes:
db:
maria-conf:
file-storage:
sqlite:
#sqlite:
gocache:
services:
@ -12,9 +13,11 @@ services:
build: ./docker
environment:
- GO111MODULE=on
- BIT4SAT_DB_PATH=/sqlite
- BIT4SAT_STORAGE_PATH=/storage
- GOPATH=/go
- DB_HOST=maria
- DB_USER=bit4sat
- DB_PASS=bit4sat
#deploy:
#replicas: 1
@ -25,7 +28,7 @@ services:
volumes:
- $PWD:/src
- gocache:/go
- ./db-storage:/sqlite
#- ./db-storage:/sqlite
- file-storage:/storage
working_dir: /src
@ -40,12 +43,13 @@ services:
volumes:
- db:/var/lib/mysql
#redis:
#image: redis:alpine
- maria-conf:/etc/mysql
redis:
image: redis:alpine
#volumes:
#- db:/data
volumes:
- db:/data
#command:
#- redis-server
#- --appendonly yes
command:
- redis-server
- --appendonly yes

View File

@ -6,7 +6,6 @@ RUN apk update && apk add --no-cache \
gcc build-base make
RUN mkdir -p /src
RUN mkdir -p /sqlite
WORKDIR /src
ENV GO111MODULE=on

10
go.mod
View File

@ -5,8 +5,18 @@ go 1.12
require (
github.com/gin-contrib/cors v0.0.0-20190301062745-f9e10995c85a
github.com/gin-gonic/gin v1.3.0
github.com/go-sql-driver/mysql v1.4.1
github.com/golang/protobuf v1.3.1 // indirect
github.com/jmoiron/sqlx v1.2.0
github.com/kr/pretty v0.1.0 // indirect
github.com/mattn/go-isatty v0.0.7 // indirect
github.com/mattn/go-sqlite3 v1.10.0
github.com/mediocregopher/radix/v3 v3.2.3
github.com/segmentio/ksuid v1.0.2
github.com/stretchr/testify v1.3.0 // indirect
github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780 // indirect
golang.org/x/net v0.0.0-20190322120337-addf6b3196f6 // indirect
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc // indirect
google.golang.org/appengine v1.5.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
)

42
go.sum
View File

@ -1,3 +1,5 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gin-contrib/cors v0.0.0-20190301062745-f9e10995c85a h1:zBycVvXa03SIX+jdMv8wGu9TMDMWdN8EhaR1FoeKHNo=
github.com/gin-contrib/cors v0.0.0-20190301062745-f9e10995c85a/go.mod h1:pL2kNE+DgDU+eQ+dary5bX0Z6LPP8nR6Mqs1iejILw4=
@ -6,32 +8,72 @@ github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NB
github.com/gin-gonic/gin v1.3.0 h1:kCmZyPklC0gVdL728E6Aj20uYBJV93nj/TkwBTKhFbs=
github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed h1:3dQJqqDouawQgl3gBE1PNHKFkJYGEuFb1DbSlaxdosE=
github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
github.com/mediocregopher/radix/v3 v3.2.3 h1:TbcGCZdo9zfPYPgevsqRn+OjvCyfOK6TzuXhqzWdCt0=
github.com/mediocregopher/radix/v3 v3.2.3/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/segmentio/ksuid v1.0.2 h1:9yBfKyw4ECGTdALaF09Snw3sLJmYIX6AbPJrAy6MrDc=
github.com/segmentio/ksuid v1.0.2/go.mod h1:BXuJDr2byAiHuQaQtSKoXh1J0YmUDurywOXgB2w+OSU=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/ugorji/go v1.1.2 h1:JON3E2/GPW2iDNGoSAusl1KDf5TRQ8k8q7Tp097pZGs=
github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2 h1:EICbibRW4JNKMcY+LsWmuwob+CRS1BmdRdjphAm9mH4=
github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780 h1:vG/gY/PxA3v3l04qxe3tDjXyu3bozii8ulSlIPOYKhI=
github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190322120337-addf6b3196f6 h1:78jEq2G3J16aXneH23HSnTQQTCwMHoyO8VEiUH+bpPM=
golang.org/x/net v0.0.0-20190322120337-addf6b3196f6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc h1:4gbWbmmPFp4ySWICouJl6emP0MyS31yy9SrTlAGFT+g=
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ=
gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=

View File

@ -14,7 +14,8 @@ var DB = db.DB
const (
DBUploadSchema = `
CREATE TABLE IF NOT EXISTS "upload" (
CREATE TABLE IF NOT EXISTS upload (
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
upload_id CHAR(20) NOT NULL,
sha256 CHAR(32) NOT NULL,
file_name CHAR(255) NOT NULL,
@ -28,14 +29,24 @@ const (
`
DBUploadView = `
CREATE VIEW IF NOT EXISTS "upload_with_status" AS SELECT * FROM upload JOIN upload_status ON
CREATE VIEW IF NOT EXISTS upload_with_status
AS SELECT
upload.upload_id,
upload.sha256,
upload_status.status,
upload.file_name,
upload.file_type,
upload.file_size,
upload.file_ext
FROM upload
JOIN upload_status ON
upload.status = upload_status.type
`
DBUploadStatusSchema = `CREATE TABLE IF NOT EXISTS upload_status
(
type INTEGER PRIMARY KEY,
status TEXT NOT NULL UNIQUE
type INT NOT NULL PRIMARY KEY,
status CHAR(255) NOT NULL UNIQUE
)
`
@ -155,7 +166,12 @@ func (u *Upload) Write() error {
}
func init() {
_, err := DB.Sql.Exec(DBUploadSchema)
_, err := DB.Sql.Exec(DBUploadStatusSchema)
if err != nil {
log.Fatal(err)
}
_, err = DB.Sql.Exec(DBUploadSchema)
if err != nil {
log.Fatal(err)
}
@ -165,11 +181,6 @@ func init() {
log.Fatal(err)
}
_, err = DB.Sql.Exec(DBUploadStatusSchema)
if err != nil {
log.Fatal(err)
}
// Populate status types
query := `INSERT INTO upload_status (type, status) VALUES(?,?)`
for k, v := range UploadStatus {