From 634e94909c13682f80a08fbe8f4ec2d0b017ec7b Mon Sep 17 00:00:00 2001 From: Chakib Benziane Date: Sun, 24 Mar 2019 22:36:28 +0100 Subject: [PATCH] working mariadb access + redis --- db/db.go | 70 ++++++++++++++++++----------------------- docker-compose.yml | 24 ++++++++------ go.mod | 10 ++++++ go.sum | 42 +++++++++++++++++++++++++ storage/upload_model.go | 31 ++++++++++++------ 5 files changed, 117 insertions(+), 60 deletions(-) diff --git a/db/db.go b/db/db.go index fd94dc4..143acd5 100644 --- a/db/db.go +++ b/db/db.go @@ -3,28 +3,23 @@ package db import ( "fmt" "log" - "net/url" "os" - "path/filepath" - "git.sp4ke.com/sp4ke/bit4sat/utils" + _ "github.com/go-sql-driver/mysql" "github.com/jmoiron/sqlx" - _ "github.com/mattn/go-sqlite3" "github.com/mediocregopher/radix/v3" ) const ( - DBName = "bit4sat.sqlite" - DBPath = "db-storage" - DBPathEnv = "BIT4SAT_DB_PATH" - DBPragma = ` PRAGMA foreign_keys = ON; ` + //DBName = "bit4sat.sqlite" + DBName = "bit4sat" + DBHostEnv = "DB_HOST" + DBUserEnv = "DB_USER" + DBPassEnv = "DB_PASS" + //DBPragma = ` PRAGMA foreign_keys = ON; ` ) var ( - DBOptions = map[string]string{ - "_journal_mode": "WAL", - } - DB *Database ) @@ -34,38 +29,38 @@ type Database struct { } func (d *Database) Open() error { + var err error - dsnOptions := &url.Values{} - for k, v := range DBOptions { - dsnOptions.Set(k, v) - } - - // Get db base path - path, set := os.LookupEnv(DBPathEnv) + // Get env vars + host, set := os.LookupEnv(DBHostEnv) if !set { - path = DBPath + log.Fatal("undefined DB_HOST env") } - // Create path if not exists - err := utils.Mkdir(path) - if err != nil { - log.Fatal(err) + user, set := os.LookupEnv(DBUserEnv) + if !set { + log.Fatal("undefined DB_USER env") } - path = filepath.Join(path, DBName) - //path = fmt.Sprintf("%s/%s", path, DBName) - - dsn := fmt.Sprintf("file:%s?%s", path, dsnOptions.Encode()) - - log.Printf("Opening sqlite db %s\n", dsn) - - d.Sql, err = sqlx.Open("sqlite3", dsn) - if err != nil { - log.Fatal(err) + pass, set := os.LookupEnv(DBPassEnv) + if !set { + log.Fatal("undefined DB_PASS env") } + dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s", user, pass, host, DBName) + + log.Printf("Opening SQL at %s\n", dsn) + + d.Sql = sqlx.MustConnect("mysql", dsn) + // Execute Pragmas - d.Sql.MustExec(DBPragma) + //d.Sql.MustExec(DBPragma) + + // Opend redis + d.Redis, err = radix.NewPool("tcp", "redis:6379", 10) + if err != nil { + log.Fatal(err) + } return nil } @@ -86,9 +81,4 @@ func init() { DB = &Database{} DB.Open() - var err error - DB.Redis, err = radix.NewPool("tcp", "redis:6379", 10) - if err != nil { - log.Fatal(err) - } } diff --git a/docker-compose.yml b/docker-compose.yml index eb892d8..278d045 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,8 +2,9 @@ version: "3.4" volumes: db: + maria-conf: file-storage: - sqlite: + #sqlite: gocache: services: @@ -12,9 +13,11 @@ services: build: ./docker environment: - GO111MODULE=on - - BIT4SAT_DB_PATH=/sqlite - BIT4SAT_STORAGE_PATH=/storage - GOPATH=/go + - DB_HOST=maria + - DB_USER=bit4sat + - DB_PASS=bit4sat #deploy: #replicas: 1 @@ -25,7 +28,7 @@ services: volumes: - $PWD:/src - gocache:/go - - ./db-storage:/sqlite + #- ./db-storage:/sqlite - file-storage:/storage working_dir: /src @@ -40,12 +43,13 @@ services: volumes: - db:/var/lib/mysql - #redis: - #image: redis:alpine + - maria-conf:/etc/mysql + redis: + image: redis:alpine - #volumes: - #- db:/data + volumes: + - db:/data - #command: - #- redis-server - #- --appendonly yes + command: + - redis-server + - --appendonly yes diff --git a/go.mod b/go.mod index 2ddede6..cbacc37 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,18 @@ go 1.12 require ( github.com/gin-contrib/cors v0.0.0-20190301062745-f9e10995c85a github.com/gin-gonic/gin v1.3.0 + github.com/go-sql-driver/mysql v1.4.1 + github.com/golang/protobuf v1.3.1 // indirect github.com/jmoiron/sqlx v1.2.0 + github.com/kr/pretty v0.1.0 // indirect + github.com/mattn/go-isatty v0.0.7 // indirect github.com/mattn/go-sqlite3 v1.10.0 github.com/mediocregopher/radix/v3 v3.2.3 github.com/segmentio/ksuid v1.0.2 + github.com/stretchr/testify v1.3.0 // indirect + github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780 // indirect + golang.org/x/net v0.0.0-20190322120337-addf6b3196f6 // indirect + golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc // indirect + google.golang.org/appengine v1.5.0 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index b45ec52..4ceeca4 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gin-contrib/cors v0.0.0-20190301062745-f9e10995c85a h1:zBycVvXa03SIX+jdMv8wGu9TMDMWdN8EhaR1FoeKHNo= github.com/gin-contrib/cors v0.0.0-20190301062745-f9e10995c85a/go.mod h1:pL2kNE+DgDU+eQ+dary5bX0Z6LPP8nR6Mqs1iejILw4= @@ -6,32 +8,72 @@ github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NB github.com/gin-gonic/gin v1.3.0 h1:kCmZyPklC0gVdL728E6Aj20uYBJV93nj/TkwBTKhFbs= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed h1:3dQJqqDouawQgl3gBE1PNHKFkJYGEuFb1DbSlaxdosE= github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= github.com/mediocregopher/radix/v3 v3.2.3 h1:TbcGCZdo9zfPYPgevsqRn+OjvCyfOK6TzuXhqzWdCt0= github.com/mediocregopher/radix/v3 v3.2.3/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/segmentio/ksuid v1.0.2 h1:9yBfKyw4ECGTdALaF09Snw3sLJmYIX6AbPJrAy6MrDc= github.com/segmentio/ksuid v1.0.2/go.mod h1:BXuJDr2byAiHuQaQtSKoXh1J0YmUDurywOXgB2w+OSU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ugorji/go v1.1.2 h1:JON3E2/GPW2iDNGoSAusl1KDf5TRQ8k8q7Tp097pZGs= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2 h1:EICbibRW4JNKMcY+LsWmuwob+CRS1BmdRdjphAm9mH4= github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780 h1:vG/gY/PxA3v3l04qxe3tDjXyu3bozii8ulSlIPOYKhI= +github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190322120337-addf6b3196f6 h1:78jEq2G3J16aXneH23HSnTQQTCwMHoyO8VEiUH+bpPM= +golang.org/x/net v0.0.0-20190322120337-addf6b3196f6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc h1:4gbWbmmPFp4ySWICouJl6emP0MyS31yy9SrTlAGFT+g= +golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= diff --git a/storage/upload_model.go b/storage/upload_model.go index 6814f8f..bab5dc2 100644 --- a/storage/upload_model.go +++ b/storage/upload_model.go @@ -14,7 +14,8 @@ var DB = db.DB const ( DBUploadSchema = ` - CREATE TABLE IF NOT EXISTS "upload" ( + CREATE TABLE IF NOT EXISTS upload ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, upload_id CHAR(20) NOT NULL, sha256 CHAR(32) NOT NULL, file_name CHAR(255) NOT NULL, @@ -28,14 +29,24 @@ const ( ` DBUploadView = ` -CREATE VIEW IF NOT EXISTS "upload_with_status" AS SELECT * FROM upload JOIN upload_status ON +CREATE VIEW IF NOT EXISTS upload_with_status +AS SELECT + upload.upload_id, + upload.sha256, + upload_status.status, + upload.file_name, + upload.file_type, + upload.file_size, + upload.file_ext + FROM upload +JOIN upload_status ON upload.status = upload_status.type ` DBUploadStatusSchema = `CREATE TABLE IF NOT EXISTS upload_status ( - type INTEGER PRIMARY KEY, - status TEXT NOT NULL UNIQUE + type INT NOT NULL PRIMARY KEY, + status CHAR(255) NOT NULL UNIQUE ) ` @@ -155,7 +166,12 @@ func (u *Upload) Write() error { } func init() { - _, err := DB.Sql.Exec(DBUploadSchema) + _, err := DB.Sql.Exec(DBUploadStatusSchema) + if err != nil { + log.Fatal(err) + } + + _, err = DB.Sql.Exec(DBUploadSchema) if err != nil { log.Fatal(err) } @@ -165,11 +181,6 @@ func init() { log.Fatal(err) } - _, err = DB.Sql.Exec(DBUploadStatusSchema) - if err != nil { - log.Fatal(err) - } - // Populate status types query := `INSERT INTO upload_status (type, status) VALUES(?,?)` for k, v := range UploadStatus {