2018-11-30 03:15:08 +00:00
|
|
|
//TODO: unit test New/NewRO should check if database is locked
|
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
//TODO: missing defer close() on sqlite funcs
|
2018-11-13 17:21:02 +00:00
|
|
|
//TODO: handle `modified` time
|
2018-11-09 17:25:50 +00:00
|
|
|
package database
|
|
|
|
|
|
|
|
import (
|
|
|
|
"database/sql"
|
2018-12-04 04:02:47 +00:00
|
|
|
"errors"
|
2018-11-09 17:25:50 +00:00
|
|
|
"fmt"
|
|
|
|
"gomark/logging"
|
|
|
|
"gomark/tree"
|
2018-12-04 03:34:30 +00:00
|
|
|
"net/url"
|
2018-11-09 17:25:50 +00:00
|
|
|
"strings"
|
|
|
|
|
2018-11-23 03:40:10 +00:00
|
|
|
"github.com/jmoiron/sqlx"
|
2018-11-30 03:15:08 +00:00
|
|
|
"github.com/mattn/go-sqlite3"
|
2018-11-09 17:25:50 +00:00
|
|
|
"github.com/sp4ke/hashmap"
|
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
_sql3conns []*sqlite3.SQLiteConn // Only used for backup hook
|
|
|
|
backupHookRegistered bool // set to true once the backup hook is registered
|
|
|
|
)
|
|
|
|
|
|
|
|
type Index = *hashmap.RBTree
|
|
|
|
type Node = tree.Node
|
|
|
|
|
|
|
|
var log = logging.GetLogger("DB")
|
|
|
|
|
|
|
|
const (
|
2018-12-04 03:34:30 +00:00
|
|
|
DBFileName = "gomarks.db"
|
|
|
|
CacheName = "memcache"
|
|
|
|
//MemcacheFmt = "file:%s?mode=memory&cache=shared"
|
|
|
|
//BufferFmt = "file:%s?mode=memory&cache=shared"
|
|
|
|
|
|
|
|
DBTypeInMemoryDSN = "file:%s?mode=memory&cache=shared"
|
|
|
|
DBTypeCacheDSN = DBTypeInMemoryDSN
|
|
|
|
DBTypeFileDSN = "file:%s"
|
2018-12-01 17:16:46 +00:00
|
|
|
|
|
|
|
DriverBackupMode = "sqlite_hook_backup"
|
|
|
|
DriverDefault = "sqlite3"
|
|
|
|
GomarkMainTable = "bookmarks"
|
2018-11-09 17:25:50 +00:00
|
|
|
)
|
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
type DBType int
|
|
|
|
|
|
|
|
const (
|
|
|
|
DBTypeInMemory DBType = iota
|
|
|
|
DBTypeRegularFile
|
|
|
|
)
|
|
|
|
|
|
|
|
// Differentiate between gomarkdb.sqlite and other sqlite DBs
|
|
|
|
const (
|
|
|
|
DBGomark DBType = iota
|
|
|
|
DBForeign
|
|
|
|
)
|
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
// Database schemas used for the creation of new databases
|
|
|
|
const (
|
|
|
|
// metadata: name or title of resource
|
2018-11-13 17:21:02 +00:00
|
|
|
// modified: time.Now().Unix()
|
2018-11-30 19:04:54 +00:00
|
|
|
//
|
|
|
|
// flags: designed to be extended in future using bitwise masks
|
|
|
|
// Masks:
|
|
|
|
// 0b00000001: set title immutable ((do not change title when updating the bookmarks from the web ))
|
2018-11-30 03:15:08 +00:00
|
|
|
QCreateGomarkDBSchema = `CREATE TABLE if not exists bookmarks (
|
2018-11-09 17:25:50 +00:00
|
|
|
id integer PRIMARY KEY,
|
|
|
|
URL text NOT NULL UNIQUE,
|
|
|
|
metadata text default '',
|
|
|
|
tags text default '',
|
|
|
|
desc text default '',
|
|
|
|
modified integer default (strftime('%s')),
|
|
|
|
flags integer default 0
|
|
|
|
)`
|
|
|
|
)
|
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
type DsnOptions map[string]string
|
2018-11-30 19:04:54 +00:00
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
type DBError struct {
|
|
|
|
// Database object where error occured
|
2018-12-04 04:02:47 +00:00
|
|
|
DBName string
|
2018-12-01 17:16:46 +00:00
|
|
|
|
|
|
|
// Error that occured
|
|
|
|
Err error
|
|
|
|
}
|
|
|
|
|
2018-12-04 04:02:47 +00:00
|
|
|
func DBErr(dbName string, err error) DBError {
|
|
|
|
return DBError{Err: err}
|
|
|
|
}
|
|
|
|
|
2018-12-03 18:55:28 +00:00
|
|
|
func (e DBError) Error() string {
|
2018-12-04 04:02:47 +00:00
|
|
|
return fmt.Sprintf("<%s>: %s", e.DBName, e.Err)
|
2018-12-01 17:16:46 +00:00
|
|
|
}
|
|
|
|
|
2018-12-04 04:02:47 +00:00
|
|
|
var (
|
|
|
|
ErrVfsLocked = errors.New("vfs locked")
|
|
|
|
)
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
type Opener interface {
|
|
|
|
Open(driver string, dsn string) error
|
|
|
|
}
|
|
|
|
|
|
|
|
type SQLXOpener interface {
|
|
|
|
Opener
|
|
|
|
Get() *sqlx.DB
|
|
|
|
}
|
|
|
|
|
|
|
|
type SQLXDBOpener struct {
|
|
|
|
handle *sqlx.DB
|
|
|
|
}
|
|
|
|
|
|
|
|
func (o *SQLXDBOpener) Open(driver string, dataSourceName string) error {
|
|
|
|
var err error
|
|
|
|
o.handle, err = sqlx.Open(driver, dataSourceName)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (o *SQLXDBOpener) Get() *sqlx.DB {
|
|
|
|
return o.handle
|
|
|
|
}
|
2018-11-30 19:04:54 +00:00
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
// DB encapsulates an sql.DB struct. All interactions with memory/buffer and
|
|
|
|
// disk databases are done through the DB object
|
|
|
|
type DB struct {
|
|
|
|
Name string
|
|
|
|
Path string
|
2018-11-23 03:40:10 +00:00
|
|
|
Handle *sqlx.DB
|
2018-11-09 17:25:50 +00:00
|
|
|
EngineMode string
|
2018-11-13 17:21:02 +00:00
|
|
|
AttachedTo []string
|
2018-11-30 19:04:54 +00:00
|
|
|
Type DBType
|
2018-12-01 17:16:46 +00:00
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
filePath string
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
SQLXOpener
|
2018-12-04 03:34:30 +00:00
|
|
|
LockChecker
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
func (db *DB) Open() error {
|
|
|
|
var err error
|
|
|
|
err = db.SQLXOpener.Open(db.EngineMode, db.Path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
db.Handle = db.SQLXOpener.Get()
|
2018-12-04 03:34:30 +00:00
|
|
|
err = db.Handle.Ping()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-12-01 17:16:46 +00:00
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
log.Debugf("<%s> opened at <%s> with driver <%s>",
|
2018-12-01 17:16:46 +00:00
|
|
|
db.Name,
|
|
|
|
db.Path,
|
|
|
|
db.EngineMode)
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
func (db *DB) Locked() (bool, error) {
|
|
|
|
return db.LockChecker.Locked()
|
|
|
|
}
|
|
|
|
|
|
|
|
// dbPath is empty string ("") when using in memory sqlite db
|
|
|
|
// Call to Init() required before using
|
|
|
|
func New(name string, dbPath string, dbFormat string, opts ...DsnOptions) *DB {
|
|
|
|
|
|
|
|
var path string
|
|
|
|
var dbType DBType
|
|
|
|
|
|
|
|
// Use name as path for in memory database
|
|
|
|
if dbPath == "" {
|
|
|
|
path = fmt.Sprintf(dbFormat, name)
|
|
|
|
dbType = DBTypeInMemory
|
|
|
|
} else {
|
|
|
|
path = fmt.Sprintf(dbFormat, dbPath)
|
|
|
|
dbType = DBTypeRegularFile
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
2018-11-30 03:15:08 +00:00
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
// Handle DSN options
|
|
|
|
if len(opts) > 0 {
|
|
|
|
dsn := url.Values{}
|
|
|
|
for _, o := range opts {
|
|
|
|
for k, v := range o {
|
|
|
|
dsn.Set(k, v)
|
|
|
|
}
|
|
|
|
}
|
2018-11-09 17:25:50 +00:00
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
// Test if path has already query params
|
|
|
|
pos := strings.IndexRune(path, '?')
|
|
|
|
|
|
|
|
// Path already has query params
|
|
|
|
if pos >= 1 {
|
|
|
|
path = fmt.Sprintf("%s&%s", path, dsn.Encode()) //append
|
|
|
|
} else {
|
|
|
|
path = fmt.Sprintf("%s?%s", path, dsn.Encode())
|
|
|
|
}
|
2018-11-20 17:33:37 +00:00
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
}
|
2018-11-20 17:33:37 +00:00
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
return &DB{
|
2018-11-30 03:15:08 +00:00
|
|
|
Name: name,
|
2018-12-04 03:34:30 +00:00
|
|
|
Path: path,
|
2018-11-30 03:15:08 +00:00
|
|
|
Handle: nil,
|
2018-12-01 17:16:46 +00:00
|
|
|
EngineMode: DriverDefault,
|
|
|
|
SQLXOpener: &SQLXDBOpener{},
|
2018-12-04 03:34:30 +00:00
|
|
|
Type: dbType,
|
|
|
|
filePath: dbPath,
|
|
|
|
LockChecker: &VFSLockChecker{
|
|
|
|
path: dbPath,
|
|
|
|
},
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
2018-12-04 03:34:30 +00:00
|
|
|
|
|
|
|
}
|
|
|
|
|
2018-12-04 04:02:47 +00:00
|
|
|
//TODO: try unlock at the browser level !
|
2018-12-04 03:34:30 +00:00
|
|
|
func (db *DB) tryUnlock() error {
|
|
|
|
log.Debug("Unlocking ...")
|
|
|
|
|
|
|
|
// Find if multiProcessAccess option is defined
|
|
|
|
//TODO:
|
|
|
|
//if v, err := mozilla.HasPref(path ???)
|
|
|
|
//
|
|
|
|
return nil
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
//TODO: Should check if DB is locked
|
|
|
|
// We should export Open() in its own method and wrap
|
|
|
|
// with interface so we can mock it and test the lock status in Init()
|
2018-11-30 03:15:08 +00:00
|
|
|
// Initialize a sqlite database with Gomark Schema if not already done
|
2018-12-01 17:16:46 +00:00
|
|
|
func (db *DB) Init() (*DB, error) {
|
2018-11-09 17:25:50 +00:00
|
|
|
|
|
|
|
// `cacheDB` is a memory replica of disk db
|
|
|
|
|
|
|
|
var err error
|
|
|
|
|
|
|
|
if db.Handle != nil {
|
2018-11-30 03:15:08 +00:00
|
|
|
log.Warningf("%s: already initialized", db)
|
2018-12-01 17:16:46 +00:00
|
|
|
return db, nil
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
// Detect if database file is locked
|
|
|
|
if db.Type == DBTypeRegularFile {
|
|
|
|
|
|
|
|
locked, err := db.Locked()
|
|
|
|
|
|
|
|
if err != nil {
|
2018-12-04 04:02:47 +00:00
|
|
|
return nil, DBError{DBName: db.Name, Err: err}
|
2018-12-04 03:34:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if locked {
|
2018-12-04 04:02:47 +00:00
|
|
|
return nil, DBErr(db.Name, ErrVfsLocked)
|
2018-12-04 03:34:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2018-12-03 18:55:28 +00:00
|
|
|
// Open database
|
|
|
|
err = db.Open()
|
2018-11-30 19:04:54 +00:00
|
|
|
|
2018-12-03 18:55:28 +00:00
|
|
|
sqlErr, _ := err.(sqlite3.Error)
|
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
// Secondary lock check provided by sqlx Ping() method
|
2018-12-03 18:55:28 +00:00
|
|
|
if err != nil && sqlErr.Code == sqlite3.ErrBusy {
|
2018-12-04 04:02:47 +00:00
|
|
|
return nil, DBError{DBName: db.Name, Err: err}
|
2018-12-04 03:34:30 +00:00
|
|
|
|
2018-12-03 18:55:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Return all other errors
|
2018-11-09 17:25:50 +00:00
|
|
|
if err != nil {
|
2018-12-04 04:02:47 +00:00
|
|
|
return nil, DBError{DBName: db.Name, Err: err}
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
return db, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (db *DB) InitSchema() error {
|
2018-11-30 19:04:54 +00:00
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
// Populate db schema
|
|
|
|
tx, err := db.Handle.Begin()
|
|
|
|
if err != nil {
|
2018-12-04 04:02:47 +00:00
|
|
|
return DBError{DBName: db.Name, Err: err}
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
2018-11-30 03:15:08 +00:00
|
|
|
stmt, err := tx.Prepare(QCreateGomarkDBSchema)
|
2018-11-09 17:25:50 +00:00
|
|
|
if err != nil {
|
2018-12-04 04:02:47 +00:00
|
|
|
return DBError{DBName: db.Name, Err: err}
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
if _, err = stmt.Exec(); err != nil {
|
2018-12-04 04:02:47 +00:00
|
|
|
return DBError{DBName: db.Name, Err: err}
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
if err = tx.Commit(); err != nil {
|
2018-12-04 04:02:47 +00:00
|
|
|
return DBError{DBName: db.Name, Err: err}
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
log.Debugf("<%s> initialized", db.Name)
|
2018-11-30 03:15:08 +00:00
|
|
|
|
2018-12-04 03:34:30 +00:00
|
|
|
return nil
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
func (db *DB) AttachTo(attached *DB) {
|
2018-11-09 17:25:50 +00:00
|
|
|
|
|
|
|
stmtStr := fmt.Sprintf("ATTACH DATABASE '%s' AS '%s'",
|
|
|
|
attached.Path,
|
|
|
|
attached.Name)
|
|
|
|
_, err := db.Handle.Exec(stmtStr)
|
2018-11-13 17:21:02 +00:00
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
|
2018-11-13 17:21:02 +00:00
|
|
|
db.AttachedTo = append(db.AttachedTo, attached.Name)
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (db *DB) Close() error {
|
|
|
|
log.Debugf("Closing DB <%s>", db.Name)
|
2018-12-04 03:34:30 +00:00
|
|
|
|
|
|
|
if db.Handle == nil {
|
|
|
|
log.Warningf("<%s> handle is nil", db.Name)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
err := db.Handle.Close()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (db *DB) IsEmpty() (bool, error) {
|
|
|
|
var count int
|
|
|
|
|
|
|
|
row := db.Handle.QueryRow("select count(*) from bookmarks")
|
|
|
|
|
|
|
|
err := row.Scan(&count)
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if count > 0 {
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
func (db *DB) CountRows(table string) int {
|
|
|
|
var count int
|
|
|
|
|
|
|
|
row := db.Handle.QueryRow("select count(*) from ?", table)
|
|
|
|
err := row.Scan(&count)
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return count
|
|
|
|
}
|
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
// Struct represetning the schema of `bookmarks` db.
|
|
|
|
// The order in the struct respects the columns order
|
|
|
|
type SBookmark struct {
|
|
|
|
id int
|
|
|
|
Url string
|
|
|
|
metadata string
|
|
|
|
tags string
|
|
|
|
desc string
|
|
|
|
modified int64
|
|
|
|
flags int
|
|
|
|
}
|
|
|
|
|
|
|
|
// Scans a row into `SBookmark` schema
|
|
|
|
func ScanBookmarkRow(row *sql.Rows) (*SBookmark, error) {
|
|
|
|
scan := new(SBookmark)
|
|
|
|
err := row.Scan(
|
|
|
|
&scan.id,
|
|
|
|
&scan.Url,
|
|
|
|
&scan.metadata,
|
|
|
|
&scan.tags,
|
|
|
|
&scan.desc,
|
|
|
|
&scan.modified,
|
|
|
|
&scan.flags,
|
|
|
|
)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return scan, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func SyncURLIndexToBuffer(urls []string, index Index, buffer *DB) {
|
|
|
|
for _, url := range urls {
|
|
|
|
iNode, exists := index.Get(url)
|
|
|
|
if !exists {
|
|
|
|
log.Warningf("url does not exist in index: %s", url)
|
|
|
|
break
|
|
|
|
}
|
|
|
|
node := iNode.(*Node)
|
|
|
|
bk := node.GetBookmark()
|
|
|
|
buffer.InsertOrUpdateBookmark(bk)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func SyncTreeToBuffer(node *Node, buffer *DB) {
|
|
|
|
if node.Type == "url" {
|
|
|
|
bk := node.GetBookmark()
|
|
|
|
buffer.InsertOrUpdateBookmark(bk)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(node.Children) > 0 {
|
|
|
|
for _, node := range node.Children {
|
|
|
|
SyncTreeToBuffer(node, buffer)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-11-23 03:40:10 +00:00
|
|
|
func flushSqliteCon(con *sqlx.DB) {
|
2018-11-09 17:25:50 +00:00
|
|
|
con.Close()
|
|
|
|
_sql3conns = _sql3conns[:len(_sql3conns)-1]
|
2018-11-13 16:11:16 +00:00
|
|
|
log.Debugf("Flushed sqlite conns -> %v", _sql3conns)
|
2018-11-09 17:25:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func registerSqliteHooks() {
|
|
|
|
// sqlite backup hook
|
2018-12-01 17:16:46 +00:00
|
|
|
log.Debugf("backup_hook: registering driver %s", DriverBackupMode)
|
2018-11-09 17:25:50 +00:00
|
|
|
// Register the hook
|
2018-12-01 17:16:46 +00:00
|
|
|
sql.Register(DriverBackupMode,
|
2018-11-09 17:25:50 +00:00
|
|
|
&sqlite3.SQLiteDriver{
|
|
|
|
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
|
|
|
|
//log.Debugf("[ConnectHook] registering new connection")
|
|
|
|
_sql3conns = append(_sql3conns, conn)
|
|
|
|
//log.Debugf("%v", _sql3conns)
|
|
|
|
return nil
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
registerSqliteHooks()
|
|
|
|
}
|