WIP use sqlx and refactor
This commit is contained in:
parent
2cd45617cb
commit
98a0079d69
@ -13,6 +13,7 @@ import (
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
sqlite3 "github.com/mattn/go-sqlite3"
|
||||
"github.com/sp4ke/hashmap"
|
||||
)
|
||||
@ -68,7 +69,7 @@ const (
|
||||
type DB struct {
|
||||
Name string
|
||||
Path string
|
||||
Handle *sql.DB
|
||||
Handle *sqlx.DB
|
||||
EngineMode string
|
||||
AttachedTo []string
|
||||
}
|
||||
@ -106,7 +107,7 @@ func NewRO(name string, path string) *DB {
|
||||
}
|
||||
|
||||
// Create the sqlite connection
|
||||
db.Handle, err = sql.Open(db.EngineMode, db.Path)
|
||||
db.Handle, err = sqlx.Open(db.EngineMode, db.Path)
|
||||
log.Debugf("<%s> opened at <%s> with mode <%s>", db.Name, db.Path,
|
||||
db.EngineMode)
|
||||
if err != nil {
|
||||
@ -130,7 +131,7 @@ func (db *DB) Init() {
|
||||
}
|
||||
|
||||
// Create the memory cache db
|
||||
db.Handle, err = sql.Open("sqlite3", db.Path)
|
||||
db.Handle, err = sqlx.Open("sqlite3", db.Path)
|
||||
//log.Debugf("db <%s> opend at at <%s>", db.Name, db.Path)
|
||||
log.Debugf("<%s> opened at <%s>", db.Name, db.Path)
|
||||
if err != nil {
|
||||
@ -410,7 +411,7 @@ func (src *DB) CopyTo(dst *DB) {
|
||||
dst.Name,
|
||||
dst.Count())
|
||||
|
||||
srcDb, err := sql.Open(DBBackupMode, src.Path)
|
||||
srcDb, err := sqlx.Open(DBBackupMode, src.Path)
|
||||
defer func() {
|
||||
srcDb.Close()
|
||||
_sql3conns = _sql3conns[:len(_sql3conns)-1]
|
||||
@ -421,7 +422,7 @@ func (src *DB) CopyTo(dst *DB) {
|
||||
|
||||
srcDb.Ping()
|
||||
|
||||
dstDb, err := sql.Open(DBBackupMode, dst.Path)
|
||||
dstDb, err := sqlx.Open(DBBackupMode, dst.Path)
|
||||
defer func() {
|
||||
dstDb.Close()
|
||||
_sql3conns = _sql3conns[:len(_sql3conns)-1]
|
||||
@ -448,7 +449,7 @@ func (src *DB) SyncToDisk(dbpath string) error {
|
||||
log.Debugf("Syncing <%s> to <%s>", src.Name, dbpath)
|
||||
|
||||
//log.Debugf("[flush] openeing <%s>", src.path)
|
||||
srcDb, err := sql.Open(DBBackupMode, src.Path)
|
||||
srcDb, err := sqlx.Open(DBBackupMode, src.Path)
|
||||
defer flushSqliteCon(srcDb)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -458,7 +459,7 @@ func (src *DB) SyncToDisk(dbpath string) error {
|
||||
//log.Debugf("[flush] opening <%s>", DB_FILENAME)
|
||||
|
||||
dbUri := fmt.Sprintf("file:%s", dbpath)
|
||||
bkDb, err := sql.Open(DBBackupMode, dbUri)
|
||||
bkDb, err := sqlx.Open(DBBackupMode, dbUri)
|
||||
defer flushSqliteCon(bkDb)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -485,7 +486,7 @@ func (dst *DB) SyncFromDisk(dbpath string) error {
|
||||
log.Debugf("Syncing <%s> to <%s>", dbpath, dst.Name)
|
||||
|
||||
dbUri := fmt.Sprintf("file:%s", dbpath)
|
||||
srcDb, err := sql.Open(DBBackupMode, dbUri)
|
||||
srcDb, err := sqlx.Open(DBBackupMode, dbUri)
|
||||
defer flushSqliteCon(srcDb)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -493,7 +494,7 @@ func (dst *DB) SyncFromDisk(dbpath string) error {
|
||||
srcDb.Ping()
|
||||
|
||||
//log.Debugf("[flush] opening <%s>", DB_FILENAME)
|
||||
bkDb, err := sql.Open(DBBackupMode, dst.Path)
|
||||
bkDb, err := sqlx.Open(DBBackupMode, dst.Path)
|
||||
defer flushSqliteCon(bkDb)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -669,7 +670,7 @@ func GetDBFullPath() string {
|
||||
return dbpath
|
||||
}
|
||||
|
||||
func flushSqliteCon(con *sql.DB) {
|
||||
func flushSqliteCon(con *sqlx.DB) {
|
||||
con.Close()
|
||||
_sql3conns = _sql3conns[:len(_sql3conns)-1]
|
||||
log.Debugf("Flushed sqlite conns -> %v", _sql3conns)
|
||||
|
||||
107
firefox.go
107
firefox.go
@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"gomark/database"
|
||||
"gomark/parsing"
|
||||
"gomark/tools"
|
||||
@ -13,19 +12,20 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
const (
|
||||
QGetBookmarkPlace = `
|
||||
SELECT id,url,description,title
|
||||
SELECT *
|
||||
FROM moz_places
|
||||
WHERE id = ?
|
||||
`
|
||||
QPlacesDelta = `
|
||||
QBookmarksChanged = `
|
||||
SELECT id,type,IFNULL(fk, -1),parent,IFNULL(title, '') from moz_bookmarks
|
||||
WHERE(lastModified > ?
|
||||
WHERE(lastModified > :last_runtime_utc
|
||||
AND lastModified < strftime('%s', 'now') * 1000 * 1000
|
||||
AND NOT id IN (%d,%d)
|
||||
AND NOT id IN (:not_root_tags)
|
||||
)
|
||||
`
|
||||
|
||||
@ -60,7 +60,7 @@ type FFBrowser struct {
|
||||
BaseBrowser //embedding
|
||||
places *database.DB
|
||||
URLIndexList []string // All elements stored in URLIndex
|
||||
tagMap map[int]*tree.Node
|
||||
tagMap map[sqlid]*tree.Node
|
||||
lastRunTime time.Time
|
||||
}
|
||||
|
||||
@ -70,8 +70,7 @@ const (
|
||||
BkTypeTagFolder
|
||||
)
|
||||
|
||||
type FFBookmarkParent int
|
||||
type FFBkType int
|
||||
type sqlid uint64
|
||||
|
||||
const (
|
||||
_ = iota
|
||||
@ -83,19 +82,31 @@ const (
|
||||
ffBkMobile
|
||||
)
|
||||
|
||||
type FFPlace struct {
|
||||
id int
|
||||
url string
|
||||
desc string
|
||||
title string
|
||||
type AutoIncr struct {
|
||||
ID sqlid
|
||||
}
|
||||
|
||||
type FFPlace struct {
|
||||
URL string
|
||||
Description sql.NullString
|
||||
Title sql.NullString
|
||||
AutoIncr
|
||||
}
|
||||
|
||||
//type FFBookmark struct {
|
||||
//BType int `db:type`
|
||||
//Parent sqlid
|
||||
//FK sql.NullInt64
|
||||
//Title sql.NullString
|
||||
//AutoIncr
|
||||
//}
|
||||
|
||||
type FFBookmark struct {
|
||||
id int
|
||||
btype FFBkType
|
||||
parent int
|
||||
fk int
|
||||
btype int `db:type`
|
||||
parent sqlid
|
||||
fk sqlid
|
||||
title string
|
||||
id sqlid
|
||||
}
|
||||
|
||||
func FFPlacesUpdateHook(op int, db string, table string, rowid int64) {
|
||||
@ -111,7 +122,7 @@ func NewFFBrowser() IBrowser {
|
||||
browser.useFileWatcher = true
|
||||
browser.Stats = &parsing.Stats{}
|
||||
browser.NodeTree = &tree.Node{Name: "root", Parent: nil, Type: "root"}
|
||||
browser.tagMap = make(map[int]*tree.Node)
|
||||
browser.tagMap = make(map[sqlid]*tree.Node)
|
||||
|
||||
// Initialize `places.sqlite`
|
||||
bookmarkPath := path.Join(browser.baseDir, browser.bkFile)
|
||||
@ -237,7 +248,7 @@ func getFFBookmarks(bw *FFBrowser) {
|
||||
*/
|
||||
for rows.Next() {
|
||||
var url, title, tagTitle, desc string
|
||||
var tagId int
|
||||
var tagId sqlid
|
||||
err = rows.Scan(&url, &title, &desc, &tagId, &tagTitle)
|
||||
//fflog.Debugf("%s|%s|%s|%d|%s", url, title, desc, tagId, tagTitle)
|
||||
if err != nil {
|
||||
@ -291,14 +302,13 @@ func getFFBookmarks(bw *FFBrowser) {
|
||||
}
|
||||
|
||||
func (bw *FFBrowser) fetchUrlChanges(rows *sql.Rows,
|
||||
bookmarks map[int]*FFBookmark,
|
||||
places map[int]*FFPlace) {
|
||||
bookmarks map[sqlid]*FFBookmark,
|
||||
places map[sqlid]*FFPlace) {
|
||||
|
||||
bk := new(FFBookmark)
|
||||
|
||||
// Get the URL that changed
|
||||
rows.Scan(&bk.id, &bk.btype, &bk.fk, &bk.parent, &bk.title)
|
||||
fflog.Debug(bk)
|
||||
|
||||
// We found URL change, urls are specified by
|
||||
// type == 1
|
||||
@ -311,13 +321,12 @@ func (bw *FFBrowser) fetchUrlChanges(rows *sql.Rows,
|
||||
// 3. A (type==1) (fk-> moz_places.url) (parent == idOf(tag))
|
||||
|
||||
if bk.btype == BkTypeURL {
|
||||
place := new(FFPlace)
|
||||
res := bw.places.Handle.QueryRow(QGetBookmarkPlace, bk.fk)
|
||||
res.Scan(&place.id, &place.url, &place.desc, &place.title)
|
||||
fflog.Debugf("Changed URL: %s", place.url)
|
||||
var place FFPlace
|
||||
bw.places.Handle.QueryRowx(QGetBookmarkPlace, bk.fk).StructScan(&place)
|
||||
fflog.Debugf("Changed URL: %s", place.URL)
|
||||
|
||||
// put url in the places map
|
||||
places[place.id] = place
|
||||
places[place.ID] = &place
|
||||
}
|
||||
|
||||
// This is the tag link
|
||||
@ -346,21 +355,35 @@ func (bw *FFBrowser) Run() {
|
||||
// watch event
|
||||
|
||||
startRun := time.Now()
|
||||
fflog.Debugf("Checking changes since %s",
|
||||
bw.lastRunTime.Local().Format("Mon Jan 2 15:04:05 MST 2006"))
|
||||
//fflog.Debugf("Checking changes since %s",
|
||||
//bw.lastRunTime.Local().Format("Mon Jan 2 15:04:05 MST 2006"))
|
||||
|
||||
rows, err := bw.places.Handle.Query(
|
||||
// Pre Populate the query
|
||||
fmt.Sprintf(QPlacesDelta, "%s", ffBkRoot, ffBkTags),
|
||||
queryArgs := map[string]interface{}{
|
||||
"not_root_tags": []int{ffBkRoot, ffBkTags},
|
||||
"last_runtime_utc": bw.lastRunTime.UTC().UnixNano() / 1000,
|
||||
}
|
||||
|
||||
// Sql parameter
|
||||
bw.lastRunTime.UnixNano()/1000,
|
||||
query, args, err := sqlx.Named(
|
||||
QBookmarksChanged,
|
||||
queryArgs,
|
||||
)
|
||||
if err != nil {
|
||||
fflog.Error(err)
|
||||
}
|
||||
|
||||
query, args, err = sqlx.In(query, args...)
|
||||
if err != nil {
|
||||
fflog.Error(err)
|
||||
}
|
||||
|
||||
query = bw.places.Handle.Rebind(query)
|
||||
rows, err := bw.places.Handle.Query(query, args...)
|
||||
defer rows.Close()
|
||||
|
||||
if err != nil {
|
||||
fflog.Error(err)
|
||||
}
|
||||
|
||||
// Found new results in places db since last time we had changes
|
||||
//database.DebugPrintRows(rows)
|
||||
if rows.Next() {
|
||||
@ -370,8 +393,8 @@ func (bw *FFBrowser) Run() {
|
||||
//fflog.Debugf("CHANGE ! Time: %s",
|
||||
//bw.lastRunTime.Local().Format("Mon Jan 2 15:04:05 MST 2006"))
|
||||
|
||||
bookmarks := make(map[int]*FFBookmark)
|
||||
places := make(map[int]*FFPlace)
|
||||
bookmarks := make(map[sqlid]*FFBookmark)
|
||||
places := make(map[sqlid]*FFPlace)
|
||||
|
||||
// Fetch all changes into bookmarks and places maps
|
||||
bw.fetchUrlChanges(rows, bookmarks, places)
|
||||
@ -379,15 +402,15 @@ func (bw *FFBrowser) Run() {
|
||||
// For each url
|
||||
for urlId, place := range places {
|
||||
var urlNode *tree.Node
|
||||
changedURLS = tools.Extends(changedURLS, place.url)
|
||||
iUrlNode, urlNodeExists := bw.URLIndex.Get(place.url)
|
||||
changedURLS = tools.Extends(changedURLS, place.URL)
|
||||
iUrlNode, urlNodeExists := bw.URLIndex.Get(place.URL)
|
||||
if !urlNodeExists {
|
||||
urlNode = new(tree.Node)
|
||||
urlNode.Type = "url"
|
||||
urlNode.URL = place.url
|
||||
urlNode.Name = place.title
|
||||
urlNode.Desc = place.desc
|
||||
bw.URLIndex.Insert(place.url, urlNode)
|
||||
urlNode.URL = place.URL
|
||||
urlNode.Name = place.Title.String
|
||||
urlNode.Desc = place.Description.String
|
||||
bw.URLIndex.Insert(place.URL, urlNode)
|
||||
|
||||
} else {
|
||||
urlNode = iUrlNode.(*tree.Node)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user