[firefox] partially working places.sqlite copy strategy

- when file change is detected on places.sqlite* the
sqlite files are copied to a tmp directory and bookmark
parsing happens from there.

This commit covers the base functionnality, the bookmark parsing seems
broken from the sqlx side but new bookmarks are effectivly detected.

browsers/base.go

@@ -78,6 +78,8 @@ type BaseBrowser struct {
 	BaseDir    string
 	BkFile     string
 
+	WatchedPaths []string
+
 	// In memory sqlite db (named `memcache`).
 	// Used to keep a browser's state of bookmarks across jobs.
 	BufferDB *database.DB

commands.go

@@ -4,6 +4,7 @@ import (
 	"os"
 
 	"git.sp4ke.com/sp4ke/gomark/parsing"
+	"git.sp4ke.com/sp4ke/gomark/utils"
 
 	"git.sp4ke.com/sp4ke/gum"
 
@@ -18,6 +19,7 @@ var startServerCmd = &cli.Command{
 }
 
 func startServer(c *cli.Context) error {
+	defer utils.CleanFiles()
 	manager := gum.NewManager()
 	manager.ShutdownOn(os.Interrupt)
 

config.go

@@ -31,7 +31,6 @@ func init() {
 		if err != nil {
 			log.Fatal(err)
 		}
-
 	}
 
 }

database/database.go

@@ -1,4 +1,5 @@
 //TODO: handle `modified` time
+// sqlite database management
 package database
 
 import (

firefox.go

@@ -158,16 +158,54 @@ func (bw *FFBrowser) Watch() bool {
 	if !bw.IsWatching {
 		go watch.WatcherThread(bw)
 		bw.IsWatching = true
-		fflog.Infof("Watching %s", bw.GetBookmarksPath())
+		for _, v := range bw.WatchedPaths {
+			fflog.Infof("Watching %s", v)
+		}
 		return true
 	}
 
 	return false
 }
 
+func (browser *FFBrowser) copyPlacesToTmp() error {
+
+	err := utils.CopyFilesToTmpFolder(path.Join(browser.BaseDir, browser.BkFile+"*"))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (browser *FFBrowser) getPathToPlacesCopy() string {
+	return path.Join(utils.TMPDIR, browser.BkFile)
+}
+
+func (browser *FFBrowser) InitPlacesCopy() error {
+
+	// Copy places.sqlite to tmp dir
+	err := browser.copyPlacesToTmp()
+	if err != nil {
+		return fmt.Errorf("Could not copy places.sqlite to tmp folder: %s",
+			err)
+	}
+
+	opts := mozilla.Config.PlacesDSN
+
+	browser.places, err = database.New("places",
+		// using the copied places file instead of the original to avoid
+		// sqlite vfs lock errors
+		browser.getPathToPlacesCopy(),
+		database.DBTypeFileDSN, opts).Init()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (browser *FFBrowser) Init() error {
 
-	// Initialize `places.sqlite`
 	bookmarkPath := path.Join(browser.BaseDir, browser.BkFile)
 
 	// Check if BookmarkPath exists
@@ -181,40 +219,21 @@ func (browser *FFBrowser) Init() error {
 		return fmt.Errorf("Bookmark path <%s> does not exist", bookmarkPath)
 	}
 
-	opts := mozilla.Config.PlacesDSN
+	err = browser.InitPlacesCopy()
-
-	browser.places, err = database.New("places",
-		bookmarkPath,
-		database.DBTypeFileDSN, opts).Init()
-	if err != nil {
-		//fflog.Error(err)
-		return err
-
-		//Check Lock Error
-		//if err == database.ErrVfsLocked {
-		//// Try to unlock db
-		//e := mozilla.UnlockPlaces()
-		//if e != nil {
-		//return e
-		//}
-		//} else {
-		//return err
-
-		//}
-	}
 
 	// Setup watcher
-
 	expandedBaseDir, err := filepath.EvalSymlinks(browser.BaseDir)
 
 	if err != nil {
 		return err
 	}
 
+	browser.WatchedPaths = []string{filepath.Join(expandedBaseDir, "places.sqlite-wal")}
+
 	w := &watch.Watch{
 		Path:       expandedBaseDir,
 		EventTypes: []fsnotify.Op{fsnotify.Write},
-		EventNames: []string{filepath.Join(expandedBaseDir, "places.sqlite-wal")},
+		EventNames: browser.WatchedPaths,
 		ResetWatch: false,
 	}
 
@@ -276,6 +295,9 @@ func (bw *FFBrowser) Load() error {
 	}
 	go CacheDB.SyncToDisk(database.GetDBFullPath())
 
+	// Close the copy places.sqlite
+	err = bw.places.Close()
+
 	return err
 }
 
@@ -388,6 +410,8 @@ func (bw *FFBrowser) fetchUrlChanges(rows *sql.Rows,
 
 	if bk.btype == BkTypeURL {
 		var place FFPlace
+		//TODO: structscan not wokring ? the bookmark change is effectively
+		//detected I could test it by manually looking up bk.fk that changed
 		bw.places.Handle.QueryRowx(QGetBookmarkPlace, bk.fk).StructScan(&place)
 		fflog.Debugf("Changed URL: %s", place.URL)
 
@@ -419,6 +443,11 @@ func (bw *FFBrowser) Run() {
 
 	//TODO: Watching is broken. Try to open a new connection on each
 	//  watch event
+	//
+	err := bw.InitPlacesCopy()
+	if err != nil {
+		fflog.Error(err)
+	}
 
 	startRun := time.Now()
 	//fflog.Debugf("Checking changes since %s",
@@ -456,8 +485,8 @@ func (bw *FFBrowser) Run() {
 		changedURLS := make([]string, 0)
 		bw.lastRunTime = time.Now().UTC()
 
-		//fflog.Debugf("CHANGE ! Time: %s",
+		fflog.Debugf("CHANGE ! Time: %s",
-		//bw.lastRunTime.Local().Format("Mon Jan 2 15:04:05 MST 2006"))
+			bw.lastRunTime.Local().Format("Mon Jan 2 15:04:05 MST 2006"))
 
 		bookmarks := make(map[sqlid]*FFBookmark)
 		places := make(map[sqlid]*FFPlace)
@@ -538,4 +567,11 @@ func (bw *FFBrowser) Run() {
 
 	bw.Stats.LastWatchRunTime = time.Since(startRun)
 	//fflog.Debugf("execution time %s", time.Since(startRun))
+	//
+
+	err = bw.places.Close()
+	if err != nil {
+		fflog.Error(err)
+	}
+
 }

utils/events.go

@@ -29,6 +29,7 @@ func ReduceEvents(interval time.Duration,
 
 		case <-timer.C:
 			if len(events) > 0 {
+				log.Debug("<reduce>: running run event")
 				w.Run()
 
 				// Empty events queue

utils/files.go

@@ -1 +1,70 @@
 package utils
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+)
+
+var (
+	TMPDIR = ""
+)
+
+func copyFileToDst(src string, dst string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+
+	dstFile, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE, 0755)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(dstFile, srcFile)
+	if err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+// Copy files from src glob to dst folder
+func CopyFilesToTmpFolder(srcglob string) error {
+	matches, err := filepath.Glob(os.ExpandEnv(srcglob))
+	if err != nil {
+		return err
+	}
+
+	for _, v := range matches {
+		dstFile := path.Join(TMPDIR, path.Base(v))
+		err = copyFileToDst(v, dstFile)
+		if err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+
+}
+
+func CleanFiles() {
+	log.Debugf("Cleaning files <%s>", TMPDIR)
+	err := os.RemoveAll(TMPDIR)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+}
+
+func init() {
+	var err error
+	TMPDIR, err = ioutil.TempDir("", "gomark*")
+	if err != nil {
+		log.Fatal(err)
+	}
+}

watch/watcher.go

@@ -78,7 +78,7 @@ func WatcherThread(w Watchable) {
 								w.Run()
 							}
 
-							//log.Warning("event: %v | eventName: %v", event.Op, event.Name)
+							//log.Warningf("event: %v | eventName: %v", event.Op, event.Name)
 
 							if watched.ResetWatch {
 								log.Debugf("resetting watchers")