From 53d31f433e658ebd3fd825a438ff516d774243fb Mon Sep 17 00:00:00 2001 From: Chakib Ben Ziane Date: Tue, 27 Dec 2022 13:24:41 +0100 Subject: [PATCH] make a separate places copy for each Load/Run job, missing tests --- firefox/firefox.go | 49 ++++++++++++++++++++++------------------------ mozilla/places.go | 47 ++++++++++++++++++++++++++++++++++++++++++-- utils/files.go | 4 ++-- utils/misc.go | 15 +++++++++++++- 4 files changed, 84 insertions(+), 31 deletions(-) diff --git a/firefox/firefox.go b/firefox/firefox.go index f7be1d9..a7f29a0 100644 --- a/firefox/firefox.go +++ b/firefox/firefox.go @@ -1,9 +1,6 @@ // TODO: unit test critical error should shutdown the browser // TODO: shutdown procedure (also close reducer) -// TODO: migrate ff commands to ff module // TODO: handle flag management from this package -// TODO: * Implement Init() and Load() for firefox -// TODO: move sql files to mozilla package firefox import ( @@ -15,10 +12,10 @@ import ( "strings" "time" - "git.sp4ke.xyz/sp4ke/gomark/mozilla" "git.sp4ke.xyz/sp4ke/gomark/browsers" "git.sp4ke.xyz/sp4ke/gomark/database" "git.sp4ke.xyz/sp4ke/gomark/logging" + "git.sp4ke.xyz/sp4ke/gomark/mozilla" "git.sp4ke.xyz/sp4ke/gomark/tree" "git.sp4ke.xyz/sp4ke/gomark/utils" "git.sp4ke.xyz/sp4ke/gomark/watch" @@ -223,10 +220,6 @@ func (f *Firefox) Init() error { log.Debugf("bookmark path is: %s", bookmarkPath) - err = f.initPlacesCopy() - if err != nil { - return err - } // Setup watcher expandedBaseDir, err := filepath.EvalSymlinks(f.BkDir) @@ -262,9 +255,16 @@ func (f Firefox) Config() *browsers.BrowserConfig { return f.BrowserConfig } + // Firefox custom logic for preloading the bookmarks when the browser module // starts. Implements browsers.Loader interface. func (f *Firefox) Load() error { + pc, err := f.initPlacesCopy() + if err != nil { + return err + } + + defer pc.Clean() // load all bookmarks start := time.Now() @@ -304,24 +304,28 @@ func (f *Firefox) Load() error { // tree.PrintTree(f.NodeTree) // Close the copy places.sqlite - err := f.places.Close() + err = f.places.Close() return err } // Implement browsers.Runner interface +// TODO: lock the copied places until the RUN operation is over func (f *Firefox) Run() { startRun := time.Now() - err := f.initPlacesCopy() + pc, err := f.initPlacesCopy() if err != nil { log.Error(err) } + defer pc.Clean() log.Debugf("Checking changes since <%d> %s", f.lastRunTime.UTC().UnixNano()/1000, f.lastRunTime.Local().Format("Mon Jan 2 15:04:05 MST 2006")) + return + queryArgs := map[string]interface{}{ "not_root_tags": []int{mozilla.RootID, mozilla.TagsID}, "last_runtime_utc": f.lastRunTime.UTC().UnixNano() / 1000, @@ -468,15 +472,6 @@ func (f *Firefox) Shutdown() { } } -func (f *Firefox) copyPlacesToTmp() error { - err := utils.CopyFilesToTmpFolder(path.Join(f.BkDir, f.BkFile+"*")) - if err != nil { - return err - } - - return nil -} - func (ff *Firefox) getPathToPlacesCopy() string { return path.Join(utils.TMPDIR, ff.BkFile) } @@ -749,11 +744,13 @@ func (f *Firefox) fetchUrlChanges(rows *sql.Rows, } // Copies places.sqlite to a tmp dir to read a VFS lock sqlite db -func (f *Firefox) initPlacesCopy() error { - err := f.copyPlacesToTmp() +func (f *Firefox) initPlacesCopy() (mozilla.PlaceCopyJob, error) { + // create a new copy job + pc := mozilla.NewPlaceCopyJob() + + err := utils.CopyFilesToTmpFolder(path.Join(f.BkDir, f.BkFile+"*"), pc.Path()) if err != nil { - return fmt.Errorf("could not copy places.sqlite to tmp folder: %s", - err) + return pc, fmt.Errorf("could not copy places.sqlite to tmp folder: %s", err) } opts := FFConfig.PlacesDSN @@ -761,12 +758,12 @@ func (f *Firefox) initPlacesCopy() error { f.places, err = database.NewDB("places", // using the copied places file instead of the original to avoid // sqlite vfs lock errors - f.getPathToPlacesCopy(), + path.Join(pc.Path(), f.BkFile), database.DBTypeFileDSN, opts).Init() if err != nil { - return err + return pc, err } - return nil + return pc, nil } diff --git a/mozilla/places.go b/mozilla/places.go index 6d60c13..8ee1a8c 100644 --- a/mozilla/places.go +++ b/mozilla/places.go @@ -1,12 +1,15 @@ package mozilla import ( - + "os" + "path" "time" + + "git.sp4ke.xyz/sp4ke/gomark/utils" ) // Constants representing the meaning if IDs defined in the table -// moz_bookmarks.id +// moz_bookmarks.id const ( _ = iota // 0 RootID // 1 @@ -86,3 +89,43 @@ func (pb *MergedPlaceBookmark) Datetime() time.Time { return time.Unix(int64(pb.BkLastModified/(1000*1000)), int64(pb.BkLastModified%(1000*1000))*1000).UTC() } + +var CopyJobs []PlaceCopyJob + +type PlaceCopyJob struct { + Id string +} + +func NewPlaceCopyJob() PlaceCopyJob { + pc := PlaceCopyJob{ + Id: utils.GenStringID(5), + } + + err := pc.makePath() + if err != nil { + log.Fatal(err) + } + + CopyJobs = append(CopyJobs, pc) + + return pc +} + +func (pc PlaceCopyJob) makePath() error { + // make sure TMPDIR is not empty + if len(utils.TMPDIR) == 0 { + log.Error("missing tmp dir") + return nil + } + + return os.Mkdir(path.Join(utils.TMPDIR, pc.Id), 0750) +} + +func (pc PlaceCopyJob) Path() string { + return path.Join(utils.TMPDIR, pc.Id) +} + +func (pc PlaceCopyJob) Clean() error { + log.Debugf("cleaning <%s>", pc.Path()) + return os.RemoveAll(pc.Path()) +} diff --git a/utils/files.go b/utils/files.go index 6c5cab5..77ed03b 100644 --- a/utils/files.go +++ b/utils/files.go @@ -41,14 +41,14 @@ func copyFileToDst(src string, dst string) error { } // Copy files from src glob to dst folder -func CopyFilesToTmpFolder(srcglob string) error { +func CopyFilesToTmpFolder(srcglob string, dst string) error { matches, err := filepath.Glob(os.ExpandEnv(srcglob)) if err != nil { return err } for _, v := range matches { - dstFile := path.Join(TMPDIR, path.Base(v)) + dstFile := path.Join(dst, path.Base(v)) err = copyFileToDst(v, dstFile) if err != nil { return err diff --git a/utils/misc.go b/utils/misc.go index 233da81..efb3895 100644 --- a/utils/misc.go +++ b/utils/misc.go @@ -1,6 +1,9 @@ package utils -import "strings" +import ( + "math/rand" + "strings" +) // Return string from slice of bytes func S(value interface{}) string { @@ -43,3 +46,13 @@ func ReplaceInList(l []string, old string, new string) []string { } return result } + +// Generate a unique random string with the specified length +func GenStringID(n int) string { + var letter = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + b := make([]rune, n) + for i := range b { + b[i] = letter[rand.Intn(len(letter))] + } + return string(b) +}