2022-09-20 10:11:16 +00:00
|
|
|
// TODO: unit test critical error should shutdown the browser
|
|
|
|
// TODO: shutdown procedure (also close reducer)
|
2022-10-23 13:08:06 +00:00
|
|
|
// TODO: handle flag management from this package
|
|
|
|
package firefox
|
2018-06-08 16:27:33 +00:00
|
|
|
|
2018-06-14 00:30:18 +00:00
|
|
|
import (
|
2018-11-05 01:40:11 +00:00
|
|
|
"database/sql"
|
2019-02-22 18:50:26 +00:00
|
|
|
"errors"
|
2019-03-01 17:30:50 +00:00
|
|
|
"fmt"
|
2018-06-14 14:42:54 +00:00
|
|
|
"path"
|
2018-11-20 17:33:37 +00:00
|
|
|
"path/filepath"
|
2022-12-11 22:06:07 +00:00
|
|
|
"strings"
|
2018-10-25 16:09:03 +00:00
|
|
|
"time"
|
2018-11-01 16:00:58 +00:00
|
|
|
|
2020-11-06 17:50:36 +00:00
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/database"
|
2022-10-23 13:08:06 +00:00
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/logging"
|
2023-01-31 21:21:39 +00:00
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/modules"
|
2022-12-27 12:24:41 +00:00
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/mozilla"
|
2023-01-31 21:21:39 +00:00
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/profiles"
|
|
|
|
|
|
|
|
// "git.sp4ke.xyz/sp4ke/gomark/profiles"
|
2020-11-06 17:50:36 +00:00
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/tree"
|
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/utils"
|
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/watch"
|
2020-08-12 18:13:01 +00:00
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
"github.com/fsnotify/fsnotify"
|
2018-11-30 03:15:08 +00:00
|
|
|
sqlite3 "github.com/mattn/go-sqlite3"
|
2018-06-14 00:30:18 +00:00
|
|
|
)
|
|
|
|
|
2022-10-23 13:08:06 +00:00
|
|
|
var (
|
|
|
|
ErrInitFirefox = errors.New("could not start Firefox watcher")
|
2022-10-27 22:33:20 +00:00
|
|
|
log = logging.GetLogger("FF")
|
2022-10-23 13:08:06 +00:00
|
|
|
)
|
|
|
|
|
2018-10-23 20:14:54 +00:00
|
|
|
const (
|
2022-12-15 21:15:11 +00:00
|
|
|
WatchMinJobInterval = 1500 * time.Millisecond
|
2023-02-18 22:08:12 +00:00
|
|
|
TagsBranchName = mozilla.TagsBranchName // name of the `tags` branch in the node tree
|
2018-10-23 20:14:54 +00:00
|
|
|
)
|
|
|
|
|
2022-12-15 21:15:11 +00:00
|
|
|
type sqlid = mozilla.Sqlid
|
2023-01-05 21:33:01 +00:00
|
|
|
type timestamp = int64
|
2018-11-10 12:22:07 +00:00
|
|
|
|
2018-11-23 03:40:10 +00:00
|
|
|
type AutoIncr struct {
|
|
|
|
ID sqlid
|
|
|
|
}
|
|
|
|
|
2022-12-15 21:15:11 +00:00
|
|
|
// TODO!: remove
|
2018-11-13 16:11:16 +00:00
|
|
|
type FFPlace struct {
|
2020-09-13 23:57:39 +00:00
|
|
|
URL string `db:"url"`
|
|
|
|
Description sql.NullString `db:"description"`
|
|
|
|
Title sql.NullString `db:"title"`
|
2018-11-23 03:40:10 +00:00
|
|
|
AutoIncr
|
2018-06-08 16:27:33 +00:00
|
|
|
}
|
|
|
|
|
2023-01-03 13:33:02 +00:00
|
|
|
// TODO!: replace by MergedPlaceBookmark and MozBookmark below
|
2018-11-10 12:22:07 +00:00
|
|
|
type FFBookmark struct {
|
2022-09-28 20:13:03 +00:00
|
|
|
btype sqlid
|
2018-11-23 03:40:10 +00:00
|
|
|
parent sqlid
|
|
|
|
fk sqlid
|
2018-11-13 16:11:16 +00:00
|
|
|
title string
|
2018-11-23 03:40:10 +00:00
|
|
|
id sqlid
|
2018-11-10 12:22:07 +00:00
|
|
|
}
|
|
|
|
|
2022-12-15 21:15:11 +00:00
|
|
|
type MozBookmark = mozilla.MozBookmark
|
|
|
|
type MozFolder = mozilla.MozFolder
|
2022-12-10 01:49:30 +00:00
|
|
|
|
2023-01-05 21:33:01 +00:00
|
|
|
type Firefox struct {
|
|
|
|
*FirefoxConfig
|
|
|
|
|
|
|
|
// sqlite con to places.sqlite
|
|
|
|
places *database.DB
|
|
|
|
|
|
|
|
// All elements stored in URLIndex
|
|
|
|
URLIndexList []string
|
|
|
|
|
|
|
|
// Map from moz_bookmarks tag ids to a tree node
|
2023-02-18 22:08:12 +00:00
|
|
|
// tagMap is used as a quick lookup table into the node tree
|
2023-01-05 21:33:01 +00:00
|
|
|
tagMap map[string]*tree.Node
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// map from moz_bookmarks folder id to a folder node in the tree
|
|
|
|
folderMap map[sqlid]*tree.Node
|
2023-01-05 21:33:01 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// internal folder map used for scanning
|
|
|
|
folderScanMap map[sqlid]*MozFolder
|
2023-01-05 21:33:01 +00:00
|
|
|
|
|
|
|
lastRunTime time.Time
|
|
|
|
}
|
2022-11-21 23:05:24 +00:00
|
|
|
|
2023-01-11 11:45:56 +00:00
|
|
|
// func (ff *Firefox) updateModifiedFolders(since timestamp) ([]*MozFolder, error) {
|
|
|
|
// // Get list of modified folders
|
|
|
|
// var folders = []*MozFolders
|
|
|
|
// folderChangeQuery := map[string]interface
|
2023-02-18 22:08:12 +00:00
|
|
|
//
|
2023-01-11 11:45:56 +00:00
|
|
|
// return nil, nil
|
|
|
|
// }
|
|
|
|
|
2022-12-05 22:21:23 +00:00
|
|
|
// scan all folders from moz_bookmarks and load them into the node tree
|
2023-01-05 21:33:01 +00:00
|
|
|
// takes a timestamp(int64) parameter to select folders based on last modified date
|
|
|
|
func (ff *Firefox) scanFolders(since timestamp) ([]*MozFolder, error) {
|
2022-12-05 22:21:23 +00:00
|
|
|
|
|
|
|
var folders []*MozFolder
|
2023-02-18 22:08:12 +00:00
|
|
|
ff.folderScanMap = make(map[sqlid]*MozFolder)
|
2023-01-05 21:33:01 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
folderQueryArgs := map[string]interface{}{
|
|
|
|
"change_since": since,
|
|
|
|
}
|
2023-01-05 21:33:01 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
boundQuery, args, err := ff.places.Handle.BindNamed(mozilla.QFolders, folderQueryArgs)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-01-05 21:33:01 +00:00
|
|
|
|
|
|
|
err = ff.places.Handle.Select(&folders, boundQuery, args...)
|
2023-02-18 22:08:12 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-12-10 01:49:30 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// store all folders in a hashmap for easier tree construction
|
|
|
|
for _, folder := range folders {
|
|
|
|
ff.folderScanMap[folder.Id] = folder
|
|
|
|
}
|
2022-12-05 22:21:23 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
for _, folder := range folders {
|
|
|
|
// Ignore the `tags` virtual folder
|
|
|
|
if folder.Id != 4 {
|
|
|
|
ff.addFolderNode(*folder)
|
|
|
|
}
|
|
|
|
}
|
2022-12-05 22:21:23 +00:00
|
|
|
|
|
|
|
return folders, err
|
|
|
|
}
|
|
|
|
|
2023-01-05 21:33:01 +00:00
|
|
|
// load bookmarks and tags into the node tree then attach them to
|
|
|
|
// their assigned folder hierarchy
|
|
|
|
func (ff *Firefox) loadBookmarksToTree(bookmarks []*MozBookmark) {
|
2022-12-11 22:06:07 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
for _, bkEntry := range bookmarks {
|
|
|
|
// Create/Update URL node and apply tag node
|
|
|
|
ok, urlNode := ff.addURLNode(bkEntry.Url, bkEntry.Title, bkEntry.PlDesc)
|
|
|
|
if !ok {
|
|
|
|
log.Infof("url <%s> already in url index", bkEntry.Url)
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Iterate through bookmark tags and synchronize new tags with
|
|
|
|
* the node tree.
|
|
|
|
*/
|
|
|
|
for _, tagName := range strings.Split(bkEntry.Tags, ",") {
|
|
|
|
if tagName == "" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
seen, tagNode := ff.addTagNode(tagName)
|
|
|
|
if !seen {
|
|
|
|
log.Infof("tag <%s> already in tag map", tagNode.Name)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add tag name to urlnode tags
|
|
|
|
urlNode.Tags = utils.Extends(urlNode.Tags, tagNode.Name)
|
|
|
|
|
|
|
|
// Add URL node as child of Tag node
|
|
|
|
// Parent will be a folder or nothing?
|
|
|
|
tree.AddChild(ff.tagMap[tagNode.Name], urlNode)
|
|
|
|
|
|
|
|
ff.CurrentUrlCount++
|
|
|
|
}
|
|
|
|
|
|
|
|
// Link this URL node to its corresponding folder node if it exists.
|
|
|
|
//TODO: add all parent folders in the tags list of this url node
|
|
|
|
folderNode, fOk := ff.folderMap[bkEntry.ParentId]
|
|
|
|
// If we found the parent folder
|
|
|
|
if fOk {
|
|
|
|
tree.AddChild(folderNode, urlNode)
|
|
|
|
}
|
|
|
|
}
|
2023-01-05 21:33:01 +00:00
|
|
|
}
|
2022-12-15 02:22:54 +00:00
|
|
|
|
2023-01-05 21:33:01 +00:00
|
|
|
// scans bookmarks from places.sqlite and loads them into the node tree
|
|
|
|
func (ff *Firefox) scanBookmarks() ([]*MozBookmark, error) {
|
2022-12-15 02:22:54 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// scan folders and load them into node tree
|
|
|
|
_, err := ff.scanFolders(0)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-01-05 21:33:01 +00:00
|
|
|
|
|
|
|
var bookmarks []*MozBookmark
|
|
|
|
|
|
|
|
dotx, err := database.DotxQueryEmbedFS(mozilla.EmbeddedSqlQueries, mozilla.MozBookmarkQueryFile)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
err = dotx.Select(ff.places.Handle, &bookmarks, mozilla.MozBookmarkQuery)
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// load bookmarks and tags into the node tree
|
|
|
|
// then attach them to their assigned folder hierarchy
|
2023-01-05 21:33:01 +00:00
|
|
|
|
2022-12-05 22:21:23 +00:00
|
|
|
return bookmarks, err
|
2022-11-21 23:05:24 +00:00
|
|
|
}
|
|
|
|
|
2023-01-05 21:33:01 +00:00
|
|
|
func (ff *Firefox) scanModifiedBookmarks(since timestamp) ([]*MozBookmark, error) {
|
2023-02-18 22:08:12 +00:00
|
|
|
// scan new/modifed folders and load them into node tree
|
|
|
|
_, err := ff.scanFolders(since)
|
|
|
|
// tree.PrintTree(ff.NodeTree)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-10-27 22:33:20 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
var bookmarks []*MozBookmark
|
2022-10-27 22:33:20 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
dotx, err := database.DotxQueryEmbedFS(mozilla.EmbeddedSqlQueries,
|
|
|
|
mozilla.MozChangedBookmarkQueryFile)
|
2022-10-27 22:33:20 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2023-01-05 21:33:01 +00:00
|
|
|
}
|
2022-12-10 01:49:30 +00:00
|
|
|
|
2023-01-05 21:33:01 +00:00
|
|
|
queryArgs := map[string]interface{}{
|
2023-02-18 22:08:12 +00:00
|
|
|
"change_since": since,
|
2023-01-05 21:33:01 +00:00
|
|
|
}
|
2022-12-10 01:49:30 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// query, args, err := dotx.NamedQuery(ff.places.Handle, mozilla.MozChangedBookmarkQuery, queryArgs)
|
|
|
|
boundQuery, args, err := dotx.BindNamed(ff.places.Handle, mozilla.MozChangedBookmarkQuery, queryArgs)
|
2023-01-05 21:33:01 +00:00
|
|
|
if err != nil {
|
2023-02-18 22:08:12 +00:00
|
|
|
return nil, err
|
2023-01-05 21:33:01 +00:00
|
|
|
}
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
err = ff.places.Handle.Select(&bookmarks, boundQuery, args...)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-01-05 21:33:01 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
return bookmarks, err
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
2023-01-31 21:21:39 +00:00
|
|
|
modules.RegisterBrowser(Firefox{FirefoxConfig: FFConfig})
|
2022-10-27 22:33:20 +00:00
|
|
|
//TIP: cmd.RegisterModCommand(BrowserName, &cli.Command{
|
2022-12-04 20:01:53 +00:00
|
|
|
// Name: "test",
|
|
|
|
// })
|
|
|
|
// cmd.RegisterModCommand(BrowserName, &cli.Command{
|
|
|
|
// Name: "test2",
|
|
|
|
// })
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
2022-11-07 21:34:01 +00:00
|
|
|
func NewFirefox() *Firefox {
|
|
|
|
return &Firefox{
|
|
|
|
FirefoxConfig: FFConfig,
|
|
|
|
places: &database.DB{},
|
|
|
|
URLIndexList: []string{},
|
2022-12-11 22:06:07 +00:00
|
|
|
tagMap: map[string]*tree.Node{},
|
2023-02-18 22:08:12 +00:00
|
|
|
folderMap: map[sqlid]*tree.Node{},
|
2022-11-07 21:34:01 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-01-31 21:21:39 +00:00
|
|
|
func (f Firefox) ModInfo() modules.ModInfo {
|
|
|
|
return modules.ModInfo{
|
|
|
|
ID: modules.ModID(f.Name),
|
2022-11-07 19:07:13 +00:00
|
|
|
//HACK: duplicate instance with init().RegisterBrowser ??
|
2023-01-31 21:21:39 +00:00
|
|
|
New: func() modules.Module {
|
2022-11-07 21:34:01 +00:00
|
|
|
return NewFirefox()
|
2022-11-07 19:07:13 +00:00
|
|
|
},
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
2018-11-01 16:00:58 +00:00
|
|
|
}
|
|
|
|
|
2023-01-31 21:21:39 +00:00
|
|
|
// Implement the profiles.ProfileManager interface
|
|
|
|
func (f *Firefox) GetProfiles() ([]*profiles.Profile, error) {
|
|
|
|
return FirefoxProfileManager.GetProfiles()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *Firefox) GetDefaultProfile() (*profiles.Profile, error) {
|
|
|
|
return FirefoxProfileManager.GetDefaultProfile()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *Firefox) GetProfilePath(p profiles.Profile) string {
|
|
|
|
return filepath.Join(FirefoxProfileManager.ConfigDir, p.Path)
|
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// TEST:
|
2023-02-18 22:08:12 +00:00
|
|
|
// TODO: implement watching of multiple profiles
|
|
|
|
// NOTE: should be done at core gomark level where multiple instances
|
|
|
|
// are spawned for each profile
|
2022-10-27 22:33:20 +00:00
|
|
|
// Implements browser.Initializer interface
|
2023-02-18 22:08:12 +00:00
|
|
|
func (f *Firefox) Init(ctx *modules.Context) error {
|
2022-10-27 22:33:20 +00:00
|
|
|
log.Infof("initializing <%s>", f.Name)
|
|
|
|
bookmarkPath, err := f.BookmarkPath()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2022-10-07 20:17:21 +00:00
|
|
|
}
|
2018-11-01 16:00:58 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
log.Debugf("bookmark path is: %s", bookmarkPath)
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
watchedPath, err := filepath.EvalSymlinks(f.BkDir)
|
2022-10-27 22:33:20 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// Setup watcher
|
2022-10-27 22:33:20 +00:00
|
|
|
w := &watch.Watch{
|
2023-02-18 22:08:12 +00:00
|
|
|
Path: watchedPath,
|
2022-10-27 22:33:20 +00:00
|
|
|
EventTypes: []fsnotify.Op{fsnotify.Write},
|
2023-02-18 22:08:12 +00:00
|
|
|
EventNames: []string{filepath.Join(watchedPath, "places.sqlite-wal")},
|
2022-10-27 22:33:20 +00:00
|
|
|
ResetWatch: false,
|
|
|
|
}
|
|
|
|
|
2023-01-31 21:21:39 +00:00
|
|
|
modules.SetupWatchersWithReducer(f.BrowserConfig, modules.ReducerChanLen, w)
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
*Run reducer to avoid duplicate jobs when a batch of events is received
|
|
|
|
*/
|
|
|
|
// TODO!: make a new copy of places for every new event change
|
|
|
|
|
|
|
|
// Add a reducer to the watcher
|
2022-12-15 21:15:11 +00:00
|
|
|
go watch.ReduceEvents(WatchMinJobInterval, f)
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
return nil
|
2019-03-01 17:30:50 +00:00
|
|
|
}
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
func (f *Firefox) Watch() *watch.WatchDescriptor {
|
|
|
|
return f.GetWatcher()
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
2023-01-31 21:21:39 +00:00
|
|
|
func (f Firefox) Config() *modules.BrowserConfig {
|
2022-11-07 19:07:13 +00:00
|
|
|
return f.BrowserConfig
|
2022-11-01 13:27:19 +00:00
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// Firefox custom logic for preloading the bookmarks when the browser module
|
2023-01-31 21:21:39 +00:00
|
|
|
// starts. Implements modules.Loader interface.
|
2022-10-27 22:33:20 +00:00
|
|
|
func (f *Firefox) Load() error {
|
2023-02-18 22:08:12 +00:00
|
|
|
pc, err := f.initPlacesCopy()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-12-27 12:24:41 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
defer pc.Clean()
|
2022-10-27 22:33:20 +00:00
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// load all bookmarks
|
2022-10-27 22:33:20 +00:00
|
|
|
start := time.Now()
|
2023-02-18 22:08:12 +00:00
|
|
|
bookmarks, err := f.scanBookmarks()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
f.loadBookmarksToTree(bookmarks)
|
|
|
|
|
2022-12-10 01:49:30 +00:00
|
|
|
f.LastFullTreeParseTime = time.Since(start)
|
2022-10-27 22:33:20 +00:00
|
|
|
f.lastRunTime = time.Now().UTC()
|
|
|
|
|
|
|
|
log.Debugf("parsed %d bookmarks and %d nodes in %s",
|
2022-12-10 01:49:30 +00:00
|
|
|
f.CurrentUrlCount,
|
|
|
|
f.CurrentNodeCount,
|
|
|
|
f.LastFullTreeParseTime)
|
2022-12-26 18:15:00 +00:00
|
|
|
f.Reset()
|
2019-03-01 18:03:48 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// Sync the URLIndex to the buffer
|
|
|
|
// We do not use the NodeTree here as firefox tags are represented
|
|
|
|
// as a flat tree which is not efficient, we use the go hashmap instead
|
2019-03-01 18:03:48 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
database.SyncURLIndexToBuffer(f.URLIndexList, f.URLIndex, f.BufferDB)
|
|
|
|
|
|
|
|
// Handle empty cache
|
|
|
|
if empty, err := database.Cache.DB.IsEmpty(); empty {
|
2019-03-01 18:03:48 +00:00
|
|
|
if err != nil {
|
2022-10-27 22:33:20 +00:00
|
|
|
return err
|
2019-03-01 18:03:48 +00:00
|
|
|
}
|
2022-10-27 22:33:20 +00:00
|
|
|
log.Info("cache empty: loading buffer to Cachedb")
|
2019-03-01 17:30:50 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
f.BufferDB.CopyTo(database.Cache.DB)
|
|
|
|
|
|
|
|
log.Debugf("syncing <%s> to disk", database.Cache.DB.Name)
|
|
|
|
} else {
|
|
|
|
f.BufferDB.SyncTo(database.Cache.DB)
|
2019-03-01 17:30:50 +00:00
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
database.Cache.DB.SyncToDisk(database.GetDBFullPath())
|
|
|
|
|
|
|
|
//DEBUG:
|
2022-12-10 01:49:30 +00:00
|
|
|
// tree.PrintTree(f.NodeTree)
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
// Close the copy places.sqlite
|
2022-12-27 12:24:41 +00:00
|
|
|
err = f.places.Close()
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
return err
|
2019-03-01 17:30:50 +00:00
|
|
|
}
|
|
|
|
|
2023-01-31 21:21:39 +00:00
|
|
|
// Implements modules.Runner interface
|
2023-01-05 21:33:01 +00:00
|
|
|
func (ff *Firefox) Run() {
|
2023-02-18 22:08:12 +00:00
|
|
|
startRun := time.Now()
|
2023-01-05 21:33:01 +00:00
|
|
|
|
|
|
|
pc, err := ff.initPlacesCopy()
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
2023-02-18 22:08:12 +00:00
|
|
|
defer pc.Clean()
|
2023-01-05 21:33:01 +00:00
|
|
|
|
|
|
|
log.Debugf("Checking changes since <%d> %s",
|
|
|
|
ff.lastRunTime.UTC().UnixNano()/1000,
|
|
|
|
ff.lastRunTime.Local().Format("Mon Jan 2 15:04:05 MST 2006"))
|
2023-02-18 22:08:12 +00:00
|
|
|
|
|
|
|
scanSince := ff.lastRunTime.UTC().UnixNano() / 1000
|
|
|
|
|
|
|
|
bookmarks, err := ff.scanModifiedBookmarks(scanSince)
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
ff.loadBookmarksToTree(bookmarks)
|
2023-01-05 21:33:01 +00:00
|
|
|
// tree.PrintTree(ff.NodeTree)
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
database.SyncURLIndexToBuffer(ff.URLIndexList, ff.URLIndex, ff.BufferDB)
|
|
|
|
ff.BufferDB.SyncTo(database.Cache.DB)
|
|
|
|
database.Cache.DB.SyncToDisk(database.GetDBFullPath())
|
2023-01-05 21:33:01 +00:00
|
|
|
|
|
|
|
ff.LastWatchRunTime = time.Since(startRun)
|
|
|
|
ff.lastRunTime = time.Now().UTC()
|
|
|
|
}
|
|
|
|
|
2023-02-07 16:10:17 +00:00
|
|
|
// Implement moduls.Shutdowner
|
2022-10-27 22:33:20 +00:00
|
|
|
func (f *Firefox) Shutdown() {
|
|
|
|
log.Debugf("shutting down ... ")
|
|
|
|
|
|
|
|
if f.places != nil {
|
|
|
|
|
|
|
|
err := f.places.Close()
|
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
|
|
|
}
|
2019-03-01 17:30:50 +00:00
|
|
|
}
|
|
|
|
|
2022-12-10 01:49:30 +00:00
|
|
|
func (ff *Firefox) getPathToPlacesCopy() string {
|
|
|
|
return path.Join(utils.TMPDIR, ff.BkFile)
|
2020-09-09 13:07:29 +00:00
|
|
|
}
|
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// HACK: addUrl and addTag share a lot of code, find a way to reuse shared code
|
|
|
|
// and only pass extra details about tag/url along in some data structure
|
|
|
|
// PROBLEM: tag nodes use IDs and URL nodes use URL as hashes
|
2023-02-18 22:08:12 +00:00
|
|
|
func (f *Firefox) addURLNode(url, title, desc string) (bool, *tree.Node) {
|
2022-12-05 22:21:23 +00:00
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
var urlNode *tree.Node
|
2022-11-01 13:27:19 +00:00
|
|
|
iUrlNode, exists := f.URLIndex.Get(url)
|
2022-10-07 20:17:21 +00:00
|
|
|
if !exists {
|
|
|
|
urlNode := &tree.Node{
|
|
|
|
Name: title,
|
2022-11-01 13:27:19 +00:00
|
|
|
Type: tree.URLNode,
|
2022-10-07 20:17:21 +00:00
|
|
|
URL: url,
|
|
|
|
Desc: desc,
|
|
|
|
}
|
|
|
|
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Debugf("inserting url %s in url index", url)
|
2022-11-01 13:27:19 +00:00
|
|
|
f.URLIndex.Insert(url, urlNode)
|
|
|
|
f.URLIndexList = append(f.URLIndexList, url)
|
2023-02-18 22:08:12 +00:00
|
|
|
f.CurrentNodeCount++
|
2022-10-07 20:17:21 +00:00
|
|
|
|
|
|
|
return true, urlNode
|
|
|
|
} else {
|
|
|
|
urlNode = iUrlNode.(*tree.Node)
|
|
|
|
}
|
|
|
|
|
|
|
|
return false, urlNode
|
|
|
|
}
|
|
|
|
|
2022-12-05 22:21:23 +00:00
|
|
|
// adds a new tagNode if it is not yet in the tagMap
|
2022-10-07 20:17:21 +00:00
|
|
|
// returns true if tag added or false if already existing
|
|
|
|
// returns the created tagNode
|
2022-12-11 22:06:07 +00:00
|
|
|
func (ff *Firefox) addTagNode(tagName string) (bool, *tree.Node) {
|
2023-02-18 22:08:12 +00:00
|
|
|
// Check if "tags" branch exists or create it
|
|
|
|
var branchOk bool
|
|
|
|
var tagsBranch *tree.Node
|
|
|
|
for _, c := range ff.NodeTree.Children {
|
|
|
|
if c.Name == TagsBranchName {
|
|
|
|
branchOk = true
|
|
|
|
tagsBranch = c
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if !branchOk {
|
|
|
|
tagsBranch = &tree.Node{
|
|
|
|
Name: TagsBranchName,
|
|
|
|
}
|
|
|
|
tree.AddChild(ff.NodeTree, tagsBranch)
|
|
|
|
}
|
2022-12-11 22:06:07 +00:00
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// node, exists :=
|
2022-12-11 22:06:07 +00:00
|
|
|
node, exists := ff.tagMap[tagName]
|
2022-10-07 20:17:21 +00:00
|
|
|
if exists {
|
|
|
|
return false, node
|
|
|
|
}
|
|
|
|
|
|
|
|
tagNode := &tree.Node{
|
|
|
|
Name: tagName,
|
2022-11-01 13:27:19 +00:00
|
|
|
Type: tree.TagNode,
|
2022-12-10 01:49:30 +00:00
|
|
|
Parent: ff.NodeTree, // root node
|
2022-10-07 20:17:21 +00:00
|
|
|
}
|
|
|
|
|
2022-12-11 22:06:07 +00:00
|
|
|
tree.AddChild(tagsBranch, tagNode)
|
|
|
|
ff.tagMap[tagName] = tagNode
|
2022-12-26 18:15:00 +00:00
|
|
|
ff.CurrentNodeCount++
|
2022-10-07 20:17:21 +00:00
|
|
|
|
|
|
|
return true, tagNode
|
|
|
|
}
|
|
|
|
|
2022-12-05 22:21:23 +00:00
|
|
|
// add a folder node to the parsed node tree under the specified folder parent
|
2022-12-10 01:49:30 +00:00
|
|
|
// returns true if a new folder is created and false if folder already exists
|
2023-02-18 22:08:12 +00:00
|
|
|
func (ff *Firefox) addFolderNode(folder MozFolder) (bool, *tree.Node) {
|
|
|
|
|
|
|
|
// use hashmap.RBTree to keep an index of scanned folders pointing
|
|
|
|
// to their corresponding nodes in the tree
|
|
|
|
|
|
|
|
folderNode, seen := ff.folderMap[folder.Id]
|
|
|
|
|
|
|
|
if seen {
|
|
|
|
// Update folder name if changed
|
|
|
|
|
|
|
|
//TODO!: trigger bookmark tag change in gomarks.db
|
|
|
|
if folderNode.Name != folder.Title &&
|
|
|
|
// Ignore root folders since we use our custom names
|
|
|
|
!utils.InList([]int{2, 3, 5, 6}, int(folder.Id)) {
|
|
|
|
log.Debugf("folder node <%s> updated to <%s>", folderNode.Name, folder.Title)
|
|
|
|
folderNode.Name = folder.Title
|
|
|
|
}
|
|
|
|
|
|
|
|
return false, folderNode
|
|
|
|
}
|
|
|
|
|
|
|
|
folderNode = &tree.Node{
|
|
|
|
// Name: folder.Title,
|
|
|
|
Type: tree.FolderNode,
|
|
|
|
}
|
|
|
|
|
|
|
|
// keeping the same folder structure as Firefox
|
|
|
|
|
|
|
|
// If this folders' is a Firefox root folder use the appropriate title
|
|
|
|
// then add it to the root node
|
|
|
|
if utils.InList([]int{2, 3, 5, 6}, int(folder.Id)) {
|
|
|
|
folderNode.Name = mozilla.RootFolderNames[folder.Id]
|
|
|
|
tree.AddChild(ff.NodeTree, folderNode)
|
|
|
|
} else {
|
|
|
|
folderNode.Name = folder.Title
|
|
|
|
}
|
|
|
|
|
|
|
|
// check if folder's parent is already in the tree
|
|
|
|
fParent, ok := ff.folderMap[folder.Parent]
|
|
|
|
|
|
|
|
// if we already saw folder's parent add it underneath
|
|
|
|
if ok {
|
|
|
|
tree.AddChild(fParent, folderNode)
|
|
|
|
|
|
|
|
// if we never saw this folders' parent
|
|
|
|
} else if folder.Parent != 1 { // recursively build the parent of this folder
|
|
|
|
_, ok := ff.folderScanMap[folder.Parent]
|
|
|
|
if ok {
|
|
|
|
_, newParentNode := ff.addFolderNode(*ff.folderScanMap[folder.Parent])
|
|
|
|
tree.AddChild(newParentNode, folderNode)
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Store a pointer to this folder
|
|
|
|
ff.folderMap[folder.Id] = folderNode
|
2022-12-10 01:49:30 +00:00
|
|
|
ff.CurrentNodeCount++
|
|
|
|
|
|
|
|
return true, folderNode
|
2020-09-09 13:07:29 +00:00
|
|
|
}
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// TODO: retire this function after scanBookmarks() is implemented
|
2022-10-07 20:17:21 +00:00
|
|
|
// load all bookmarks from `places.sqlite` and store them in BaseBrowser.NodeTree
|
|
|
|
// this method is used the first time gomark is started or to extract bookmarks
|
|
|
|
// using a command
|
2022-11-16 23:06:02 +00:00
|
|
|
func loadBookmarks(f *Firefox) {
|
2022-11-01 13:27:19 +00:00
|
|
|
log.Debugf("root tree children len is %d", len(f.NodeTree.Children))
|
2018-10-24 16:16:42 +00:00
|
|
|
//QGetTags := "SELECT id,title from moz_bookmarks WHERE parent = %d"
|
2018-11-20 17:33:37 +00:00
|
|
|
//
|
2018-10-23 20:14:54 +00:00
|
|
|
|
2022-12-15 21:15:11 +00:00
|
|
|
rows, err := f.places.Handle.Query(mozilla.QgetBookmarks, mozilla.TagsID)
|
2020-09-01 12:43:01 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
2018-11-30 03:15:08 +00:00
|
|
|
|
|
|
|
// Locked database is critical
|
|
|
|
if e, ok := err.(sqlite3.Error); ok {
|
|
|
|
if e.Code == sqlite3.ErrBusy {
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Critical(err)
|
2022-11-01 13:27:19 +00:00
|
|
|
f.Shutdown()
|
2018-11-30 03:15:08 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
2022-11-01 13:27:19 +00:00
|
|
|
log.Errorf("%s: %s", f.places.Name, err)
|
2018-11-20 17:33:37 +00:00
|
|
|
return
|
2018-10-28 19:19:12 +00:00
|
|
|
}
|
2018-10-23 20:14:54 +00:00
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// Rebuilding node tree
|
|
|
|
// Note: the node tree is built only for compatilibity with tree based
|
|
|
|
// bookmark parsing and might later be useful for debug/UI features.
|
|
|
|
// For efficiency reading after the initial Load() from
|
2018-11-13 18:54:20 +00:00
|
|
|
// places.sqlite should be done using a loop instad of tree traversal.
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-10-26 01:04:26 +00:00
|
|
|
/*
|
|
|
|
*This pass is used only for fetching bookmarks from firefox.
|
|
|
|
*Checking against the URLIndex should not be done here
|
|
|
|
*/
|
2018-10-23 20:14:54 +00:00
|
|
|
for rows.Next() {
|
2018-10-24 16:16:42 +00:00
|
|
|
var url, title, tagTitle, desc string
|
2018-11-23 03:40:10 +00:00
|
|
|
var tagId sqlid
|
2022-10-07 20:17:21 +00:00
|
|
|
|
2018-10-24 16:16:42 +00:00
|
|
|
err = rows.Scan(&url, &title, &desc, &tagId, &tagTitle)
|
2022-10-23 13:08:06 +00:00
|
|
|
// log.Debugf("%s|%s|%s|%d|%s", url, title, desc, tagId, tagTitle)
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Error(err)
|
2018-10-28 19:19:12 +00:00
|
|
|
}
|
2018-10-24 16:16:42 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is the first time we see this tag
|
|
|
|
* add it to the tagMap and create its node
|
|
|
|
*/
|
2022-12-11 22:06:07 +00:00
|
|
|
ok, tagNode := f.addTagNode(tagTitle)
|
2022-10-07 20:17:21 +00:00
|
|
|
if !ok {
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Infof("tag <%s> already in tag map", tagNode.Name)
|
2018-10-24 16:16:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add the url to the tag
|
2022-11-16 23:06:02 +00:00
|
|
|
// NOTE: this call is responsible for updating URLIndexList
|
2023-02-18 22:08:12 +00:00
|
|
|
ok, urlNode := f.addURLNode(url, title, desc)
|
2022-10-07 20:17:21 +00:00
|
|
|
if !ok {
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Infof("url <%s> already in url index", url)
|
2018-10-26 01:04:26 +00:00
|
|
|
}
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// Add tag name to urlnode tags
|
2018-10-26 01:04:26 +00:00
|
|
|
urlNode.Tags = append(urlNode.Tags, tagNode.Name)
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-10-26 01:04:26 +00:00
|
|
|
// Set tag as parent to urlnode
|
2022-12-11 22:06:07 +00:00
|
|
|
tree.AddChild(f.tagMap[tagTitle], urlNode)
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2022-12-26 18:15:00 +00:00
|
|
|
f.CurrentUrlCount++
|
2018-10-23 20:14:54 +00:00
|
|
|
}
|
2022-10-07 20:17:21 +00:00
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
log.Debugf("root tree children len is %d", len(f.NodeTree.Children))
|
2018-11-13 16:11:16 +00:00
|
|
|
}
|
2018-10-26 01:04:26 +00:00
|
|
|
|
2022-12-10 01:49:30 +00:00
|
|
|
// Copies places.sqlite to a tmp dir to read a VFS lock sqlite db
|
2022-12-27 12:24:41 +00:00
|
|
|
func (f *Firefox) initPlacesCopy() (mozilla.PlaceCopyJob, error) {
|
2023-02-18 22:08:12 +00:00
|
|
|
// create a new copy job
|
|
|
|
pc := mozilla.NewPlaceCopyJob()
|
2022-12-27 12:24:41 +00:00
|
|
|
|
|
|
|
err := utils.CopyFilesToTmpFolder(path.Join(f.BkDir, f.BkFile+"*"), pc.Path())
|
2022-12-10 01:49:30 +00:00
|
|
|
if err != nil {
|
2022-12-27 12:24:41 +00:00
|
|
|
return pc, fmt.Errorf("could not copy places.sqlite to tmp folder: %s", err)
|
2022-12-10 01:49:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
opts := FFConfig.PlacesDSN
|
|
|
|
|
|
|
|
f.places, err = database.NewDB("places",
|
|
|
|
// using the copied places file instead of the original to avoid
|
|
|
|
// sqlite vfs lock errors
|
2022-12-27 12:24:41 +00:00
|
|
|
path.Join(pc.Path(), f.BkFile),
|
2022-12-10 01:49:30 +00:00
|
|
|
database.DBTypeFileDSN, opts).Init()
|
|
|
|
|
|
|
|
if err != nil {
|
2022-12-27 12:24:41 +00:00
|
|
|
return pc, err
|
2022-12-10 01:49:30 +00:00
|
|
|
}
|
|
|
|
|
2022-12-27 12:24:41 +00:00
|
|
|
return pc, nil
|
2022-12-10 01:49:30 +00:00
|
|
|
}
|