2022-09-20 10:11:16 +00:00
|
|
|
// TODO: unit test critical error should shutdown the browser
|
|
|
|
// TODO: shutdown procedure (also close reducer)
|
2022-10-23 13:08:06 +00:00
|
|
|
// TODO: migrate own commands to here
|
|
|
|
// TODO: handle flag management from this package
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
// TODO: Refactoring:
|
|
|
|
// TODO: * Implement Init() and Load() for firefox
|
2022-10-23 13:08:06 +00:00
|
|
|
package firefox
|
2018-06-08 16:27:33 +00:00
|
|
|
|
2018-06-14 00:30:18 +00:00
|
|
|
import (
|
2018-11-05 01:40:11 +00:00
|
|
|
"database/sql"
|
2019-02-22 18:50:26 +00:00
|
|
|
"errors"
|
2019-03-01 17:30:50 +00:00
|
|
|
"fmt"
|
2018-06-14 14:42:54 +00:00
|
|
|
"path"
|
2018-11-20 17:33:37 +00:00
|
|
|
"path/filepath"
|
2018-10-25 16:09:03 +00:00
|
|
|
"time"
|
2018-11-01 16:00:58 +00:00
|
|
|
|
2020-11-06 17:50:36 +00:00
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/browsers"
|
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/database"
|
2022-10-23 13:08:06 +00:00
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/logging"
|
2020-11-06 17:50:36 +00:00
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/tree"
|
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/utils"
|
|
|
|
"git.sp4ke.xyz/sp4ke/gomark/watch"
|
2020-08-12 18:13:01 +00:00
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
"github.com/fsnotify/fsnotify"
|
2018-11-23 03:40:10 +00:00
|
|
|
"github.com/jmoiron/sqlx"
|
2018-11-30 03:15:08 +00:00
|
|
|
sqlite3 "github.com/mattn/go-sqlite3"
|
2018-06-14 00:30:18 +00:00
|
|
|
)
|
|
|
|
|
2022-10-23 13:08:06 +00:00
|
|
|
// sql queries
|
2018-11-13 16:11:16 +00:00
|
|
|
const (
|
|
|
|
QGetBookmarkPlace = `
|
2018-11-23 03:40:10 +00:00
|
|
|
SELECT *
|
2018-11-13 17:21:02 +00:00
|
|
|
FROM moz_places
|
|
|
|
WHERE id = ?
|
|
|
|
`
|
2022-10-07 20:17:21 +00:00
|
|
|
//TEST:
|
2018-11-23 03:40:10 +00:00
|
|
|
QBookmarksChanged = `
|
2022-10-23 13:08:06 +00:00
|
|
|
SELECT id,type,IFNULL(fk, -1) AS fk,parent,IFNULL(title, '') AS title from moz_bookmarks
|
|
|
|
WHERE(lastModified > :last_runtime_utc
|
|
|
|
AND lastModified < strftime('%s', 'now')*1000*1000
|
|
|
|
AND NOT id IN (:not_root_tags)
|
|
|
|
)
|
2018-11-13 16:11:16 +00:00
|
|
|
`
|
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
//TEST:
|
2022-10-27 22:33:20 +00:00
|
|
|
QgetBookmarks = `
|
2022-10-23 13:08:06 +00:00
|
|
|
WITH bookmarks AS
|
2018-11-13 16:11:16 +00:00
|
|
|
(SELECT moz_places.url AS url,
|
|
|
|
moz_places.description as desc,
|
|
|
|
moz_places.title as urlTitle,
|
|
|
|
moz_bookmarks.parent AS tagId
|
|
|
|
FROM moz_places LEFT OUTER JOIN moz_bookmarks
|
|
|
|
ON moz_places.id = moz_bookmarks.fk
|
|
|
|
WHERE moz_bookmarks.parent
|
|
|
|
IN (SELECT id FROM moz_bookmarks WHERE parent = ? ))
|
|
|
|
|
|
|
|
SELECT url, IFNULL(urlTitle, ''), IFNULL(desc,''),
|
|
|
|
tagId, moz_bookmarks.title AS tagTitle
|
|
|
|
|
|
|
|
FROM bookmarks LEFT OUTER JOIN moz_bookmarks
|
|
|
|
ON tagId = moz_bookmarks.id
|
2022-10-23 13:08:06 +00:00
|
|
|
ORDER BY url
|
|
|
|
`
|
2021-04-08 17:18:54 +00:00
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
//TEST:
|
|
|
|
//TODO:
|
2021-04-08 17:18:54 +00:00
|
|
|
QGetBookmarkFolders = `
|
2022-10-23 13:08:06 +00:00
|
|
|
SELECT
|
2021-04-08 17:18:54 +00:00
|
|
|
moz_places.id as placesId,
|
|
|
|
moz_places.url as url,
|
|
|
|
moz_places.description as description,
|
|
|
|
moz_bookmarks.title as title,
|
|
|
|
moz_bookmarks.fk ISNULL as isFolder
|
|
|
|
|
|
|
|
FROM moz_bookmarks LEFT OUTER JOIN moz_places
|
|
|
|
ON moz_places.id = moz_bookmarks.fk
|
|
|
|
WHERE moz_bookmarks.parent = 3
|
|
|
|
`
|
2018-11-13 16:11:16 +00:00
|
|
|
)
|
|
|
|
|
2022-10-23 13:08:06 +00:00
|
|
|
var (
|
|
|
|
ErrInitFirefox = errors.New("could not start Firefox watcher")
|
2022-10-27 22:33:20 +00:00
|
|
|
log = logging.GetLogger("FF")
|
2022-10-23 13:08:06 +00:00
|
|
|
)
|
|
|
|
|
2018-10-23 20:14:54 +00:00
|
|
|
const (
|
2022-09-28 20:13:03 +00:00
|
|
|
MozMinJobInterval = 1500 * time.Millisecond
|
2018-10-23 20:14:54 +00:00
|
|
|
)
|
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// moz_bookmarks.type
|
2018-11-10 12:22:07 +00:00
|
|
|
const (
|
|
|
|
_ = iota
|
|
|
|
BkTypeURL
|
|
|
|
BkTypeTagFolder
|
|
|
|
)
|
|
|
|
|
2022-09-28 20:13:03 +00:00
|
|
|
type sqlid int64
|
2018-11-10 12:22:07 +00:00
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// moz_bookmarks.id
|
2018-11-10 12:22:07 +00:00
|
|
|
const (
|
2022-10-07 20:17:21 +00:00
|
|
|
_ = iota // 0
|
|
|
|
ffBkRoot // 1
|
|
|
|
ffBkMenu // 2 Main bookmarks menu
|
|
|
|
ffBkToolbar // 3 Bk tookbar that can be toggled under URL zone
|
|
|
|
ffBkTags // 4 Hidden menu used for tags, stored as a flat one level menu
|
|
|
|
ffBkOther // 5 Most bookmarks are automatically stored here
|
|
|
|
ffBkMobile // 6 Mobile bookmarks stored here by default
|
2018-11-10 12:22:07 +00:00
|
|
|
)
|
|
|
|
|
2018-11-23 03:40:10 +00:00
|
|
|
type AutoIncr struct {
|
|
|
|
ID sqlid
|
|
|
|
}
|
|
|
|
|
2018-11-13 16:11:16 +00:00
|
|
|
type FFPlace struct {
|
2020-09-13 23:57:39 +00:00
|
|
|
URL string `db:"url"`
|
|
|
|
Description sql.NullString `db:"description"`
|
|
|
|
Title sql.NullString `db:"title"`
|
2018-11-23 03:40:10 +00:00
|
|
|
AutoIncr
|
2018-06-08 16:27:33 +00:00
|
|
|
}
|
|
|
|
|
2022-12-04 20:01:53 +00:00
|
|
|
// TODO!: replace by MergedPlaceBokmark and MozBookmark below
|
2018-11-10 12:22:07 +00:00
|
|
|
type FFBookmark struct {
|
2022-09-28 20:13:03 +00:00
|
|
|
btype sqlid
|
2018-11-23 03:40:10 +00:00
|
|
|
parent sqlid
|
|
|
|
fk sqlid
|
2018-11-13 16:11:16 +00:00
|
|
|
title string
|
2018-11-23 03:40:10 +00:00
|
|
|
id sqlid
|
2018-11-10 12:22:07 +00:00
|
|
|
}
|
|
|
|
|
2022-12-04 20:01:53 +00:00
|
|
|
// placeId title parentFolderId folders url plDesc lastModified
|
|
|
|
// Type used for scanning from `recursive-all-bookmarks.sql`
|
|
|
|
type MozBookmark struct {
|
|
|
|
PlId sqlid `db:"plId"`
|
|
|
|
Title string
|
|
|
|
Tags string
|
|
|
|
Folders string
|
|
|
|
ParentId sqlid `db:"parentFolderId"`
|
|
|
|
Url string
|
|
|
|
PlDesc string `db:"plDesc"`
|
|
|
|
BkLastModified sqlid `db:"lastModified"`
|
|
|
|
}
|
|
|
|
|
2022-12-05 02:59:22 +00:00
|
|
|
const MozBookmarkQueryFile = "recursive_all_bookmarks.sql"
|
|
|
|
const MozBookmarkQuery = "recursive-all-bookmarks"
|
|
|
|
|
|
|
|
|
|
|
|
|
2022-12-04 20:01:53 +00:00
|
|
|
// Type is used for scanning from `merged-places-bookmarks.sql`
|
2022-11-16 23:06:02 +00:00
|
|
|
// plId plUrl plDescription bkId bkTitle bkLastModified isFolder isTag isBk bkParent
|
2022-11-24 20:25:05 +00:00
|
|
|
type MergedPlaceBookmark struct {
|
2022-11-16 23:06:02 +00:00
|
|
|
PlId sqlid `db:"plId"`
|
|
|
|
PlUrl string `db:"plUrl"`
|
|
|
|
PlDesc string `db:"plDescription"`
|
|
|
|
BkId sqlid `db:"bkId"`
|
|
|
|
BkTitle string `db:"bkTitle"`
|
|
|
|
|
|
|
|
//firefox stores timestamps in milliseconds as integer
|
|
|
|
//sqlite3 strftime('%s', ...) returns seconds
|
|
|
|
//This field stores the timestamp as raw milliseconds
|
|
|
|
BkLastModified sqlid `db:"bkLastModified"`
|
|
|
|
|
2022-11-21 23:05:24 +00:00
|
|
|
//NOTE: parsing into time.Time not working, I need to have a sqlite column of
|
|
|
|
//time Datetime [see](https://github.com/mattn/go-sqlite3/issues/748)!!
|
2022-11-16 23:06:02 +00:00
|
|
|
//Our query converts to the format scannable by go-sqlite3 SQLiteTimestampFormats
|
|
|
|
//This field stores the timestamp parsable as time.Time
|
|
|
|
// BkLastModifiedDateTime time.Time `db:"bkLastModifiedDateTime"`
|
|
|
|
|
|
|
|
IsFolder bool `db:"isFolder"`
|
|
|
|
IsTag bool `db:"isTag"`
|
|
|
|
IsBk bool `db:"isBk"`
|
|
|
|
BkParent sqlid `db:"bkParent"`
|
|
|
|
}
|
|
|
|
|
2022-11-24 20:25:05 +00:00
|
|
|
func (pb *MergedPlaceBookmark) datetime() time.Time {
|
2022-11-21 23:05:24 +00:00
|
|
|
return time.Unix(int64(pb.BkLastModified/(1000*1000)),
|
|
|
|
int64(pb.BkLastModified%(1000*1000))*1000).UTC()
|
|
|
|
}
|
|
|
|
|
2022-12-04 20:01:53 +00:00
|
|
|
// WIP
|
|
|
|
// load bookmarks from places.sqlite
|
|
|
|
// returns a []*MergedPlaceBookmark
|
2022-12-05 02:59:22 +00:00
|
|
|
func scanBookmarks(db *sqlx.DB) ([]*MozBookmark, error) {
|
|
|
|
var bookmarks []*MozBookmark
|
|
|
|
|
|
|
|
dotx, err := database.DotxQuery(MozBookmarkQueryFile)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = dotx.Select(db, &bookmarks, MozBookmarkQuery)
|
2022-11-21 23:05:24 +00:00
|
|
|
|
2022-12-05 02:59:22 +00:00
|
|
|
return bookmarks, err
|
2022-11-21 23:05:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
//WIP
|
2022-11-16 23:06:02 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
type Firefox struct {
|
|
|
|
*FirefoxConfig
|
|
|
|
|
|
|
|
// sqlite con to places.sqlite
|
|
|
|
places *database.DB
|
|
|
|
|
|
|
|
// All elements stored in URLIndex
|
|
|
|
URLIndexList []string
|
|
|
|
|
|
|
|
// Map from place tag IDs to the parse node tree
|
|
|
|
tagMap map[sqlid]*tree.Node
|
|
|
|
|
|
|
|
lastRunTime time.Time
|
|
|
|
}
|
|
|
|
|
|
|
|
func init() {
|
2022-11-07 19:07:13 +00:00
|
|
|
browsers.RegisterBrowser(Firefox{FirefoxConfig: FFConfig})
|
2022-10-27 22:33:20 +00:00
|
|
|
//TIP: cmd.RegisterModCommand(BrowserName, &cli.Command{
|
2022-12-04 20:01:53 +00:00
|
|
|
// Name: "test",
|
|
|
|
// })
|
|
|
|
// cmd.RegisterModCommand(BrowserName, &cli.Command{
|
|
|
|
// Name: "test2",
|
|
|
|
// })
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
2022-11-07 21:34:01 +00:00
|
|
|
func NewFirefox() *Firefox {
|
|
|
|
return &Firefox{
|
|
|
|
FirefoxConfig: FFConfig,
|
|
|
|
places: &database.DB{},
|
|
|
|
URLIndexList: []string{},
|
|
|
|
tagMap: map[sqlid]*tree.Node{},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
func (f Firefox) ModInfo() browsers.ModInfo {
|
2022-10-27 22:33:20 +00:00
|
|
|
return browsers.ModInfo{
|
2022-11-07 19:07:13 +00:00
|
|
|
ID: browsers.ModID(f.Name),
|
|
|
|
//HACK: duplicate instance with init().RegisterBrowser ??
|
|
|
|
New: func() browsers.Module {
|
2022-11-07 21:34:01 +00:00
|
|
|
return NewFirefox()
|
2022-11-07 19:07:13 +00:00
|
|
|
},
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
2018-11-01 16:00:58 +00:00
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// TEST:
|
|
|
|
// Implements browser.Initializer interface
|
|
|
|
func (f *Firefox) Init() error {
|
|
|
|
log.Infof("initializing <%s>", f.Name)
|
|
|
|
bookmarkPath, err := f.BookmarkPath()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2022-10-07 20:17:21 +00:00
|
|
|
}
|
2018-11-01 16:00:58 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
log.Debugf("bookmark path is: %s", bookmarkPath)
|
|
|
|
|
|
|
|
err = f.initPlacesCopy()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Setup watcher
|
|
|
|
expandedBaseDir, err := filepath.EvalSymlinks(f.BkDir)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
w := &watch.Watch{
|
|
|
|
Path: expandedBaseDir,
|
|
|
|
EventTypes: []fsnotify.Op{fsnotify.Write},
|
|
|
|
EventNames: []string{filepath.Join(expandedBaseDir, "places.sqlite-wal")},
|
|
|
|
ResetWatch: false,
|
|
|
|
}
|
|
|
|
|
|
|
|
browsers.SetupWatchersWithReducer(f.BrowserConfig, browsers.ReducerChanLen, w)
|
|
|
|
|
|
|
|
/*
|
|
|
|
*Run reducer to avoid duplicate jobs when a batch of events is received
|
|
|
|
*/
|
|
|
|
// TODO!: make a new copy of places for every new event change
|
|
|
|
|
|
|
|
// Add a reducer to the watcher
|
|
|
|
go watch.ReduceEvents(MozMinJobInterval, f)
|
|
|
|
|
|
|
|
return nil
|
2019-03-01 17:30:50 +00:00
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
func (f *Firefox) Watcher() *watch.WatchDescriptor {
|
2022-11-07 19:07:13 +00:00
|
|
|
return f.BrowserConfig.Watcher()
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
func (f Firefox) Config() *browsers.BrowserConfig {
|
2022-11-07 19:07:13 +00:00
|
|
|
return f.BrowserConfig
|
2022-11-01 13:27:19 +00:00
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// Firefox custom logic for preloading the bookmarks when the browser module
|
|
|
|
// starts. Implements browsers.Loader interface.
|
|
|
|
func (f *Firefox) Load() error {
|
|
|
|
|
|
|
|
// Parse bookmarks to a flat tree (for compatibility with tree system)
|
|
|
|
start := time.Now()
|
2022-11-16 23:06:02 +00:00
|
|
|
loadBookmarks(f)
|
2022-10-27 22:33:20 +00:00
|
|
|
f.Stats.LastFullTreeParseTime = time.Since(start)
|
|
|
|
f.lastRunTime = time.Now().UTC()
|
|
|
|
|
|
|
|
log.Debugf("parsed %d bookmarks and %d nodes in %s",
|
|
|
|
f.Stats.CurrentUrlCount,
|
|
|
|
f.Stats.CurrentNodeCount,
|
|
|
|
f.Stats.LastFullTreeParseTime)
|
|
|
|
f.Stats.Reset()
|
2019-03-01 18:03:48 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// Sync the URLIndex to the buffer
|
|
|
|
// We do not use the NodeTree here as firefox tags are represented
|
|
|
|
// as a flat tree which is not efficient, we use the go hashmap instead
|
2019-03-01 18:03:48 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
database.SyncURLIndexToBuffer(f.URLIndexList, f.URLIndex, f.BufferDB)
|
|
|
|
|
|
|
|
// Handle empty cache
|
|
|
|
if empty, err := database.Cache.DB.IsEmpty(); empty {
|
2019-03-01 18:03:48 +00:00
|
|
|
if err != nil {
|
2022-10-27 22:33:20 +00:00
|
|
|
return err
|
2019-03-01 18:03:48 +00:00
|
|
|
}
|
2022-10-27 22:33:20 +00:00
|
|
|
log.Info("cache empty: loading buffer to Cachedb")
|
2019-03-01 17:30:50 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
f.BufferDB.CopyTo(database.Cache.DB)
|
|
|
|
|
|
|
|
log.Debugf("syncing <%s> to disk", database.Cache.DB.Name)
|
|
|
|
} else {
|
|
|
|
f.BufferDB.SyncTo(database.Cache.DB)
|
2019-03-01 17:30:50 +00:00
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
database.Cache.DB.SyncToDisk(database.GetDBFullPath())
|
|
|
|
|
|
|
|
//DEBUG:
|
|
|
|
tree.PrintTree(f.NodeTree)
|
|
|
|
|
|
|
|
// Close the copy places.sqlite
|
|
|
|
err := f.places.Close()
|
|
|
|
|
|
|
|
return err
|
2019-03-01 17:30:50 +00:00
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// Implement browsers.Runner interface
|
|
|
|
func (f *Firefox) Run() {
|
|
|
|
startRun := time.Now()
|
|
|
|
|
|
|
|
err := f.initPlacesCopy()
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Debugf("Checking changes since <%d> %s",
|
2022-11-01 13:27:19 +00:00
|
|
|
f.lastRunTime.UTC().UnixNano()/1000,
|
|
|
|
f.lastRunTime.Local().Format("Mon Jan 2 15:04:05 MST 2006"))
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
queryArgs := map[string]interface{}{
|
|
|
|
"not_root_tags": []int{ffBkRoot, ffBkTags},
|
2022-11-01 13:27:19 +00:00
|
|
|
"last_runtime_utc": f.lastRunTime.UTC().UnixNano() / 1000,
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
query, args, err := sqlx.Named(
|
|
|
|
QBookmarksChanged,
|
|
|
|
queryArgs,
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
query, args, err = sqlx.In(query, args...)
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
query = f.places.Handle.Rebind(query)
|
2022-10-27 22:33:20 +00:00
|
|
|
utils.PrettyPrint(query)
|
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
rows, err := f.places.Handle.Query(query, args...)
|
2022-10-27 22:33:20 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Found new results in places db since last time we had changes
|
|
|
|
// database.DebugPrintRows(rows) // WARN: This will disable reading rows
|
|
|
|
// TEST: implement this in a func and unit test it
|
|
|
|
// NOTE: this looks like a lot of code reuse in fetchUrlChanges()
|
|
|
|
for rows.Next() {
|
|
|
|
// next := rows.Next()
|
|
|
|
// log.Debug("next rows is: ", next)
|
|
|
|
// if !next {
|
|
|
|
// break
|
|
|
|
// }
|
|
|
|
changedURLS := make([]string, 0)
|
|
|
|
|
|
|
|
log.Debugf("Found changes since: %s",
|
2022-11-01 13:27:19 +00:00
|
|
|
f.lastRunTime.Local().Format("Mon Jan 2 15:04:05 MST 2006"))
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
// extract bookmarks to this map
|
|
|
|
bookmarks := make(map[sqlid]*FFBookmark)
|
|
|
|
|
|
|
|
// record new places to this map
|
|
|
|
places := make(map[sqlid]*FFPlace)
|
|
|
|
|
|
|
|
// Fetch all changes into bookmarks and places maps
|
2022-11-01 13:27:19 +00:00
|
|
|
f.fetchUrlChanges(rows, bookmarks, places)
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
// utils.PrettyPrint(places)
|
|
|
|
// For each url
|
|
|
|
for urlId, place := range places {
|
|
|
|
var urlNode *tree.Node
|
|
|
|
changedURLS = utils.Extends(changedURLS, place.URL)
|
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
ok, urlNode := f.addUrlNode(place.URL, place.Title.String, place.Description.String)
|
2022-10-27 22:33:20 +00:00
|
|
|
if !ok {
|
|
|
|
log.Infof("url <%s> already in url index", place.URL)
|
|
|
|
}
|
|
|
|
|
|
|
|
// First get any new bookmarks
|
|
|
|
for bkId, bk := range bookmarks {
|
|
|
|
|
|
|
|
// if bookmark type is folder or tag
|
|
|
|
if bk.btype == BkTypeTagFolder &&
|
|
|
|
|
|
|
|
// Ignore root directories
|
|
|
|
// NOTE: ffBkTags change time shows last time bookmark tags
|
|
|
|
// whre changed ?
|
|
|
|
bkId != ffBkTags {
|
|
|
|
|
|
|
|
log.Debugf("adding tag node %s", bk.title)
|
2022-11-01 13:27:19 +00:00
|
|
|
ok, tagNode := f.addTagNode(bkId, bk.title)
|
2022-10-27 22:33:20 +00:00
|
|
|
if !ok {
|
|
|
|
log.Infof("tag <%s> already in tag map", tagNode.Name)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// link tags(moz_bookmark) to urls (moz_places)
|
|
|
|
for _, bk := range bookmarks {
|
|
|
|
|
|
|
|
// This effectively applies the tag to the URL
|
|
|
|
// The tag link should have a parent over 6 and fk->urlId
|
|
|
|
log.Debugf("Bookmark parent %d", bk.parent)
|
|
|
|
if bk.fk == urlId &&
|
|
|
|
bk.parent > ffBkMobile {
|
|
|
|
|
|
|
|
// The tag node should have already been created
|
2022-11-01 13:27:19 +00:00
|
|
|
tagNode, tagNodeExists := f.tagMap[bk.parent]
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
if tagNodeExists && urlNode != nil {
|
|
|
|
log.Debugf("URL has tag %s", tagNode.Name)
|
|
|
|
|
|
|
|
urlNode.Tags = utils.Extends(urlNode.Tags, tagNode.Name)
|
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
tree.AddChild(f.tagMap[bk.parent], urlNode)
|
2022-10-27 22:33:20 +00:00
|
|
|
//TEST: remove after testing this code section
|
2022-11-01 13:27:19 +00:00
|
|
|
// urlNode.Parent = f.tagMap[bk.parent]
|
|
|
|
// tree.Insert(f.tagMap[bk.parent].Children, urlNode)
|
2022-10-27 22:33:20 +00:00
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
f.Stats.CurrentUrlCount++
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-09-09 13:07:29 +00:00
|
|
|
}
|
2022-10-27 22:33:20 +00:00
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
database.SyncURLIndexToBuffer(changedURLS, f.URLIndex, f.BufferDB)
|
|
|
|
f.BufferDB.SyncTo(database.Cache.DB)
|
2022-10-27 22:33:20 +00:00
|
|
|
database.Cache.DB.SyncToDisk(database.GetDBFullPath())
|
|
|
|
|
2019-03-01 17:30:50 +00:00
|
|
|
}
|
2022-10-27 22:33:20 +00:00
|
|
|
err = rows.Close()
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
f.Stats.LastWatchRunTime = time.Since(startRun)
|
2022-10-27 22:33:20 +00:00
|
|
|
// log.Debugf("execution time %s", time.Since(startRun))
|
2019-03-01 17:30:50 +00:00
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
// tree.PrintTree(f.NodeTree) // debugging
|
2022-10-27 22:33:20 +00:00
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
err = f.places.Close()
|
2022-10-27 22:33:20 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
f.lastRunTime = time.Now().UTC()
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Implement browsers.Shutdowner
|
|
|
|
func (f *Firefox) Shutdown() {
|
|
|
|
log.Debugf("shutting down ... ")
|
|
|
|
|
|
|
|
if f.places != nil {
|
|
|
|
|
|
|
|
err := f.places.Close()
|
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
|
|
|
}
|
2019-03-01 17:30:50 +00:00
|
|
|
}
|
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
func (f *Firefox) copyPlacesToTmp() error {
|
|
|
|
err := utils.CopyFilesToTmpFolder(path.Join(f.BkDir, f.BkFile+"*"))
|
2020-09-09 13:07:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
func (browser *Firefox) getPathToPlacesCopy() string {
|
2020-09-09 13:07:29 +00:00
|
|
|
return path.Join(utils.TMPDIR, browser.BkFile)
|
|
|
|
}
|
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// TEST:
|
|
|
|
// HACK: addUrl and addTag share a lot of code, find a way to reuse shared code
|
|
|
|
// and only pass extra details about tag/url along in some data structure
|
|
|
|
// PROBLEM: tag nodes use IDs and URL nodes use URL as hashes
|
2022-11-01 13:27:19 +00:00
|
|
|
func (f *Firefox) addUrlNode(url, title, desc string) (bool, *tree.Node) {
|
2022-10-07 20:17:21 +00:00
|
|
|
var urlNode *tree.Node
|
2022-11-01 13:27:19 +00:00
|
|
|
iUrlNode, exists := f.URLIndex.Get(url)
|
2022-10-07 20:17:21 +00:00
|
|
|
if !exists {
|
|
|
|
urlNode := &tree.Node{
|
|
|
|
Name: title,
|
2022-11-01 13:27:19 +00:00
|
|
|
Type: tree.URLNode,
|
2022-10-07 20:17:21 +00:00
|
|
|
URL: url,
|
|
|
|
Desc: desc,
|
|
|
|
}
|
|
|
|
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Debugf("inserting url %s in url index", url)
|
2022-11-01 13:27:19 +00:00
|
|
|
f.URLIndex.Insert(url, urlNode)
|
|
|
|
f.URLIndexList = append(f.URLIndexList, url)
|
2022-10-07 20:17:21 +00:00
|
|
|
|
|
|
|
return true, urlNode
|
|
|
|
} else {
|
|
|
|
urlNode = iUrlNode.(*tree.Node)
|
|
|
|
}
|
|
|
|
|
|
|
|
return false, urlNode
|
|
|
|
}
|
|
|
|
|
|
|
|
// adds a new tagNode if it is not existing in the tagMap
|
|
|
|
// returns true if tag added or false if already existing
|
|
|
|
// returns the created tagNode
|
2022-10-27 22:33:20 +00:00
|
|
|
func (browser *Firefox) addTagNode(tagId sqlid, tagName string) (bool, *tree.Node) {
|
2022-10-07 20:17:21 +00:00
|
|
|
// node, exists :=
|
|
|
|
node, exists := browser.tagMap[tagId]
|
|
|
|
if exists {
|
|
|
|
return false, node
|
|
|
|
}
|
|
|
|
|
|
|
|
tagNode := &tree.Node{
|
|
|
|
Name: tagName,
|
2022-11-01 13:27:19 +00:00
|
|
|
Type: tree.TagNode,
|
2022-10-07 20:17:21 +00:00
|
|
|
Parent: browser.NodeTree, // root node
|
|
|
|
}
|
|
|
|
|
|
|
|
tree.AddChild(browser.NodeTree, tagNode)
|
|
|
|
browser.tagMap[tagId] = tagNode
|
|
|
|
browser.Stats.CurrentNodeCount++
|
|
|
|
|
|
|
|
return true, tagNode
|
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// Copies places.sqlite to a tmp dir to read a VFS lock sqlite db
|
|
|
|
func (f *Firefox) initPlacesCopy() error {
|
|
|
|
err := f.copyPlacesToTmp()
|
2020-09-09 13:07:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Could not copy places.sqlite to tmp folder: %s",
|
|
|
|
err)
|
|
|
|
}
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
opts := FFConfig.PlacesDSN
|
2020-09-09 13:07:29 +00:00
|
|
|
|
2022-11-07 19:07:13 +00:00
|
|
|
f.places, err = database.NewDB("places",
|
2020-09-09 13:07:29 +00:00
|
|
|
// using the copied places file instead of the original to avoid
|
|
|
|
// sqlite vfs lock errors
|
2022-10-27 22:33:20 +00:00
|
|
|
f.getPathToPlacesCopy(),
|
2020-09-09 13:07:29 +00:00
|
|
|
database.DBTypeFileDSN, opts).Init()
|
2022-09-28 20:13:03 +00:00
|
|
|
|
2020-09-09 13:07:29 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// load all bookmarks from `places.sqlite` and store them in BaseBrowser.NodeTree
|
|
|
|
// this method is used the first time gomark is started or to extract bookmarks
|
|
|
|
// using a command
|
2022-11-16 23:06:02 +00:00
|
|
|
func loadBookmarks(f *Firefox) {
|
2022-11-01 13:27:19 +00:00
|
|
|
log.Debugf("root tree children len is %d", len(f.NodeTree.Children))
|
2018-10-24 16:16:42 +00:00
|
|
|
//QGetTags := "SELECT id,title from moz_bookmarks WHERE parent = %d"
|
2018-11-20 17:33:37 +00:00
|
|
|
//
|
2018-10-23 20:14:54 +00:00
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
rows, err := f.places.Handle.Query(QgetBookmarks, ffBkTags)
|
2020-09-01 12:43:01 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
2018-11-30 03:15:08 +00:00
|
|
|
|
|
|
|
// Locked database is critical
|
|
|
|
if e, ok := err.(sqlite3.Error); ok {
|
|
|
|
if e.Code == sqlite3.ErrBusy {
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Critical(err)
|
2022-11-01 13:27:19 +00:00
|
|
|
f.Shutdown()
|
2018-11-30 03:15:08 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
2022-11-01 13:27:19 +00:00
|
|
|
log.Errorf("%s: %s", f.places.Name, err)
|
2018-11-20 17:33:37 +00:00
|
|
|
return
|
2018-10-28 19:19:12 +00:00
|
|
|
}
|
2018-10-23 20:14:54 +00:00
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// Rebuilding node tree
|
|
|
|
// Note: the node tree is built only for compatilibity with tree based
|
|
|
|
// bookmark parsing and might later be useful for debug/UI features.
|
|
|
|
// For efficiency reading after the initial Load() from
|
2018-11-13 18:54:20 +00:00
|
|
|
// places.sqlite should be done using a loop instad of tree traversal.
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-10-26 01:04:26 +00:00
|
|
|
/*
|
|
|
|
*This pass is used only for fetching bookmarks from firefox.
|
|
|
|
*Checking against the URLIndex should not be done here
|
|
|
|
*/
|
2018-10-23 20:14:54 +00:00
|
|
|
for rows.Next() {
|
2018-10-24 16:16:42 +00:00
|
|
|
var url, title, tagTitle, desc string
|
2018-11-23 03:40:10 +00:00
|
|
|
var tagId sqlid
|
2022-10-07 20:17:21 +00:00
|
|
|
|
2018-10-24 16:16:42 +00:00
|
|
|
err = rows.Scan(&url, &title, &desc, &tagId, &tagTitle)
|
2022-10-23 13:08:06 +00:00
|
|
|
// log.Debugf("%s|%s|%s|%d|%s", url, title, desc, tagId, tagTitle)
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Error(err)
|
2018-10-28 19:19:12 +00:00
|
|
|
}
|
2018-10-24 16:16:42 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is the first time we see this tag
|
|
|
|
* add it to the tagMap and create its node
|
|
|
|
*/
|
2022-11-01 13:27:19 +00:00
|
|
|
ok, tagNode := f.addTagNode(tagId, tagTitle)
|
2022-10-07 20:17:21 +00:00
|
|
|
if !ok {
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Infof("tag <%s> already in tag map", tagNode.Name)
|
2018-10-24 16:16:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add the url to the tag
|
2022-11-16 23:06:02 +00:00
|
|
|
// NOTE: this call is responsible for updating URLIndexList
|
2022-11-01 13:27:19 +00:00
|
|
|
ok, urlNode := f.addUrlNode(url, title, desc)
|
2022-10-07 20:17:21 +00:00
|
|
|
if !ok {
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Infof("url <%s> already in url index", url)
|
2018-10-26 01:04:26 +00:00
|
|
|
}
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// Add tag name to urlnode tags
|
2018-10-26 01:04:26 +00:00
|
|
|
urlNode.Tags = append(urlNode.Tags, tagNode.Name)
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-10-26 01:04:26 +00:00
|
|
|
// Set tag as parent to urlnode
|
2022-11-01 13:27:19 +00:00
|
|
|
tree.AddChild(f.tagMap[tagId], urlNode)
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
f.Stats.CurrentUrlCount++
|
2018-10-23 20:14:54 +00:00
|
|
|
}
|
2022-10-07 20:17:21 +00:00
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
log.Debugf("root tree children len is %d", len(f.NodeTree.Children))
|
2018-11-13 16:11:16 +00:00
|
|
|
}
|
2018-10-26 01:04:26 +00:00
|
|
|
|
2022-10-07 20:17:21 +00:00
|
|
|
// fetchUrlChanges method
|
|
|
|
// scan rows from a firefox `places.sqlite` db and extract all bookmarks and
|
|
|
|
// places (moz_bookmarks, moz_places tables) that changed/are new since the browser.lastRunTime
|
|
|
|
// using the QBookmarksChanged query
|
2022-11-01 13:27:19 +00:00
|
|
|
func (f *Firefox) fetchUrlChanges(rows *sql.Rows,
|
2018-11-23 03:40:10 +00:00
|
|
|
bookmarks map[sqlid]*FFBookmark,
|
2022-09-28 20:13:03 +00:00
|
|
|
places map[sqlid]*FFPlace,
|
|
|
|
) {
|
2022-10-07 20:17:21 +00:00
|
|
|
bk := &FFBookmark{}
|
2018-11-13 16:11:16 +00:00
|
|
|
|
|
|
|
// Get the URL that changed
|
2022-09-28 20:13:03 +00:00
|
|
|
err := rows.Scan(&bk.id, &bk.btype, &bk.fk, &bk.parent, &bk.title)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// database.DebugPrintRow(rows)
|
2018-11-13 16:11:16 +00:00
|
|
|
|
|
|
|
// We found URL change, urls are specified by
|
|
|
|
// type == 1
|
|
|
|
// fk -> id of url in moz_places
|
|
|
|
// parent == tag id
|
|
|
|
//
|
|
|
|
// Each tag on a url generates 2 or 3 entries in moz_bookmarks
|
|
|
|
// 1. If not existing, a (type==2) entry for the tag itself
|
2022-09-28 20:13:03 +00:00
|
|
|
// 2. A (type==1) entry for the bookmakred url with (fk -> moz_places.id)
|
|
|
|
// 3. A (type==1) (fk-> moz_places.id) (parent == idOf(tag))
|
2018-11-13 16:11:16 +00:00
|
|
|
|
|
|
|
if bk.btype == BkTypeURL {
|
2018-11-23 03:40:10 +00:00
|
|
|
var place FFPlace
|
2020-09-13 23:57:39 +00:00
|
|
|
|
|
|
|
// Use unsafe db to ignore non existant columns in
|
|
|
|
// dest field
|
2022-11-01 13:27:19 +00:00
|
|
|
udb := f.places.Handle.Unsafe()
|
2020-09-13 23:57:39 +00:00
|
|
|
err := udb.QueryRowx(QGetBookmarkPlace, bk.fk).StructScan(&place)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Debugf("Changed URL: %s", place.URL)
|
|
|
|
log.Debugf("%v", place)
|
2018-11-13 16:11:16 +00:00
|
|
|
|
|
|
|
// put url in the places map
|
2018-11-23 03:40:10 +00:00
|
|
|
places[place.ID] = &place
|
2018-11-13 16:11:16 +00:00
|
|
|
}
|
2018-10-26 01:04:26 +00:00
|
|
|
|
2018-11-13 16:11:16 +00:00
|
|
|
// This is the tag link
|
|
|
|
if bk.btype == BkTypeURL &&
|
2022-10-23 13:08:06 +00:00
|
|
|
// ignore original tags/folder from mozilla
|
2018-11-13 16:11:16 +00:00
|
|
|
bk.parent > ffBkMobile {
|
|
|
|
|
|
|
|
bookmarks[bk.id] = bk
|
|
|
|
}
|
2018-10-26 01:04:26 +00:00
|
|
|
|
2018-11-13 16:11:16 +00:00
|
|
|
// Tags are specified by:
|
|
|
|
// type == 2
|
|
|
|
// parent == (Id of root )
|
|
|
|
|
|
|
|
if bk.btype == BkTypeTagFolder {
|
|
|
|
bookmarks[bk.id] = bk
|
|
|
|
}
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-11-13 16:11:16 +00:00
|
|
|
for rows.Next() {
|
2022-11-01 13:27:19 +00:00
|
|
|
f.fetchUrlChanges(rows, bookmarks, places)
|
2018-11-13 16:11:16 +00:00
|
|
|
}
|
2022-10-23 13:08:06 +00:00
|
|
|
log.Debugf("fetching changes done !")
|
2018-10-23 20:14:54 +00:00
|
|
|
}
|