2018-06-08 16:27:33 +00:00
|
|
|
package main
|
|
|
|
|
2018-06-14 00:30:18 +00:00
|
|
|
import (
|
2018-06-14 14:42:54 +00:00
|
|
|
"path"
|
2018-10-25 16:09:03 +00:00
|
|
|
"time"
|
2018-11-01 16:00:58 +00:00
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
"github.com/fsnotify/fsnotify"
|
2018-06-14 00:30:18 +00:00
|
|
|
)
|
|
|
|
|
2018-06-08 16:27:33 +00:00
|
|
|
var Firefox = BrowserPaths{
|
2018-11-02 17:21:18 +00:00
|
|
|
BookmarkFile: "places.sqlite",
|
|
|
|
BookmarkDir: "/home/spike/.mozilla/firefox/p1rrgord.default/",
|
2018-06-08 16:27:33 +00:00
|
|
|
}
|
|
|
|
|
2018-10-23 20:14:54 +00:00
|
|
|
const (
|
|
|
|
MozPlacesRootID = 1
|
|
|
|
MozPlacesTagsRootID = 4
|
|
|
|
MozPlacesMobileRootID = 6
|
2018-11-02 17:21:18 +00:00
|
|
|
MozMinJobInterval = 2 * time.Second
|
2018-10-23 20:14:54 +00:00
|
|
|
)
|
|
|
|
|
2018-06-08 16:27:33 +00:00
|
|
|
type FFBrowser struct {
|
|
|
|
BaseBrowser //embedding
|
2018-10-28 19:19:12 +00:00
|
|
|
places *DB
|
2018-11-01 16:00:58 +00:00
|
|
|
urlMap map[string]*Node // Used instead of node tree for syncing to buffer
|
2018-10-23 20:14:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type FFTag struct {
|
|
|
|
id int
|
|
|
|
title string
|
2018-06-08 16:27:33 +00:00
|
|
|
}
|
|
|
|
|
2018-11-01 16:00:58 +00:00
|
|
|
func FFPlacesUpdateHook(op int, db string, table string, rowid int64) {
|
|
|
|
log.Debug(op)
|
|
|
|
}
|
|
|
|
|
2018-06-08 16:27:33 +00:00
|
|
|
func NewFFBrowser() IBrowser {
|
|
|
|
browser := &FFBrowser{}
|
|
|
|
browser.name = "firefox"
|
|
|
|
browser.bType = TFirefox
|
|
|
|
browser.baseDir = Firefox.BookmarkDir
|
|
|
|
browser.bkFile = Firefox.BookmarkFile
|
2018-11-02 17:21:18 +00:00
|
|
|
browser.useFileWatcher = true
|
2018-06-08 16:27:33 +00:00
|
|
|
browser.Stats = &ParserStats{}
|
2018-10-25 16:09:03 +00:00
|
|
|
browser.NodeTree = &Node{Name: "root", Parent: nil, Type: "root"}
|
2018-11-01 16:00:58 +00:00
|
|
|
browser.urlMap = make(map[string]*Node)
|
|
|
|
|
2018-10-28 19:19:12 +00:00
|
|
|
// Initialize `places.sqlite`
|
|
|
|
bookmarkPath := path.Join(browser.baseDir, browser.bkFile)
|
|
|
|
browser.places = DB{}.New("Places", bookmarkPath)
|
|
|
|
browser.places.InitRO()
|
|
|
|
|
2018-11-01 16:00:58 +00:00
|
|
|
// Buffer that lives accross Run() jobs
|
2018-06-08 16:27:33 +00:00
|
|
|
browser.InitBuffer()
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
// Setup watcher
|
|
|
|
|
|
|
|
w := &Watch{
|
|
|
|
path: path.Join(browser.baseDir),
|
|
|
|
eventTypes: []fsnotify.Op{fsnotify.Write},
|
|
|
|
eventNames: []string{path.Join(browser.baseDir, "places.sqlite-wal")},
|
|
|
|
resetWatch: false,
|
|
|
|
}
|
|
|
|
|
|
|
|
browser.SetupFileWatcher(w)
|
|
|
|
|
2018-10-23 20:14:54 +00:00
|
|
|
/*
|
2018-11-02 17:21:18 +00:00
|
|
|
*Run reducer to avoid duplicate running of jobs
|
2018-10-23 20:14:54 +00:00
|
|
|
*when a batch of events is received
|
|
|
|
*/
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
browser.eventsChan = make(chan fsnotify.Event, EventsChanLen)
|
|
|
|
|
|
|
|
go reducer(MozMinJobInterval, browser.eventsChan, browser)
|
2018-06-14 00:30:18 +00:00
|
|
|
|
2018-06-08 16:27:33 +00:00
|
|
|
return browser
|
|
|
|
}
|
|
|
|
|
2018-10-28 19:19:12 +00:00
|
|
|
func (bw *FFBrowser) Shutdown() {
|
|
|
|
|
|
|
|
log.Debugf("<%s> shutting down ... ", bw.name)
|
|
|
|
|
|
|
|
err := bw.BaseBrowser.Close()
|
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = bw.places.Close()
|
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-08 16:27:33 +00:00
|
|
|
func (bw *FFBrowser) Watch() bool {
|
|
|
|
|
2018-10-28 19:19:12 +00:00
|
|
|
log.Debugf("<%s> TODO ... ", bw.name)
|
|
|
|
|
2018-11-01 16:00:58 +00:00
|
|
|
if !bw.isWatching {
|
2018-11-02 17:21:18 +00:00
|
|
|
go WatcherThread(bw)
|
2018-11-01 16:00:58 +00:00
|
|
|
bw.isWatching = true
|
2018-11-02 17:21:18 +00:00
|
|
|
log.Infof("<%s> Watching %s", bw.name, bw.GetPath())
|
2018-11-01 16:00:58 +00:00
|
|
|
return true
|
|
|
|
}
|
2018-06-08 16:27:33 +00:00
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
func (bw *FFBrowser) Load() {
|
|
|
|
bw.BaseBrowser.Load()
|
2018-11-01 16:00:58 +00:00
|
|
|
|
|
|
|
// Parse bookmarks to a flat tree (for compatibility with tree system)
|
|
|
|
start := time.Now()
|
|
|
|
getFFBookmarks(bw)
|
|
|
|
bw.Stats.lastParseTime = time.Since(start)
|
|
|
|
|
|
|
|
// Finished parsing
|
|
|
|
//go PrintTree(bw.NodeTree) // debugging
|
|
|
|
log.Debugf("<%s> parsed %d bookmarks and %d nodes in %s", bw.name,
|
|
|
|
bw.Stats.currentUrlCount, bw.Stats.currentNodeCount, bw.Stats.lastParseTime)
|
|
|
|
|
|
|
|
bw.ResetStats()
|
|
|
|
|
|
|
|
// Sync the urlMap to the buffer
|
|
|
|
// We do not use the NodeTree here as firefox tags are represented
|
|
|
|
// as a flat tree which is not efficient, we use the go hashmap instead
|
|
|
|
// The map contains map[url]*Node elements with urls already containing the
|
|
|
|
// right tags.
|
|
|
|
|
|
|
|
syncURLMapToBuffer(bw.urlMap, bw.BufferDB)
|
|
|
|
|
|
|
|
bw.BufferDB.SyncTo(CacheDB)
|
2018-06-08 16:27:33 +00:00
|
|
|
}
|
|
|
|
|
2018-10-24 16:16:42 +00:00
|
|
|
func getFFBookmarks(bw *FFBrowser) {
|
|
|
|
|
|
|
|
QGetBookmarks := `WITH bookmarks AS
|
|
|
|
|
|
|
|
(SELECT moz_places.url AS url,
|
|
|
|
moz_places.description as desc,
|
|
|
|
moz_places.title as urlTitle,
|
|
|
|
moz_bookmarks.parent AS tagId
|
|
|
|
FROM moz_places LEFT OUTER JOIN moz_bookmarks
|
|
|
|
ON moz_places.id = moz_bookmarks.fk
|
|
|
|
WHERE moz_bookmarks.parent
|
|
|
|
IN (SELECT id FROM moz_bookmarks WHERE parent = ? ))
|
|
|
|
|
|
|
|
SELECT url, IFNULL(urlTitle, ''), IFNULL(desc,''),
|
|
|
|
tagId, moz_bookmarks.title AS tagTitle
|
|
|
|
|
|
|
|
FROM bookmarks LEFT OUTER JOIN moz_bookmarks
|
|
|
|
ON tagId = moz_bookmarks.id
|
|
|
|
ORDER BY url`
|
|
|
|
|
|
|
|
//QGetTags := "SELECT id,title from moz_bookmarks WHERE parent = %d"
|
2018-10-23 20:14:54 +00:00
|
|
|
|
2018-11-01 16:00:58 +00:00
|
|
|
rows, err := bw.places.handle.Query(QGetBookmarks, MozPlacesTagsRootID)
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
2018-10-23 20:14:54 +00:00
|
|
|
|
2018-10-24 16:16:42 +00:00
|
|
|
tagMap := make(map[int]*Node)
|
|
|
|
|
|
|
|
// Rebuild node tree
|
2018-11-01 16:00:58 +00:00
|
|
|
// Note: the node tree is build only for compatilibity
|
|
|
|
// pruposes with tree based bookmark parsing.
|
|
|
|
// For efficiency reading after the initial parse from places.sqlite,
|
|
|
|
// reading should be done in loops in instead of tree parsing.
|
2018-10-24 16:16:42 +00:00
|
|
|
rootNode := bw.NodeTree
|
|
|
|
|
2018-10-26 01:04:26 +00:00
|
|
|
/*
|
|
|
|
*This pass is used only for fetching bookmarks from firefox.
|
|
|
|
*Checking against the URLIndex should not be done here
|
|
|
|
*/
|
2018-10-23 20:14:54 +00:00
|
|
|
for rows.Next() {
|
2018-10-24 16:16:42 +00:00
|
|
|
var url, title, tagTitle, desc string
|
|
|
|
var tagId int
|
|
|
|
err = rows.Scan(&url, &title, &desc, &tagId, &tagTitle)
|
2018-10-26 01:04:26 +00:00
|
|
|
//log.Debugf("%s|%s|%s|%d|%s", url, title, desc, tagId, tagTitle)
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
2018-10-24 16:16:42 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is the first time we see this tag
|
|
|
|
* add it to the tagMap and create its node
|
|
|
|
*/
|
|
|
|
tagNode, tagNodeExists := tagMap[tagId]
|
|
|
|
if !tagNodeExists {
|
|
|
|
// Add the tag as a node
|
|
|
|
tagNode = new(Node)
|
|
|
|
tagNode.Type = "tag"
|
|
|
|
tagNode.Name = tagTitle
|
|
|
|
tagNode.Parent = rootNode
|
|
|
|
rootNode.Children = append(rootNode.Children, tagNode)
|
|
|
|
tagMap[tagId] = tagNode
|
|
|
|
bw.Stats.currentNodeCount++
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add the url to the tag
|
2018-11-01 16:00:58 +00:00
|
|
|
urlNode, urlNodeExists := bw.urlMap[url]
|
2018-10-26 01:04:26 +00:00
|
|
|
if !urlNodeExists {
|
|
|
|
urlNode = new(Node)
|
|
|
|
urlNode.Type = "url"
|
|
|
|
urlNode.URL = url
|
|
|
|
urlNode.Name = title
|
|
|
|
urlNode.Desc = desc
|
2018-11-01 16:00:58 +00:00
|
|
|
bw.urlMap[url] = urlNode
|
2018-10-26 01:04:26 +00:00
|
|
|
}
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-10-26 01:04:26 +00:00
|
|
|
// Add tag to urlnode tags
|
|
|
|
urlNode.Tags = append(urlNode.Tags, tagNode.Name)
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-10-26 01:04:26 +00:00
|
|
|
// Set tag as parent to urlnode
|
|
|
|
urlNode.Parent = tagMap[tagId]
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-10-26 01:04:26 +00:00
|
|
|
// Add urlnode as child to tag node
|
|
|
|
tagMap[tagId].Children = append(tagMap[tagId].Children, urlNode)
|
2018-10-24 16:16:42 +00:00
|
|
|
|
|
|
|
bw.Stats.currentUrlCount++
|
|
|
|
bw.Stats.currentNodeCount++
|
2018-10-23 20:14:54 +00:00
|
|
|
}
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-10-26 01:04:26 +00:00
|
|
|
/*
|
|
|
|
*Build tags for each url then check against URLIndex
|
|
|
|
*for changes
|
|
|
|
*/
|
|
|
|
|
|
|
|
// Check if url already in index TODO: should be done in new pass
|
|
|
|
//iVal, found := bw.URLIndex.Get(urlNode.URL)
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The fields where tags may change are hashed together
|
|
|
|
* to detect changes in futre parses
|
|
|
|
* To handle tag changes we need to get all parent nodes
|
|
|
|
* (tags) for this url then hash their concatenation
|
|
|
|
*/
|
|
|
|
|
|
|
|
//nameHash := xxhash.ChecksumString64(urlNode.Name)
|
2018-10-24 16:16:42 +00:00
|
|
|
|
2018-10-23 20:14:54 +00:00
|
|
|
}
|
|
|
|
|
2018-06-08 16:27:33 +00:00
|
|
|
func (bw *FFBrowser) Run() {
|
2018-11-02 17:21:18 +00:00
|
|
|
log.Debugf("<%s>", bw.name)
|
2018-06-08 16:27:33 +00:00
|
|
|
|
|
|
|
}
|