2018-05-27 16:02:50 +00:00
|
|
|
// ### Browsers API
|
2018-05-27 16:07:41 +00:00
|
|
|
//
|
|
|
|
// All constants and common API for browsers should be implemented here.
|
|
|
|
//
|
|
|
|
// For *browser specific* implementation create a new file for that browser.
|
2018-05-27 16:09:53 +00:00
|
|
|
//
|
2018-05-27 16:10:49 +00:00
|
|
|
// You must then implement a `func New[BrowserType]() IBrowser` function and
|
2018-05-27 16:07:41 +00:00
|
|
|
// implement parsing.
|
2017-11-19 16:02:53 +00:00
|
|
|
package main
|
|
|
|
|
2017-11-19 20:28:24 +00:00
|
|
|
import (
|
2017-11-20 15:05:44 +00:00
|
|
|
"fmt"
|
|
|
|
"path"
|
2017-11-19 20:28:24 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
"github.com/fsnotify/fsnotify"
|
2017-11-30 15:08:12 +00:00
|
|
|
"github.com/sp4ke/hashmap"
|
2017-11-19 20:28:24 +00:00
|
|
|
)
|
|
|
|
|
2017-11-19 19:46:24 +00:00
|
|
|
type BrowserType uint8
|
2017-11-19 16:02:53 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
// Browser types
|
2017-11-19 16:02:53 +00:00
|
|
|
const (
|
2017-11-20 15:05:44 +00:00
|
|
|
TChrome BrowserType = iota
|
|
|
|
TFirefox
|
2017-11-19 16:02:53 +00:00
|
|
|
)
|
2017-11-19 20:28:24 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
// Chrome details
|
|
|
|
var Chrome = struct {
|
|
|
|
BookmarkFile string
|
|
|
|
BookmarkDir string
|
|
|
|
}{
|
|
|
|
"Bookmarks",
|
2017-11-23 01:51:23 +00:00
|
|
|
"/home/spike/.config/google-chrome-unstable/Default/",
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type IBrowser interface {
|
2017-11-20 18:07:15 +00:00
|
|
|
IWatchable
|
2017-11-20 15:05:44 +00:00
|
|
|
InitBuffer() // init buffer db, should be defered to close after call
|
2018-05-26 13:34:55 +00:00
|
|
|
InitIndex() // Creates in memory Index (RB-Tree)
|
2017-11-20 18:07:15 +00:00
|
|
|
RegisterHooks(...ParseHook)
|
|
|
|
Load() // Loads bookmarks to db without watching
|
2017-11-19 20:28:24 +00:00
|
|
|
//Parse(...ParseHook) // Main parsing method with different parsing hooks
|
|
|
|
Close() // Gracefully finish work and stop watching
|
|
|
|
}
|
|
|
|
|
2017-11-19 20:42:20 +00:00
|
|
|
// Base browser class serves as reference for implmented browser types
|
|
|
|
// Browser should contain enough data internally to not rely on any global
|
|
|
|
// variable or constant if possible.
|
2018-05-26 13:34:55 +00:00
|
|
|
// To create new browsers, you must implement a New<BrowserType>() IBrowser function
|
2018-05-27 15:36:03 +00:00
|
|
|
//
|
2018-05-27 16:17:17 +00:00
|
|
|
// `URLIndex` (HashMap RBTree):
|
2018-05-26 13:34:55 +00:00
|
|
|
// Used as fast query db representing the last known browser bookmarks.
|
2018-05-27 15:36:03 +00:00
|
|
|
//
|
2018-05-27 16:17:17 +00:00
|
|
|
// `nodeTree` (Tree DAG):
|
2018-05-27 15:36:03 +00:00
|
|
|
// Used in each job to represent bookmarks in a tree
|
|
|
|
//
|
2018-05-27 16:21:27 +00:00
|
|
|
// `BufferDB`: sqlite buffer used across jobs
|
2017-11-19 20:28:24 +00:00
|
|
|
type BaseBrowser struct {
|
2018-05-27 16:21:27 +00:00
|
|
|
watcher *fsnotify.Watcher
|
|
|
|
baseDir string
|
|
|
|
bkFile string
|
|
|
|
|
|
|
|
// In memory sqlite db (named `memcache`).
|
|
|
|
// Used to keep a browser's state of bookmarks across jobs.
|
|
|
|
BufferDB *DB
|
|
|
|
|
|
|
|
// Fast query db using an RB-Tree hashmap.
|
2018-05-27 16:22:24 +00:00
|
|
|
// It represents a URL index of the last running job
|
2018-05-27 16:21:27 +00:00
|
|
|
URLIndex *hashmap.RBTree
|
2018-05-27 16:17:17 +00:00
|
|
|
|
|
|
|
// Pointer to the root of the node tree
|
|
|
|
// The node tree is built again for every Run job on a browser
|
|
|
|
NodeTree *Node
|
|
|
|
|
|
|
|
// Various parsing and timing stats
|
|
|
|
Stats *ParserStats
|
2017-11-20 15:05:44 +00:00
|
|
|
bType BrowserType
|
|
|
|
name string
|
|
|
|
isWatching bool
|
2017-11-20 18:07:15 +00:00
|
|
|
parseHooks []ParseHook
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (bw *BaseBrowser) Watcher() *fsnotify.Watcher {
|
|
|
|
return bw.watcher
|
|
|
|
}
|
|
|
|
|
|
|
|
func (bw *BaseBrowser) Load() {
|
|
|
|
log.Debug("BaseBrowser Load()")
|
2017-12-02 12:44:15 +00:00
|
|
|
bw.InitIndex()
|
|
|
|
|
|
|
|
// Check if cache is initialized
|
2018-05-27 15:55:27 +00:00
|
|
|
if CacheDB == nil || CacheDB.Handle == nil {
|
2017-12-02 12:44:15 +00:00
|
|
|
log.Critical("cache is not yet initialized !")
|
|
|
|
panic("cache is not yet initialized !")
|
|
|
|
}
|
|
|
|
|
|
|
|
if bw.watcher == nil {
|
|
|
|
log.Fatal("watcher not initialized, use SetupWatcher() when creating the browser !")
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Debug("preloading bookmarks")
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (bw *BaseBrowser) GetPath() string {
|
|
|
|
return path.Join(bw.baseDir, bw.bkFile)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (bw *BaseBrowser) GetDir() string {
|
|
|
|
return bw.baseDir
|
|
|
|
}
|
|
|
|
|
|
|
|
func (bw *BaseBrowser) SetupWatcher() {
|
|
|
|
var err error
|
|
|
|
bw.watcher, err = fsnotify.NewWatcher()
|
|
|
|
logPanic(err)
|
|
|
|
err = bw.watcher.Add(bw.baseDir)
|
|
|
|
logPanic(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (bw *BaseBrowser) Close() {
|
|
|
|
err := bw.watcher.Close()
|
2018-05-27 16:21:27 +00:00
|
|
|
bw.BufferDB.Close()
|
2017-11-20 15:05:44 +00:00
|
|
|
logPanic(err)
|
2017-11-19 20:28:24 +00:00
|
|
|
}
|
|
|
|
|
2017-11-30 15:08:12 +00:00
|
|
|
func (b *BaseBrowser) InitIndex() {
|
|
|
|
b.URLIndex = NewIndex()
|
|
|
|
}
|
|
|
|
|
2018-05-26 13:34:55 +00:00
|
|
|
func (b *BaseBrowser) RebuildIndex() {
|
|
|
|
log.Debugf("Rebuilding index based on current nodeTree")
|
|
|
|
b.URLIndex = NewIndex()
|
2018-05-27 16:17:17 +00:00
|
|
|
WalkBuildIndex(b.NodeTree, b)
|
2018-05-26 13:34:55 +00:00
|
|
|
}
|
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
func (b *BaseBrowser) InitBuffer() {
|
2017-11-23 01:51:23 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
bufferName := fmt.Sprintf("buffer_%s", b.name)
|
|
|
|
bufferPath := fmt.Sprintf(DBBufferFmt, bufferName)
|
2017-11-23 01:51:23 +00:00
|
|
|
|
2018-05-27 16:21:27 +00:00
|
|
|
b.BufferDB = DB{}.New(bufferName, bufferPath)
|
|
|
|
b.BufferDB.Init()
|
2017-11-23 01:51:23 +00:00
|
|
|
|
2018-05-27 16:21:27 +00:00
|
|
|
b.BufferDB.Attach(CacheDB)
|
2017-11-19 20:28:24 +00:00
|
|
|
}
|
2017-11-20 18:07:15 +00:00
|
|
|
|
|
|
|
func (b *BaseBrowser) RegisterHooks(hooks ...ParseHook) {
|
|
|
|
log.Debug("Registering hooks")
|
|
|
|
for _, hook := range hooks {
|
|
|
|
b.parseHooks = append(b.parseHooks, hook)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Runs browsed defined hooks on bookmark
|
2017-11-30 15:08:12 +00:00
|
|
|
func (b *BaseBrowser) RunParseHooks(node *Node) {
|
2017-11-20 18:07:15 +00:00
|
|
|
for _, hook := range b.parseHooks {
|
2017-11-30 15:08:12 +00:00
|
|
|
hook(node)
|
2017-11-20 18:07:15 +00:00
|
|
|
}
|
|
|
|
}
|