2018-05-27 16:02:50 +00:00
|
|
|
// ### Browsers API
|
2018-05-27 16:07:41 +00:00
|
|
|
//
|
|
|
|
// All constants and common API for browsers should be implemented here.
|
|
|
|
//
|
|
|
|
// For *browser specific* implementation create a new file for that browser.
|
2018-05-27 16:09:53 +00:00
|
|
|
//
|
2018-05-27 16:10:49 +00:00
|
|
|
// You must then implement a `func New[BrowserType]() IBrowser` function and
|
2018-05-27 16:07:41 +00:00
|
|
|
// implement parsing.
|
2017-11-19 16:02:53 +00:00
|
|
|
package main
|
|
|
|
|
2017-11-19 20:28:24 +00:00
|
|
|
import (
|
2017-11-20 15:05:44 +00:00
|
|
|
"fmt"
|
2018-11-09 17:25:50 +00:00
|
|
|
"gomark/database"
|
|
|
|
"gomark/index"
|
|
|
|
"gomark/parsing"
|
|
|
|
"gomark/tree"
|
|
|
|
"gomark/watch"
|
2018-11-30 19:04:54 +00:00
|
|
|
"io"
|
2017-11-20 15:05:44 +00:00
|
|
|
"path"
|
2018-11-20 17:33:37 +00:00
|
|
|
"path/filepath"
|
2018-11-01 16:00:58 +00:00
|
|
|
"reflect"
|
2017-11-19 20:28:24 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
"github.com/fsnotify/fsnotify"
|
2017-11-30 15:08:12 +00:00
|
|
|
"github.com/sp4ke/hashmap"
|
2017-11-19 20:28:24 +00:00
|
|
|
)
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
type IWatchable = watch.Watchable
|
2018-11-09 17:25:50 +00:00
|
|
|
type Watcher = watch.Watcher
|
|
|
|
type Watch = watch.Watch
|
2018-11-13 16:11:16 +00:00
|
|
|
|
2017-11-19 19:46:24 +00:00
|
|
|
type BrowserType uint8
|
2017-11-19 16:02:53 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
// Browser types
|
2017-11-19 16:02:53 +00:00
|
|
|
const (
|
2017-11-20 15:05:44 +00:00
|
|
|
TChrome BrowserType = iota
|
|
|
|
TFirefox
|
2017-11-19 16:02:53 +00:00
|
|
|
)
|
2017-11-19 20:28:24 +00:00
|
|
|
|
2018-06-14 00:30:18 +00:00
|
|
|
// Channel parameters
|
|
|
|
const EventsChanLen = 1000
|
|
|
|
|
2018-06-08 15:38:57 +00:00
|
|
|
// Used to store bookmark paths and other
|
|
|
|
// data related to a particular browser kind
|
|
|
|
type BrowserPaths struct {
|
2017-11-20 15:05:44 +00:00
|
|
|
BookmarkFile string
|
|
|
|
BookmarkDir string
|
|
|
|
}
|
|
|
|
|
|
|
|
type IBrowser interface {
|
2017-11-20 18:07:15 +00:00
|
|
|
IWatchable
|
2018-11-30 03:15:08 +00:00
|
|
|
InitBuffer() error // init buffer db, TODO: defer closings and shutdown
|
|
|
|
InitIndex() // Creates in memory Index (RB-Tree)
|
2018-11-09 17:25:50 +00:00
|
|
|
RegisterHooks(...parsing.Hook)
|
2018-11-30 19:04:54 +00:00
|
|
|
Load() // Loads bookmarks to db without watching
|
2018-10-28 19:19:12 +00:00
|
|
|
Shutdown() // Graceful shutdown, it should call the BaseBrowser.Close()
|
2017-11-19 20:28:24 +00:00
|
|
|
}
|
|
|
|
|
2017-11-19 20:42:20 +00:00
|
|
|
// Base browser class serves as reference for implmented browser types
|
|
|
|
// Browser should contain enough data internally to not rely on any global
|
|
|
|
// variable or constant if possible.
|
2018-05-26 13:34:55 +00:00
|
|
|
// To create new browsers, you must implement a New<BrowserType>() IBrowser function
|
2018-05-27 15:36:03 +00:00
|
|
|
//
|
2018-05-27 16:17:17 +00:00
|
|
|
// `URLIndex` (HashMap RBTree):
|
2018-05-26 13:34:55 +00:00
|
|
|
// Used as fast query db representing the last known browser bookmarks.
|
2018-05-27 15:36:03 +00:00
|
|
|
//
|
2018-05-27 16:17:17 +00:00
|
|
|
// `nodeTree` (Tree DAG):
|
2018-05-27 15:36:03 +00:00
|
|
|
// Used in each job to represent bookmarks in a tree
|
|
|
|
//
|
2018-05-27 16:21:27 +00:00
|
|
|
// `BufferDB`: sqlite buffer used across jobs
|
2017-11-19 20:28:24 +00:00
|
|
|
type BaseBrowser struct {
|
2018-11-02 17:21:18 +00:00
|
|
|
watcher *Watcher
|
2018-06-14 00:30:18 +00:00
|
|
|
eventsChan chan fsnotify.Event
|
|
|
|
baseDir string
|
|
|
|
bkFile string
|
2018-05-27 16:21:27 +00:00
|
|
|
|
|
|
|
// In memory sqlite db (named `memcache`).
|
|
|
|
// Used to keep a browser's state of bookmarks across jobs.
|
|
|
|
BufferDB *DB
|
|
|
|
|
|
|
|
// Fast query db using an RB-Tree hashmap.
|
2018-05-27 16:22:24 +00:00
|
|
|
// It represents a URL index of the last running job
|
2018-05-27 16:21:27 +00:00
|
|
|
URLIndex *hashmap.RBTree
|
2018-05-27 16:17:17 +00:00
|
|
|
|
|
|
|
// Pointer to the root of the node tree
|
|
|
|
// The node tree is built again for every Run job on a browser
|
2018-11-09 17:25:50 +00:00
|
|
|
NodeTree *tree.Node
|
2018-05-27 16:17:17 +00:00
|
|
|
// Various parsing and timing stats
|
2018-11-09 17:25:50 +00:00
|
|
|
Stats *parsing.Stats
|
2018-11-01 16:00:58 +00:00
|
|
|
bType BrowserType
|
|
|
|
name string
|
|
|
|
isWatching bool
|
|
|
|
useFileWatcher bool
|
2018-11-09 17:25:50 +00:00
|
|
|
parseHooks []parsing.Hook
|
2018-11-30 19:04:54 +00:00
|
|
|
|
|
|
|
io.Closer // Close database connections
|
2018-11-01 16:00:58 +00:00
|
|
|
}
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
func (bw *BaseBrowser) GetWatcher() *Watcher {
|
|
|
|
watcherType := reflect.TypeOf((*Watcher)(nil)).Elem()
|
|
|
|
// In case we use other types of watchers/events
|
|
|
|
if reflect.TypeOf(bw.watcher) == reflect.PtrTo(watcherType) {
|
2018-11-01 16:00:58 +00:00
|
|
|
return bw.watcher
|
|
|
|
}
|
|
|
|
return nil
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
func (bw *BaseBrowser) Load() {
|
|
|
|
log.Debug("BaseBrowser Load()")
|
2019-02-22 18:50:26 +00:00
|
|
|
|
2017-12-02 12:44:15 +00:00
|
|
|
bw.InitIndex()
|
|
|
|
|
|
|
|
// Check if cache is initialized
|
2018-11-09 17:25:50 +00:00
|
|
|
if CacheDB == nil || CacheDB.Handle == nil {
|
2018-10-28 19:19:12 +00:00
|
|
|
log.Criticalf("<%s> Loading bookmarks while cache not yet initialized !", bw.name)
|
2017-12-02 12:44:15 +00:00
|
|
|
}
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
// In case we use other types of watchers/events
|
2018-11-01 16:00:58 +00:00
|
|
|
if bw.useFileWatcher && bw.watcher == nil {
|
2018-11-02 17:21:18 +00:00
|
|
|
log.Warningf("<%s> watcher not initialized, use SetupFileWatcher() when creating the browser !", bw.name)
|
2017-12-02 12:44:15 +00:00
|
|
|
}
|
|
|
|
|
2018-10-23 20:14:54 +00:00
|
|
|
log.Debugf("<%s> preloading bookmarks", bw.name)
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
|
2018-11-30 03:15:08 +00:00
|
|
|
func (bw *BaseBrowser) GetBookmarksPath() string {
|
2018-11-20 17:33:37 +00:00
|
|
|
path, err := filepath.EvalSymlinks(path.Join(bw.baseDir, bw.bkFile))
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
return path
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
|
2018-06-14 00:30:18 +00:00
|
|
|
func (bw *BaseBrowser) EventsChan() chan fsnotify.Event {
|
|
|
|
return bw.eventsChan
|
|
|
|
}
|
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
func (bw *BaseBrowser) GetDir() string {
|
|
|
|
return bw.baseDir
|
|
|
|
}
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
// Setup file watcher using the provided []Watch elements
|
|
|
|
func (bw *BaseBrowser) SetupFileWatcher(watches ...*Watch) {
|
|
|
|
var err error
|
|
|
|
|
2018-11-01 16:00:58 +00:00
|
|
|
if !bw.useFileWatcher {
|
|
|
|
return
|
|
|
|
}
|
2018-11-02 17:21:18 +00:00
|
|
|
|
|
|
|
fswatcher, err := fsnotify.NewWatcher()
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
2018-11-02 17:21:18 +00:00
|
|
|
|
|
|
|
watchedMap := make(map[string]*Watch)
|
|
|
|
for _, v := range watches {
|
2018-11-09 17:25:50 +00:00
|
|
|
watchedMap[v.Path] = v
|
2018-11-02 17:21:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bw.watcher = &Watcher{
|
2018-11-09 17:25:50 +00:00
|
|
|
W: fswatcher,
|
|
|
|
Watched: watchedMap,
|
|
|
|
Watches: watches,
|
2018-11-02 17:21:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add all watched paths
|
|
|
|
for _, v := range watches {
|
|
|
|
|
2018-11-20 17:33:37 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
2018-11-09 17:25:50 +00:00
|
|
|
err = bw.watcher.W.Add(v.Path)
|
2018-11-02 17:21:18 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
2018-10-28 19:19:12 +00:00
|
|
|
}
|
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
|
2018-10-26 16:25:07 +00:00
|
|
|
func (bw *BaseBrowser) ResetWatcher() {
|
2018-11-09 17:25:50 +00:00
|
|
|
err := bw.watcher.W.Close()
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
2018-11-09 17:25:50 +00:00
|
|
|
bw.SetupFileWatcher(bw.watcher.Watches...)
|
2018-10-26 16:25:07 +00:00
|
|
|
}
|
|
|
|
|
2018-10-28 19:19:12 +00:00
|
|
|
func (bw *BaseBrowser) Close() error {
|
2018-11-09 17:25:50 +00:00
|
|
|
err := bw.watcher.W.Close()
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = bw.BufferDB.Close()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
2017-11-19 20:28:24 +00:00
|
|
|
}
|
|
|
|
|
2018-11-30 19:04:54 +00:00
|
|
|
func (bw *BaseBrowser) Shutdown() {
|
|
|
|
err := bw.Close()
|
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-11-30 15:08:12 +00:00
|
|
|
func (b *BaseBrowser) InitIndex() {
|
2018-11-09 17:25:50 +00:00
|
|
|
b.URLIndex = index.NewIndex()
|
2017-11-30 15:08:12 +00:00
|
|
|
}
|
|
|
|
|
2018-05-26 13:34:55 +00:00
|
|
|
func (b *BaseBrowser) RebuildIndex() {
|
|
|
|
log.Debugf("Rebuilding index based on current nodeTree")
|
2018-11-09 17:25:50 +00:00
|
|
|
b.URLIndex = index.NewIndex()
|
|
|
|
tree.WalkBuildIndex(b.NodeTree, b.URLIndex)
|
2018-05-26 13:34:55 +00:00
|
|
|
}
|
|
|
|
|
2018-11-01 16:00:58 +00:00
|
|
|
func (b *BaseBrowser) RebuildNodeTree() {
|
2018-11-09 17:25:50 +00:00
|
|
|
b.NodeTree = &tree.Node{
|
|
|
|
Name: "root",
|
|
|
|
Parent: nil,
|
|
|
|
Type: "root",
|
|
|
|
}
|
2018-11-01 16:00:58 +00:00
|
|
|
}
|
|
|
|
|
2018-11-30 03:15:08 +00:00
|
|
|
func (b *BaseBrowser) InitBuffer() error {
|
|
|
|
var err error
|
2017-11-23 01:51:23 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
bufferName := fmt.Sprintf("buffer_%s", b.name)
|
2018-12-04 03:34:30 +00:00
|
|
|
b.BufferDB, err = database.New(bufferName, "", database.DBTypeInMemoryDSN).Init()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = b.BufferDB.InitSchema()
|
2018-11-30 03:15:08 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
2017-11-19 20:28:24 +00:00
|
|
|
}
|
2017-11-20 18:07:15 +00:00
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
func (b *BaseBrowser) RegisterHooks(hooks ...parsing.Hook) {
|
2017-11-20 18:07:15 +00:00
|
|
|
log.Debug("Registering hooks")
|
|
|
|
for _, hook := range hooks {
|
|
|
|
b.parseHooks = append(b.parseHooks, hook)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-24 16:16:42 +00:00
|
|
|
func (b *BaseBrowser) ResetStats() {
|
2018-11-09 17:25:50 +00:00
|
|
|
b.Stats.LastURLCount = b.Stats.CurrentUrlCount
|
|
|
|
b.Stats.LastNodeCount = b.Stats.CurrentNodeCount
|
|
|
|
b.Stats.CurrentNodeCount = 0
|
|
|
|
b.Stats.CurrentUrlCount = 0
|
2018-10-24 16:16:42 +00:00
|
|
|
}
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
func (b *BaseBrowser) HasReducer() bool {
|
|
|
|
return b.eventsChan != nil
|
|
|
|
}
|
|
|
|
|
2018-12-01 17:16:46 +00:00
|
|
|
func (b *BaseBrowser) String() string {
|
2018-11-02 17:21:18 +00:00
|
|
|
return b.name
|
|
|
|
}
|
|
|
|
|
2017-11-20 18:07:15 +00:00
|
|
|
// Runs browsed defined hooks on bookmark
|
2018-11-09 17:25:50 +00:00
|
|
|
func (b *BaseBrowser) RunParseHooks(node *tree.Node) {
|
2017-11-20 18:07:15 +00:00
|
|
|
for _, hook := range b.parseHooks {
|
2017-11-30 15:08:12 +00:00
|
|
|
hook(node)
|
2017-11-20 18:07:15 +00:00
|
|
|
}
|
|
|
|
}
|