2018-05-27 16:02:50 +00:00
|
|
|
// ### Browsers API
|
2018-05-27 16:07:41 +00:00
|
|
|
//
|
|
|
|
// All constants and common API for browsers should be implemented here.
|
|
|
|
//
|
|
|
|
// For *browser specific* implementation create a new file for that browser.
|
2018-05-27 16:09:53 +00:00
|
|
|
//
|
2018-05-27 16:10:49 +00:00
|
|
|
// You must then implement a `func New[BrowserType]() IBrowser` function and
|
2018-05-27 16:07:41 +00:00
|
|
|
// implement parsing.
|
2017-11-19 16:02:53 +00:00
|
|
|
package main
|
|
|
|
|
2017-11-19 20:28:24 +00:00
|
|
|
import (
|
2017-11-20 15:05:44 +00:00
|
|
|
"fmt"
|
2018-11-09 17:25:50 +00:00
|
|
|
"gomark/database"
|
|
|
|
"gomark/index"
|
|
|
|
"gomark/parsing"
|
|
|
|
"gomark/tree"
|
|
|
|
"gomark/watch"
|
2017-11-20 15:05:44 +00:00
|
|
|
"path"
|
2018-11-01 16:00:58 +00:00
|
|
|
"reflect"
|
2017-11-19 20:28:24 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
"github.com/fsnotify/fsnotify"
|
2017-11-30 15:08:12 +00:00
|
|
|
"github.com/sp4ke/hashmap"
|
2017-11-19 20:28:24 +00:00
|
|
|
)
|
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
type IWatchable = watch.IWatchable
|
|
|
|
type Watcher = watch.Watcher
|
|
|
|
type Watch = watch.Watch
|
2018-11-13 16:11:16 +00:00
|
|
|
|
2017-11-19 19:46:24 +00:00
|
|
|
type BrowserType uint8
|
2017-11-19 16:02:53 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
// Browser types
|
2017-11-19 16:02:53 +00:00
|
|
|
const (
|
2017-11-20 15:05:44 +00:00
|
|
|
TChrome BrowserType = iota
|
|
|
|
TFirefox
|
2017-11-19 16:02:53 +00:00
|
|
|
)
|
2017-11-19 20:28:24 +00:00
|
|
|
|
2018-06-14 00:30:18 +00:00
|
|
|
// Channel parameters
|
|
|
|
const EventsChanLen = 1000
|
|
|
|
|
2018-06-08 15:38:57 +00:00
|
|
|
// Used to store bookmark paths and other
|
|
|
|
// data related to a particular browser kind
|
|
|
|
type BrowserPaths struct {
|
2017-11-20 15:05:44 +00:00
|
|
|
BookmarkFile string
|
|
|
|
BookmarkDir string
|
|
|
|
}
|
|
|
|
|
|
|
|
type IBrowser interface {
|
2017-11-20 18:07:15 +00:00
|
|
|
IWatchable
|
2018-10-28 19:19:12 +00:00
|
|
|
InitBuffer() // init buffer db, TODO: defer closings and shutdown
|
2018-05-26 13:34:55 +00:00
|
|
|
InitIndex() // Creates in memory Index (RB-Tree)
|
2018-11-09 17:25:50 +00:00
|
|
|
RegisterHooks(...parsing.Hook)
|
2017-11-20 18:07:15 +00:00
|
|
|
Load() // Loads bookmarks to db without watching
|
2018-11-09 17:25:50 +00:00
|
|
|
//Parse(...parsing.Hook) // Main parsing method with different parsing hooks
|
2018-10-28 19:19:12 +00:00
|
|
|
Shutdown() // Graceful shutdown, it should call the BaseBrowser.Close()
|
2017-11-19 20:28:24 +00:00
|
|
|
}
|
|
|
|
|
2017-11-19 20:42:20 +00:00
|
|
|
// Base browser class serves as reference for implmented browser types
|
|
|
|
// Browser should contain enough data internally to not rely on any global
|
|
|
|
// variable or constant if possible.
|
2018-05-26 13:34:55 +00:00
|
|
|
// To create new browsers, you must implement a New<BrowserType>() IBrowser function
|
2018-05-27 15:36:03 +00:00
|
|
|
//
|
2018-05-27 16:17:17 +00:00
|
|
|
// `URLIndex` (HashMap RBTree):
|
2018-05-26 13:34:55 +00:00
|
|
|
// Used as fast query db representing the last known browser bookmarks.
|
2018-05-27 15:36:03 +00:00
|
|
|
//
|
2018-05-27 16:17:17 +00:00
|
|
|
// `nodeTree` (Tree DAG):
|
2018-05-27 15:36:03 +00:00
|
|
|
// Used in each job to represent bookmarks in a tree
|
|
|
|
//
|
2018-05-27 16:21:27 +00:00
|
|
|
// `BufferDB`: sqlite buffer used across jobs
|
2017-11-19 20:28:24 +00:00
|
|
|
type BaseBrowser struct {
|
2018-11-02 17:21:18 +00:00
|
|
|
watcher *Watcher
|
2018-06-14 00:30:18 +00:00
|
|
|
eventsChan chan fsnotify.Event
|
|
|
|
baseDir string
|
|
|
|
bkFile string
|
2018-05-27 16:21:27 +00:00
|
|
|
|
|
|
|
// In memory sqlite db (named `memcache`).
|
|
|
|
// Used to keep a browser's state of bookmarks across jobs.
|
|
|
|
BufferDB *DB
|
|
|
|
|
|
|
|
// Fast query db using an RB-Tree hashmap.
|
2018-05-27 16:22:24 +00:00
|
|
|
// It represents a URL index of the last running job
|
2018-05-27 16:21:27 +00:00
|
|
|
URLIndex *hashmap.RBTree
|
2018-05-27 16:17:17 +00:00
|
|
|
|
|
|
|
// Pointer to the root of the node tree
|
|
|
|
// The node tree is built again for every Run job on a browser
|
2018-11-09 17:25:50 +00:00
|
|
|
NodeTree *tree.Node
|
2018-05-27 16:17:17 +00:00
|
|
|
// Various parsing and timing stats
|
2018-11-09 17:25:50 +00:00
|
|
|
Stats *parsing.Stats
|
2018-11-01 16:00:58 +00:00
|
|
|
bType BrowserType
|
|
|
|
name string
|
|
|
|
isWatching bool
|
|
|
|
useFileWatcher bool
|
2018-11-09 17:25:50 +00:00
|
|
|
parseHooks []parsing.Hook
|
2018-11-01 16:00:58 +00:00
|
|
|
}
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
func (bw *BaseBrowser) GetWatcher() *Watcher {
|
|
|
|
watcherType := reflect.TypeOf((*Watcher)(nil)).Elem()
|
|
|
|
// In case we use other types of watchers/events
|
|
|
|
if reflect.TypeOf(bw.watcher) == reflect.PtrTo(watcherType) {
|
2018-11-01 16:00:58 +00:00
|
|
|
return bw.watcher
|
|
|
|
}
|
|
|
|
return nil
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
func (bw *BaseBrowser) Load() {
|
|
|
|
log.Debug("BaseBrowser Load()")
|
2017-12-02 12:44:15 +00:00
|
|
|
bw.InitIndex()
|
|
|
|
|
|
|
|
// Check if cache is initialized
|
2018-11-09 17:25:50 +00:00
|
|
|
if CacheDB == nil || CacheDB.Handle == nil {
|
2018-10-28 19:19:12 +00:00
|
|
|
log.Criticalf("<%s> Loading bookmarks while cache not yet initialized !", bw.name)
|
2017-12-02 12:44:15 +00:00
|
|
|
}
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
// In case we use other types of watchers/events
|
2018-11-01 16:00:58 +00:00
|
|
|
if bw.useFileWatcher && bw.watcher == nil {
|
2018-11-02 17:21:18 +00:00
|
|
|
log.Warningf("<%s> watcher not initialized, use SetupFileWatcher() when creating the browser !", bw.name)
|
2017-12-02 12:44:15 +00:00
|
|
|
}
|
|
|
|
|
2018-10-23 20:14:54 +00:00
|
|
|
log.Debugf("<%s> preloading bookmarks", bw.name)
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (bw *BaseBrowser) GetPath() string {
|
|
|
|
return path.Join(bw.baseDir, bw.bkFile)
|
|
|
|
}
|
|
|
|
|
2018-06-14 00:30:18 +00:00
|
|
|
func (bw *BaseBrowser) EventsChan() chan fsnotify.Event {
|
|
|
|
return bw.eventsChan
|
|
|
|
}
|
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
func (bw *BaseBrowser) GetDir() string {
|
|
|
|
return bw.baseDir
|
|
|
|
}
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
// Setup file watcher using the provided []Watch elements
|
|
|
|
func (bw *BaseBrowser) SetupFileWatcher(watches ...*Watch) {
|
|
|
|
var err error
|
|
|
|
|
2018-11-01 16:00:58 +00:00
|
|
|
if !bw.useFileWatcher {
|
|
|
|
return
|
|
|
|
}
|
2018-11-02 17:21:18 +00:00
|
|
|
|
|
|
|
fswatcher, err := fsnotify.NewWatcher()
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
2018-11-02 17:21:18 +00:00
|
|
|
|
|
|
|
watchedMap := make(map[string]*Watch)
|
|
|
|
for _, v := range watches {
|
2018-11-09 17:25:50 +00:00
|
|
|
watchedMap[v.Path] = v
|
2018-11-02 17:21:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bw.watcher = &Watcher{
|
2018-11-09 17:25:50 +00:00
|
|
|
W: fswatcher,
|
|
|
|
Watched: watchedMap,
|
|
|
|
Watches: watches,
|
2018-11-02 17:21:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add all watched paths
|
|
|
|
for _, v := range watches {
|
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
err = bw.watcher.W.Add(v.Path)
|
2018-11-02 17:21:18 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
2018-10-28 19:19:12 +00:00
|
|
|
}
|
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
}
|
|
|
|
|
2018-10-26 16:25:07 +00:00
|
|
|
func (bw *BaseBrowser) ResetWatcher() {
|
2018-11-09 17:25:50 +00:00
|
|
|
err := bw.watcher.W.Close()
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Critical(err)
|
|
|
|
}
|
2018-11-09 17:25:50 +00:00
|
|
|
bw.SetupFileWatcher(bw.watcher.Watches...)
|
2018-10-26 16:25:07 +00:00
|
|
|
}
|
|
|
|
|
2018-10-28 19:19:12 +00:00
|
|
|
func (bw *BaseBrowser) Close() error {
|
2018-11-09 17:25:50 +00:00
|
|
|
err := bw.watcher.W.Close()
|
2018-10-28 19:19:12 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = bw.BufferDB.Close()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
2017-11-19 20:28:24 +00:00
|
|
|
}
|
|
|
|
|
2017-11-30 15:08:12 +00:00
|
|
|
func (b *BaseBrowser) InitIndex() {
|
2018-11-09 17:25:50 +00:00
|
|
|
b.URLIndex = index.NewIndex()
|
2017-11-30 15:08:12 +00:00
|
|
|
}
|
|
|
|
|
2018-05-26 13:34:55 +00:00
|
|
|
func (b *BaseBrowser) RebuildIndex() {
|
|
|
|
log.Debugf("Rebuilding index based on current nodeTree")
|
2018-11-09 17:25:50 +00:00
|
|
|
b.URLIndex = index.NewIndex()
|
|
|
|
tree.WalkBuildIndex(b.NodeTree, b.URLIndex)
|
2018-05-26 13:34:55 +00:00
|
|
|
}
|
|
|
|
|
2018-11-01 16:00:58 +00:00
|
|
|
func (b *BaseBrowser) RebuildNodeTree() {
|
2018-11-09 17:25:50 +00:00
|
|
|
b.NodeTree = &tree.Node{
|
|
|
|
Name: "root",
|
|
|
|
Parent: nil,
|
|
|
|
Type: "root",
|
|
|
|
}
|
2018-11-01 16:00:58 +00:00
|
|
|
}
|
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
func (b *BaseBrowser) InitBuffer() {
|
2017-11-23 01:51:23 +00:00
|
|
|
|
2017-11-20 15:05:44 +00:00
|
|
|
bufferName := fmt.Sprintf("buffer_%s", b.name)
|
2018-11-09 17:25:50 +00:00
|
|
|
bufferPath := fmt.Sprintf(database.DBBufferFmt, bufferName)
|
2018-11-13 16:11:16 +00:00
|
|
|
b.BufferDB = database.New(bufferName, bufferPath)
|
2018-05-27 16:21:27 +00:00
|
|
|
b.BufferDB.Init()
|
|
|
|
b.BufferDB.Attach(CacheDB)
|
2017-11-19 20:28:24 +00:00
|
|
|
}
|
2017-11-20 18:07:15 +00:00
|
|
|
|
2018-11-09 17:25:50 +00:00
|
|
|
func (b *BaseBrowser) RegisterHooks(hooks ...parsing.Hook) {
|
2017-11-20 18:07:15 +00:00
|
|
|
log.Debug("Registering hooks")
|
|
|
|
for _, hook := range hooks {
|
|
|
|
b.parseHooks = append(b.parseHooks, hook)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-24 16:16:42 +00:00
|
|
|
func (b *BaseBrowser) ResetStats() {
|
2018-11-09 17:25:50 +00:00
|
|
|
b.Stats.LastURLCount = b.Stats.CurrentUrlCount
|
|
|
|
b.Stats.LastNodeCount = b.Stats.CurrentNodeCount
|
|
|
|
b.Stats.CurrentNodeCount = 0
|
|
|
|
b.Stats.CurrentUrlCount = 0
|
2018-10-24 16:16:42 +00:00
|
|
|
}
|
|
|
|
|
2018-11-02 17:21:18 +00:00
|
|
|
func (b *BaseBrowser) HasReducer() bool {
|
|
|
|
return b.eventsChan != nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *BaseBrowser) Name() string {
|
|
|
|
return b.name
|
|
|
|
}
|
|
|
|
|
2017-11-20 18:07:15 +00:00
|
|
|
// Runs browsed defined hooks on bookmark
|
2018-11-09 17:25:50 +00:00
|
|
|
func (b *BaseBrowser) RunParseHooks(node *tree.Node) {
|
2017-11-20 18:07:15 +00:00
|
|
|
for _, hook := range b.parseHooks {
|
2017-11-30 15:08:12 +00:00
|
|
|
hook(node)
|
2017-11-20 18:07:15 +00:00
|
|
|
}
|
|
|
|
}
|