2023-01-31 21:21:39 +00:00
|
|
|
package modules
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"path"
|
|
|
|
"path/filepath"
|
2023-09-08 09:56:29 +00:00
|
|
|
"time"
|
2022-10-27 22:33:20 +00:00
|
|
|
|
2023-02-18 23:13:23 +00:00
|
|
|
"git.blob42.xyz/gomark/gosuki/database"
|
|
|
|
"git.blob42.xyz/gomark/gosuki/index"
|
|
|
|
"git.blob42.xyz/gomark/gosuki/logging"
|
|
|
|
"git.blob42.xyz/gomark/gosuki/parsing"
|
2023-09-08 13:01:29 +00:00
|
|
|
"git.blob42.xyz/gomark/gosuki/hooks"
|
2023-02-18 23:13:23 +00:00
|
|
|
"git.blob42.xyz/gomark/gosuki/tree"
|
|
|
|
"git.blob42.xyz/gomark/gosuki/utils"
|
|
|
|
"git.blob42.xyz/gomark/gosuki/watch"
|
2022-10-27 22:33:20 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type BrowserType uint8
|
|
|
|
|
|
|
|
// Browser types
|
|
|
|
const (
|
|
|
|
// Chromium based browsers (chrome, brave ... )
|
|
|
|
TChrome BrowserType = iota
|
|
|
|
|
|
|
|
// Firefox based browsers ie. they relay on places.sqlite
|
|
|
|
TFirefox
|
|
|
|
|
|
|
|
// Other
|
|
|
|
TCustom
|
|
|
|
)
|
|
|
|
|
|
|
|
// reducer channel length, bigger means less sensitivity to events
|
|
|
|
var (
|
2022-11-01 13:27:19 +00:00
|
|
|
log = logging.GetLogger("BASE")
|
|
|
|
ReducerChanLen = 1000
|
2022-10-27 22:33:20 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type Browser interface {
|
|
|
|
// Returns a pointer to an initialized browser config
|
|
|
|
Config() *BrowserConfig
|
|
|
|
}
|
|
|
|
|
2023-09-05 17:45:21 +00:00
|
|
|
// The profile preferences for modules with builtin profile management.
|
2023-09-05 19:11:21 +00:00
|
|
|
type ProfilePrefs struct {
|
2023-09-05 17:45:21 +00:00
|
|
|
|
|
|
|
// Whether to watch all the profiles for multi-profile modules
|
|
|
|
WatchAllProfiles bool `toml:"watch_all_profiles"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// BrowserConfig is the main browser configuration shared by all browser modules.
|
2022-10-27 22:33:20 +00:00
|
|
|
type BrowserConfig struct {
|
|
|
|
Name string
|
|
|
|
Type BrowserType
|
|
|
|
|
2023-09-05 17:45:21 +00:00
|
|
|
// Absolute path to the browser's bookmark directory
|
2022-10-27 22:33:20 +00:00
|
|
|
BkDir string
|
|
|
|
|
|
|
|
// Name of bookmarks file
|
|
|
|
BkFile string
|
|
|
|
|
2023-09-05 17:45:21 +00:00
|
|
|
|
2023-09-05 19:11:21 +00:00
|
|
|
ProfilePrefs
|
2023-09-05 17:45:21 +00:00
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// In memory sqlite db (named `memcache`).
|
|
|
|
// Used to keep a browser's state of bookmarks across jobs.
|
|
|
|
BufferDB *database.DB
|
|
|
|
|
|
|
|
// Fast query db using an RB-Tree hashmap.
|
|
|
|
// It represents a URL index of the last running job
|
2023-09-08 09:56:29 +00:00
|
|
|
URLIndex index.HashTree
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
// Pointer to the root of the node tree
|
|
|
|
// The node tree is built again for every Run job on a browser
|
|
|
|
NodeTree *tree.Node
|
|
|
|
// Various parsing and timing stats
|
|
|
|
*parsing.Stats
|
|
|
|
|
|
|
|
watcher *watch.WatchDescriptor
|
|
|
|
UseFileWatcher bool
|
|
|
|
|
2023-09-08 09:56:29 +00:00
|
|
|
// Hooks registered by the browser module identified by name
|
|
|
|
UseHooks []string
|
|
|
|
|
|
|
|
// Registered hooks
|
2023-09-08 13:01:29 +00:00
|
|
|
hooks map[string]hooks.Hook
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
2023-09-08 09:56:29 +00:00
|
|
|
func (b *BrowserConfig) GetWatcher() *watch.WatchDescriptor {
|
|
|
|
return b.watcher
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
2023-09-08 09:56:29 +00:00
|
|
|
func (b BrowserConfig) BookmarkDir() (string, error) {
|
2023-09-06 12:24:30 +00:00
|
|
|
var err error
|
2023-09-08 09:56:29 +00:00
|
|
|
bDir, err := filepath.EvalSymlinks(utils.ExpandPath(b.BkDir))
|
2023-09-06 12:24:30 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
exists, err := utils.CheckDirExists(bDir)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
if !exists {
|
|
|
|
return "", fmt.Errorf("not a bookmark dir: %s ", bDir)
|
|
|
|
}
|
|
|
|
|
|
|
|
return bDir, nil
|
|
|
|
}
|
|
|
|
|
2023-09-08 09:56:29 +00:00
|
|
|
// CallHooks calls all registered hooks for this browser for the given
|
|
|
|
// *tree.Node. The hooks are called in the order they were registered. This is
|
|
|
|
// usually done within the parsing logic of a browser module, typically in the
|
|
|
|
// Run() method.
|
|
|
|
func (b BrowserConfig) CallHooks(node *tree.Node) error {
|
2023-09-09 08:31:00 +00:00
|
|
|
|
|
|
|
if node == nil {
|
|
|
|
return fmt.Errorf("hook node is nil")
|
|
|
|
}
|
|
|
|
|
2023-09-08 09:56:29 +00:00
|
|
|
for _, hook := range b.hooks {
|
2023-09-09 08:31:00 +00:00
|
|
|
log.Debugf("<%s> calling hook <%s> on node <%s>",b.Name, hook.Name, node.URL)
|
2023-09-08 09:56:29 +00:00
|
|
|
if err := hook.Func(node); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Registers hooks for this browser. Hooks are identified by their name.
|
2023-09-08 13:01:29 +00:00
|
|
|
func (b BrowserConfig) AddHooks(hooks ...hooks.Hook) {
|
2023-09-08 09:56:29 +00:00
|
|
|
for _, hook := range hooks {
|
|
|
|
b.hooks[hook.Name] = hook
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2023-09-08 13:01:29 +00:00
|
|
|
func (b BrowserConfig) HasHook(hook hooks.Hook) bool {
|
2023-09-08 09:56:29 +00:00
|
|
|
_, ok := b.hooks[hook.Name]
|
|
|
|
return ok
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2023-09-06 12:24:30 +00:00
|
|
|
//TODO!: use this method instead of manually building bookmark path
|
2023-09-09 14:08:34 +00:00
|
|
|
// BookmarkPath returns the absolute path to the bookmark file.
|
|
|
|
// It expands the path by concatenating the base directory and bookmarks file,
|
|
|
|
// then checks if it exists.
|
2023-09-08 09:56:29 +00:00
|
|
|
func (b BrowserConfig) BookmarkPath() (string, error) {
|
|
|
|
bPath, err := filepath.EvalSymlinks(path.Join(utils.ExpandPath(b.BkDir),
|
|
|
|
b.BkFile))
|
2022-10-27 22:33:20 +00:00
|
|
|
if err != nil {
|
|
|
|
log.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
exists, err := utils.CheckFileExists(bPath)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
if !exists {
|
|
|
|
return "", fmt.Errorf("not a bookmark path: %s ", bPath)
|
|
|
|
}
|
|
|
|
|
|
|
|
return bPath, nil
|
|
|
|
}
|
|
|
|
|
2023-09-08 09:56:29 +00:00
|
|
|
// Rebuilds the memory url index after parsing all bookmarks.
|
|
|
|
// Keeps the memory url index in sync with last known state of browser bookmarks
|
|
|
|
func (b BrowserConfig) RebuildIndex() {
|
|
|
|
start := time.Now()
|
|
|
|
log.Debugf("<%s> rebuilding index based on current nodeTree", b.Name)
|
|
|
|
b.URLIndex = index.NewIndex()
|
|
|
|
tree.WalkBuildIndex(b.NodeTree, b.URLIndex)
|
|
|
|
log.Debugf("<%s> index rebuilt in %s", b.Name, time.Since(start))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b BrowserConfig) ResetStats() {
|
|
|
|
log.Debugf("<%s> resetting stats", b.Name)
|
2023-09-08 13:01:29 +00:00
|
|
|
b.LastURLCount = b.CurrentURLCount
|
|
|
|
b.LastNodeCount = b.CurrentNodeCount
|
2023-09-08 09:56:29 +00:00
|
|
|
b.CurrentNodeCount = 0
|
|
|
|
b.CurrentURLCount = 0
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// Browser who implement this interface need to handle all shuttind down and
|
|
|
|
// cleanup logic in the defined methods. This is usually called at the end of
|
|
|
|
// the browser instance lifetime
|
|
|
|
type Shutdowner interface {
|
|
|
|
Shutdown() error
|
|
|
|
}
|
|
|
|
|
2023-02-23 10:59:29 +00:00
|
|
|
// Loader is an interface for modules which is run only once when the module
|
|
|
|
// starts. It should have the same effect as Watchable.Run().
|
|
|
|
// Run() is automatically called for watched events, Load() is called once
|
|
|
|
// before starting to watch events.
|
|
|
|
//
|
|
|
|
// Loader allows modules to do a first pass of Run() logic before the watcher
|
|
|
|
// threads is spawned
|
2022-10-27 22:33:20 +00:00
|
|
|
type Loader interface {
|
|
|
|
|
|
|
|
// Load() will be called right after a browser is initialized
|
|
|
|
Load() error
|
|
|
|
}
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
// Initialize the module before any data loading or callbacks
|
|
|
|
// If a module wants to do any preparation and prepare custom state before Loader.Load()
|
2022-10-27 22:33:20 +00:00
|
|
|
// is called and before any Watchable.Run() or other callbacks are executed.
|
|
|
|
type Initializer interface {
|
|
|
|
|
|
|
|
// Init() is the first method called after a browser instance is created
|
|
|
|
// and registered.
|
|
|
|
// Return ok, error
|
2023-02-18 22:08:12 +00:00
|
|
|
Init(*Context) error
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Every browser is setup once, the following methods are called in order of
|
|
|
|
// their corresponding interfaces are implemented.
|
|
|
|
// TODO!: integrate with refactoring
|
|
|
|
// 0- Provision: Sets up and custom configiguration to the browser
|
|
|
|
// 1- Init : any variable and state initialization
|
|
|
|
// 2- Load: Does the first loading of data (ex first loading of bookmarks )
|
2023-02-18 22:08:12 +00:00
|
|
|
func Setup(browser BrowserModule, c *Context) error {
|
2022-10-27 22:33:20 +00:00
|
|
|
|
2022-11-01 13:27:19 +00:00
|
|
|
//TODO!: default init
|
2022-11-07 19:07:13 +00:00
|
|
|
bConf := browser.Config()
|
2023-09-08 09:56:29 +00:00
|
|
|
|
|
|
|
// Setup registered hooks
|
2023-09-08 13:01:29 +00:00
|
|
|
bConf.hooks = make(map[string]hooks.Hook)
|
2023-09-08 09:56:29 +00:00
|
|
|
for _, hookName := range bConf.UseHooks {
|
2023-09-08 13:01:29 +00:00
|
|
|
hook, ok := hooks.Predefined[hookName]
|
2023-09-08 09:56:29 +00:00
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("hook <%s> not defined", hookName)
|
|
|
|
}
|
|
|
|
bConf.AddHooks(hook)
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Init browsers' BufferDB
|
2022-11-07 19:07:13 +00:00
|
|
|
buffer, err := database.NewBuffer(bConf.Name)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
bConf.BufferDB = buffer
|
2022-10-27 22:33:20 +00:00
|
|
|
// Creates in memory Index (RB-Tree)
|
2022-11-07 19:07:13 +00:00
|
|
|
bConf.URLIndex = index.NewIndex()
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
log.Infof("setting up browser <%s>", browser.ModInfo().ID)
|
2023-09-05 20:28:26 +00:00
|
|
|
browserID := browser.ModInfo().ID
|
2022-10-27 22:33:20 +00:00
|
|
|
|
|
|
|
// Handle Initializers custom Init from Browser module
|
|
|
|
initializer, ok := browser.(Initializer)
|
|
|
|
if ok {
|
2023-09-05 20:28:26 +00:00
|
|
|
log.Debugf("<%s> custom init", browserID)
|
2023-02-18 22:08:12 +00:00
|
|
|
if err := initializer.Init(c); err != nil {
|
2023-09-05 20:28:26 +00:00
|
|
|
return fmt.Errorf("<%s> initialization error: %v", browserID, err)
|
2022-11-01 13:27:19 +00:00
|
|
|
}
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
} else {
|
2023-09-05 20:28:26 +00:00
|
|
|
log.Warningf("<%s> does not implement Initializer, not calling Init()", browserID)
|
2022-10-27 22:33:20 +00:00
|
|
|
}
|
|
|
|
|
2023-02-18 22:08:12 +00:00
|
|
|
|
2022-11-07 20:58:35 +00:00
|
|
|
// Default browser loading logic
|
2022-11-01 13:27:19 +00:00
|
|
|
// Make sure that cache is initialized
|
|
|
|
if !database.Cache.IsInitialized() {
|
2023-09-05 20:28:26 +00:00
|
|
|
return fmt.Errorf("<%s> Loading bookmarks while cache not yet initialized", browserID)
|
2022-11-01 13:27:19 +00:00
|
|
|
}
|
|
|
|
|
2022-11-07 20:58:35 +00:00
|
|
|
//handle Loader interface
|
2022-10-27 22:33:20 +00:00
|
|
|
loader, ok := browser.(Loader)
|
|
|
|
if ok {
|
2023-09-05 20:28:26 +00:00
|
|
|
log.Debugf("<%s> custom loading", browserID)
|
2022-10-27 22:33:20 +00:00
|
|
|
err := loader.Load()
|
|
|
|
if err != nil {
|
2023-09-05 20:28:26 +00:00
|
|
|
return fmt.Errorf("loading error <%s>: %v", browserID, err)
|
2022-10-27 22:33:20 +00:00
|
|
|
// continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Setup a watcher service using the provided []Watch elements
|
|
|
|
// Returns true if a new watcher was created. false if it was previously craeted
|
|
|
|
// or if the browser does not need a watcher (UseFileWatcher == false).
|
|
|
|
func SetupWatchers(browserConf *BrowserConfig, watches ...*watch.Watch) (bool, error) {
|
|
|
|
var err error
|
|
|
|
if !browserConf.UseFileWatcher {
|
2023-09-06 12:24:30 +00:00
|
|
|
log.Warningf("<%s> does not use file watcher but asked for it", browserConf.Name)
|
2022-10-27 22:33:20 +00:00
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
browserConf.watcher, err = watch.NewWatcher(browserConf.Name, watches...)
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func SetupWatchersWithReducer(browserConf *BrowserConfig,
|
|
|
|
reducerChanLen int,
|
|
|
|
watches ...*watch.Watch) (bool, error) {
|
|
|
|
var err error
|
|
|
|
|
|
|
|
if !browserConf.UseFileWatcher {
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
browserConf.watcher, err = watch.NewWatcherWithReducer(browserConf.Name, reducerChanLen, watches...)
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return true, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2023-01-31 21:21:39 +00:00
|
|
|
|
|
|
|
|
2022-10-27 22:33:20 +00:00
|
|
|
// Used to store bookmark paths and other
|
|
|
|
// data related to a particular browser kind
|
|
|
|
// _TODO: replace in chrome with ProfileManager and remove this ref
|
|
|
|
// type BrowserPaths struct {
|
|
|
|
// BookmarkFile string
|
|
|
|
// BookmarkDir string
|
|
|
|
// }
|