blob42/gosuki
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chrome: loads and watches bookmark changes + hook calling

Browse Source
This commit is contained in:
blob42 2023-09-08 11:56:54 +02:00
parent e3e8f0d5ad
commit 94112e5f0b
2 changed files with 422 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

741
chrome/chrome.go
View File

@ -1,321 +1,422 @@
package chrome package chrome
//
// import ( import (
// "errors" "errors"
// "fmt" "fmt"
// "path" "os"
// "path/filepath" "path/filepath"
// "time" "time"
//
// "github.com/fsnotify/fsnotify" "github.com/OneOfOne/xxhash"
// "github.com/fsnotify/fsnotify"
// "git.blob42.xyz/gomark/gosuki/database"
// "git.blob42.xyz/gomark/gosuki/logging" "github.com/buger/jsonparser"
// "git.blob42.xyz/gomark/gosuki/modules"
// "git.blob42.xyz/gomark/gosuki/tree" "git.blob42.xyz/gomark/gosuki/database"
// "git.blob42.xyz/gomark/gosuki/utils" "git.blob42.xyz/gomark/gosuki/logging"
// "git.blob42.xyz/gomark/gosuki/watch" "git.blob42.xyz/gomark/gosuki/modules"
// ) "git.blob42.xyz/gomark/gosuki/tree"
// "git.blob42.xyz/gomark/gosuki/watch"
// var ( )
// log = logging.GetLogger("Chrome")
// ) var (
// log = logging.GetLogger("Chrome")
// // Chrome browser module )
// type Chrome struct {
// // holds browsers.BrowserConfig type ParseChildJSONFunc func([]byte, jsonparser.ValueType, int, error)
// *ChromeConfig type RecursiveParseJSONFunc func([]byte, []byte, jsonparser.ValueType, int) error
// }
// var jsonNodeTypes = map[string]tree.NodeType{
// // Init() is the first method called after a browser instance is created "folder": tree.FolderNode,
// // and registered. "url": tree.URLNode,
// // Return ok, error }
// func (ch *Chrome) Init(_ *modules.Context) error {
// log.Infof("initializing <%s>", ch.Name) var jsonNodePaths = struct {
// Type, Children, URL string
// bookmarkDir, err := ch.BookmarkDir() }{"type", "children", "url"}
// log.Debugf("Watching path: %s", bookmarkDir)
// if err != nil { // stores json nodes in memory
// return err type RawNode struct {
// } name []byte
// nType []byte
// bookmarkPath := filepath.Join(bookmarkDir, ch.BkFile) url []byte
// children []byte
// // Setup watcher childrenType jsonparser.ValueType
// w := &watch.Watch{ }
// Path: bookmarkDir,
// EventTypes: []fsnotify.Op{fsnotify.Create}, func (rawNode *RawNode) parseItems(nodeData []byte) {
// EventNames: []string{bookmarkPath},
// ResetWatch: true, // Paths to lookup in node payload
// } paths := [][]string{
// {"type"},
// ok, err := modules.SetupWatchers(ch.BrowserConfig, w) {"name"}, // Title of page
// if err != nil { {"url"},
// return fmt.Errorf("could not setup watcher: %s", err) {"children"},
// } }
// if !ok {
// return errors.New("could not setup watcher") jsonparser.EachKey(nodeData, func(idx int, value []byte, vt jsonparser.ValueType, err error) {
// } if err != nil {
// log.Critical("error parsing node items")
// return nil }
// }
// switch idx {
// // Returns a pointer to an initialized browser config case 0:
// func (ch Chrome) Config() *modules.BrowserConfig { rawNode.nType = value
// return ch.BrowserConfig
// } case 1: // name or title
// //currentNode.Name = s(value)
// func (ch Chrome) ModInfo() modules.ModInfo { rawNode.name = value
// return modules.ModInfo{ case 2:
// ID: modules.ModID(ch.Name), //currentNode.URL = s(value)
// New: func() modules.Module { rawNode.url = value
// return NewChrome() case 3:
// }, rawNode.children, rawNode.childrenType = value, vt
// } }
// } }, paths...)
// }
// func (ch *Chrome) Watch() *watch.WatchDescriptor {
// // calls modules.BrowserConfig.GetWatcher() // Returns *tree.Node from *RawNode
// return ch.GetWatcher() func (rawNode *RawNode) getNode() *tree.Node {
// } node := new(tree.Node)
// nType, ok := jsonNodeTypes[string(rawNode.nType)]
// func (ch *Chrome) Run() { if !ok {
// startRun := time.Now() log.Criticalf("unknown node type: %s", rawNode.nType)
// }
// // Rebuild node tree node.Type = nType
// ch.NodeTree = &tree.Node{
// Name: RootNodeName, node.Name = string(rawNode.name)
// Parent: nil,
// Type: tree.RootNode, return node
// } }
//
// // Load bookmark file // Chrome browser module
// //TODO: use base.GetBookmarksPath type Chrome struct {
// bookmarkPath := path.Join(ch.BkDir, ch.BkFile) // holds browsers.BrowserConfig
// f, err := ioutil.ReadFile(bookmarkPath) *ChromeConfig
// if err != nil { }
// log.Critical(err)
// } // Init() is the first method called after a browser instance is created
// // and registered.
// var parseChildren ParseChildJsonFunc // Return ok, error
// var jsonParseRecursive RecursiveParseJsonFunc func (ch *Chrome) Init(_ *modules.Context) error {
// log.Infof("initializing <%s>", ch.Name)
// parseChildren = func(childVal []byte, dataType jsonparser.ValueType, offset int, err error) { return ch.setupWatchers()
// if err != nil { }
// log.Panic(err)
// } func (ch *Chrome) setupWatchers() error {
// bookmarkDir, err := ch.BookmarkDir()
// jsonParseRecursive(nil, childVal, dataType, offset) log.Debugf("Watching path: %s", bookmarkDir)
// } if err != nil {
// return err
// // Needed to store the parent of each child node }
// var parentNodes []*tree.Node bookmarkPath := filepath.Join(bookmarkDir, ch.BkFile)
// // Setup watcher
// jsonParseRoots := func(key []byte, node []byte, dataType jsonparser.ValueType, offset int) error { w := &watch.Watch{
// Path: bookmarkDir,
// // If node type is string ignore (needed for sync_transaction_version) EventTypes: []fsnotify.Op{fsnotify.Create},
// if dataType == jsonparser.String { EventNames: []string{bookmarkPath},
// return nil ResetWatch: true,
// } }
//
// ch.Stats.CurrentNodeCount++ ok, err := modules.SetupWatchers(ch.BrowserConfig, w)
// rawNode := new(RawNode) if err != nil {
// rawNode.parseItems(node) return fmt.Errorf("could not setup watcher: %s", err)
// //log.Debugf("Parsing root folder %s", rawNode.name) }
// if !ok {
// currentNode := rawNode.getNode() return errors.New("could not setup watcher")
// }
// // Process this node as parent node later
// parentNodes = append(parentNodes, currentNode) return nil
// }
// // add the root node as parent to this node
// currentNode.Parent = ch.NodeTree func (ch *Chrome) ResetWatcher() error {
// w := ch.GetWatcher()
// // Add this root node as a child of the root node if err := w.W.Close(); err != nil {
// ch.NodeTree.Children = append(ch.NodeTree.Children, currentNode) return err
// }
// // Call recursive parsing of this node which must return ch.setupWatchers()
// // a root folder node }
// jsonparser.ArrayEach(node, parseChildren, jsonNodePaths.Children)
// // Returns a pointer to an initialized browser config
// // Finished parsing this root, it is not anymore a parent func (ch Chrome) Config() *modules.BrowserConfig {
// _, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1] return ch.BrowserConfig
// }
// //log.Debugf("Parsed root %s folder", rawNode.name)
// func (ch Chrome) ModInfo() modules.ModInfo {
// return nil return modules.ModInfo{
// } ID: modules.ModID(ch.Name),
// New: func() modules.Module {
// // Main recursive parsing function that parses underneath return NewChrome()
// // each root folder },
// jsonParseRecursive = func(key []byte, node []byte, dataType jsonparser.ValueType, offset int) error { }
// }
// // If node type is string ignore (needed for sync_transaction_version)
// if dataType == jsonparser.String { func (ch *Chrome) Watch() *watch.WatchDescriptor {
// return nil // calls modules.BrowserConfig.GetWatcher()
// } return ch.GetWatcher()
// }
// ch.Stats.CurrentNodeCount++
// func (ch *Chrome) Run() {
// rawNode := new(RawNode) startRun := time.Now()
// rawNode.parseItems(node)
// // Rebuild node tree
// currentNode := rawNode.getNode() ch.NodeTree = &tree.Node{
// //log.Debugf("parsing node %s", currentNode.Name) Name: RootNodeName,
// Parent: nil,
// // if parents array is not empty Type: tree.RootNode,
// if len(parentNodes) != 0 { }
// parent := parentNodes[len(parentNodes)-1]
// //log.Debugf("Adding current node to parent %s", parent.Name) // Load bookmark file
// //WIP: use builting path helpers
// // Add current node to closest parent bookmarkPath, err := ch.BookmarkPath()
// currentNode.Parent = parent if err != nil {
// log.Critical(err)
// // Add current node as child to parent return
// currentNode.Parent.Children = append(currentNode.Parent.Children, currentNode) }
// }
// f, err := os.ReadFile(bookmarkPath)
// // if node is a folder with children if err != nil {
// if rawNode.childrenType == jsonparser.Array && len(rawNode.children) > 2 { // if len(children) > len("[]") log.Critical(err)
// return
// //log.Debugf("Started folder %s", rawNode.name) }
// parentNodes = append(parentNodes, currentNode)
// var parseChildren ParseChildJSONFunc
// // Process recursively all child nodes of this folder node var jsonParseRecursive RecursiveParseJSONFunc
// jsonparser.ArrayEach(node, parseChildren, jsonNodePaths.Children)
// parseChildren = func(childVal []byte,
// //log.Debugf("Finished folder %s", rawNode.name) dataType jsonparser.ValueType,
// _, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1] offset int,
// err error) {
// } if err != nil {
// log.Panic(err)
// // if node is url(leaf), handle the url }
// if utils.S(rawNode.nType) == jsonNodeTypes.URL {
// err = jsonParseRecursive(nil, childVal, dataType, offset)
// currentNode.URL = utils.S(rawNode.url) if err != nil {
// ch.Stats.CurrentUrlCount++ log.Critical(err)
// // Check if url-node already in index }
// var nodeVal *tree.Node
// iVal, found := ch.URLIndex.Get(currentNode.URL) }
//
// nameHash := xxhash.ChecksumString64(currentNode.Name) // Needed to store the parent of each child node
// // If node url not in index, add it to index var parentNodes []*tree.Node
// if !found {
// //log.Debugf("Not found") jsonParseRoots := func(key []byte,
// node []byte,
// // store hash(name) dataType jsonparser.ValueType,
// currentNode.NameHash = nameHash offset int,
// ) error {
// // The value in the index will be a
// // pointer to currentNode // If node type is string ignore (needed for sync_transaction_version)
// //log.Debugf("Inserting url %s to index", nodeURL) if dataType == jsonparser.String {
// ch.URLIndex.Insert(currentNode.URL, currentNode) return nil
// }
// // Run tag parsing hooks
// ch.RunParseHooks(currentNode) ch.CurrentNodeCount++
// rawNode := new(RawNode)
// // If we find the node already in index rawNode.parseItems(node)
// // we check if the hash(name) changed meaning
// // the data changed //log.Debugf("Parsing root folder %s", rawNode.name)
// } else {
// //log.Debugf("URL Found in index") currentNode := rawNode.getNode()
// nodeVal = iVal.(*tree.Node)
// // Process this node as parent node later
// // hash(name) is different meaning new commands/tags could parentNodes = append(parentNodes, currentNode)
// // be added, we need to process the parsing hoos
// if nodeVal.NameHash != nameHash { // add the root node as parent to this node
// //log.Debugf("URL name changed !") currentNode.Parent = ch.NodeTree
//
// // Run parse hooks on node // Add this root node as a child of the root node
// ch.RunParseHooks(currentNode) ch.NodeTree.Children = append(ch.NodeTree.Children, currentNode)
// }
// // Call recursive parsing of this node which must
// // Else we do nothing, the node will not // a root folder node
// // change jsonparser.ArrayEach(node, parseChildren, jsonNodePaths.Children)
// }
// // Finished parsing this root, it is not anymore a parent
// //If parent is folder, add it as tag and add current node as child _, parentNodes = parentNodes[len(parentNodes)-1],
// //And add this link as child parentNodes[:len(parentNodes)-1]
// if currentNode.Parent.Type == jsonNodeTypes.Folder {
// //log.Debug("Parent is folder, parsing as tag ...") //log.Debugf("Parsed root %s folder", rawNode.name)
// currentNode.Tags = append(currentNode.Tags, currentNode.Parent.Name)
// } return nil
// }
// }
// // Main recursive parsing underneath each root folder
// return nil jsonParseRecursive = func(key []byte,
// } node []byte,
// dataType jsonparser.ValueType,
// rootsData, _, _, _ := jsonparser.Get(f, "roots") offset int,
// ) error {
// // Start a new node tree building job
// jsonparser.ObjectEach(rootsData, jsonParseRoots) // If node type is string ignore (needed for sync_transaction_version)
// ch.Stats.LastFullTreeParseTime = time.Since(startRun) if dataType == jsonparser.String {
// log.Debugf("<%s> parsed tree in %s", ch.Name, ch.Stats.LastFullTreeParseTime) return nil
// // Finished node tree building job }
//
// // Debug walk tree ch.CurrentNodeCount++
// //go PrintTree(ch.NodeTree)
// rawNode := new(RawNode)
// // Reset the index to represent the nodetree rawNode.parseItems(node)
// ch.RebuildIndex()
// currentNode := rawNode.getNode()
// // Finished parsing //log.Debugf("parsing node %s", currentNode.Name)
// log.Debugf("<%s> parsed %d bookmarks and %d nodes", ch.Name, ch.Stats.CurrentUrlCount, ch.Stats.CurrentNodeCount)
// // Reset parser counter // if parents array is not empty
// ch.ResetStats() if len(parentNodes) != 0 {
// parent := parentNodes[len(parentNodes)-1]
// //Add nodeTree to Cache //log.Debugf("Adding current node to parent %s", parent.Name)
// //log.Debugf("<%s> buffer content", ch.Name)
// //ch.BufferDB.Print() // Add current node to closest parent
// currentNode.Parent = parent
// log.Debugf("<%s> syncing to buffer", ch.Name)
// database.SyncTreeToBuffer(ch.NodeTree, ch.BufferDB) // Add current node as child to parent
// log.Debugf("<%s> tree synced to buffer", ch.Name) currentNode.Parent.Children = append(currentNode.Parent.Children, currentNode)
// }
// //ch.BufferDB.Print()
// // if node is a folder with children
// // database.Cache represents bookmarks across all browsers if rawNode.childrenType == jsonparser.Array && len(rawNode.children) > 2 { // if len(children) > len("[]")
// // From browsers it should support: add/update
// // Delete method should only be possible through admin interface //log.Debugf("Started folder %s", rawNode.name)
// // We could have an @ignore command to ignore a bookmark parentNodes = append(parentNodes, currentNode)
//
// // URLIndex is a hashmap index of all URLS representing current state // Process recursively all child nodes of this folder node
// // of the browser jsonparser.ArrayEach(node, parseChildren, jsonNodePaths.Children)
//
// // nodeTree is current state of the browser as tree //log.Debugf("Finished folder %s", rawNode.name)
// _, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1]
// // Buffer is the current state of the browser represetned by
// // URLIndex and nodeTree }
//
// // If the cache is empty just copy buffer to cache // if node is url(leaf), handle the url
// // until local db is already populated and preloaded if currentNode.Type == tree.URLNode {
// //debugPrint("%d", BufferDB.Count())
// if empty, err := database.Cache.DB.IsEmpty(); empty { currentNode.URL = string(rawNode.url)
// if err != nil { ch.CurrentURLCount++
// log.Error(err)
// } // Check if url-node already in index
// log.Info("cache empty: loading buffer to CacheDB") var nodeVal *tree.Node
// iVal, found := ch.URLIndex.Get(currentNode.URL)
// ch.BufferDB.CopyTo(database.Cache.DB)
// nameHash := xxhash.ChecksumString64(currentNode.Name)
// log.Debugf("syncing <%s> to disk", database.Cache.DB.Name)
// } else { // If node url not in index, add it to index
// ch.BufferDB.SyncTo(database.Cache.DB) if !found {
// } //log.Debugf("Not found")
//
// go database.Cache.DB.SyncToDisk(database.GetDBFullPath()) // store hash(name)
// ch.Stats.LastWatchRunTime = time.Since(startRun) currentNode.NameHash = nameHash
//
// } // The value in the index will be a
// // pointer to currentNode
// log.Debugf("Inserting url %s to index", currentNode.URL)
// func NewChrome() *Chrome { ch.URLIndex.Insert(currentNode.URL, currentNode)
// return &Chrome{
// ChromeConfig: ChromeCfg, // Run registered bookmark parsing hooks
// } err = ch.CallHooks(currentNode)
// } if err != nil {
// return err
// func init() { }
// modules.RegisterBrowser(Chrome{ChromeConfig: ChromeCfg})
// } // If we find the node already in index
// we check if the hash(name) changed meaning
// the data changed
} else {
// log.Debugf("URL Found in index")
nodeVal = iVal.(*tree.Node)
// hash(name) is different meaning new commands/tags could
// be added, we need to process the parsing hoos
if nodeVal.NameHash != nameHash {
log.Debugf("URL name changed !")
// Run parse hooks on node
ch.CallHooks(currentNode)
}
// Else we do nothing, the node will not
// change
}
//If parent is folder, add it as tag and add current node as child
//And add this link as child
if currentNode.Parent.Type == tree.FolderNode {
log.Debug("Parent is folder, parsing as tag ...")
currentNode.Tags = append(currentNode.Tags, currentNode.Parent.Name)
}
}
return nil
}
// starts from the "roots" key of chrome json bookmark file
rootsData, _, _, _ := jsonparser.Get(f, "roots")
// Start a new node tree building job
jsonparser.ObjectEach(rootsData, jsonParseRoots)
ch.LastFullTreeParseTime = time.Since(startRun)
log.Debugf("<%s> parsed tree in %s", ch.Name, ch.LastFullTreeParseTime)
// Finished node tree building job
// Debug walk tree
//go PrintTree(ch.NodeTree)
// Reset the index to represent the nodetree
ch.RebuildIndex()
// Finished parsing
log.Debugf("<%s> parsed %d bookmarks and %d nodes", ch.Name, ch.CurrentURLCount, ch.CurrentNodeCount)
//Add nodeTree to Cache
//log.Debugf("<%s> buffer content", ch.Name)
//ch.BufferDB.Print()
log.Debugf("<%s> syncing to buffer", ch.Name)
database.SyncTreeToBuffer(ch.NodeTree, ch.BufferDB)
log.Debugf("<%s> tree synced to buffer", ch.Name)
//ch.BufferDB.Print()
// database.Cache represents bookmarks across all browsers
// From browsers it should support: add/update
// Delete method should only be possible through admin interface
// We could have an @ignore command to ignore a bookmark
// URLIndex is a hashmap index of all URLS representing current state
// of the browser
// nodeTree is current state of the browser as tree
// Buffer is the current state of the browser represetned by
// URLIndex and nodeTree
// If the cache is empty just copy buffer to cache
// until local db is already populated and preloaded
//debugPrint("%d", BufferDB.Count())
if empty, err := database.Cache.DB.IsEmpty(); empty {
if err != nil {
log.Error(err)
}
log.Info("cache empty: loading buffer to CacheDB")
ch.BufferDB.CopyTo(database.Cache.DB)
log.Debugf("syncing <%s> to disk", database.Cache.DB.Name)
} else {
ch.BufferDB.SyncTo(database.Cache.DB)
}
go database.Cache.DB.SyncToDisk(database.GetDBFullPath())
ch.LastWatchRunTime = time.Since(startRun)
}
func NewChrome() *Chrome {
return &Chrome{
ChromeConfig: ChromeCfg,
}
}
func init() {
modules.RegisterBrowser(Chrome{ChromeConfig: ChromeCfg})
}

1
chrome/config.go
View File

@ -34,6 +34,7 @@ var (
}, },
Stats: &parsing.Stats{}, Stats: &parsing.Stats{},
UseFileWatcher: true, UseFileWatcher: true,
UseHooks: []string{"tags_from_name"},
}, },
//TODO: profile //TODO: profile
} }