351 lines
9.1 KiB
Go
351 lines
9.1 KiB
Go
package main
|
|
|
|
import (
|
|
"gomark/database"
|
|
"gomark/parsing"
|
|
"gomark/tree"
|
|
"gomark/utils"
|
|
"gomark/watch"
|
|
"io/ioutil"
|
|
"path"
|
|
"time"
|
|
|
|
"github.com/OneOfOne/xxhash"
|
|
"github.com/buger/jsonparser"
|
|
"github.com/fsnotify/fsnotify"
|
|
)
|
|
|
|
//TODO: replace with new profile manager
|
|
var ChromeData = BrowserPaths{
|
|
BookmarkFile: "Bookmarks",
|
|
BookmarkDir: "/home/spike/.config/google-chrome-unstable/Default/",
|
|
}
|
|
|
|
var jsonNodeTypes = struct {
|
|
Folder, URL string
|
|
}{"folder", "url"}
|
|
|
|
var jsonNodePaths = struct {
|
|
Type, Children, URL string
|
|
}{"type", "children", "url"}
|
|
|
|
type ChromeBrowser struct {
|
|
BaseBrowser //embedding
|
|
}
|
|
|
|
type ParseChildFunc func([]byte, jsonparser.ValueType, int, error)
|
|
type RecursiveParseFunc func([]byte, []byte, jsonparser.ValueType, int) error
|
|
|
|
type RawNode struct {
|
|
name []byte
|
|
nType []byte
|
|
url []byte
|
|
children []byte
|
|
childrenType jsonparser.ValueType
|
|
}
|
|
|
|
func (rawNode *RawNode) parseItems(nodeData []byte) {
|
|
|
|
// Paths to lookup in node payload
|
|
paths := [][]string{
|
|
[]string{"type"},
|
|
[]string{"name"}, // Title of page
|
|
[]string{"url"},
|
|
[]string{"children"},
|
|
}
|
|
|
|
jsonparser.EachKey(nodeData, func(idx int, value []byte, vt jsonparser.ValueType, err error) {
|
|
switch idx {
|
|
case 0:
|
|
rawNode.nType = value
|
|
//currentNode.Type = s(value)
|
|
|
|
case 1: // name or title
|
|
//currentNode.Name = s(value)
|
|
rawNode.name = value
|
|
case 2:
|
|
//currentNode.URL = s(value)
|
|
rawNode.url = value
|
|
case 3:
|
|
rawNode.children, rawNode.childrenType = value, vt
|
|
}
|
|
}, paths...)
|
|
}
|
|
|
|
// Returns *tree.Node from *RawNode
|
|
func (rawNode *RawNode) getNode() *tree.Node {
|
|
node := new(tree.Node)
|
|
node.Type = utils.S(rawNode.nType)
|
|
node.Name = utils.S(rawNode.name)
|
|
|
|
return node
|
|
}
|
|
|
|
func NewChromeBrowser() IBrowser {
|
|
browser := new(ChromeBrowser)
|
|
browser.name = "chrome"
|
|
browser.bType = TChrome
|
|
browser.baseDir = ChromeData.BookmarkDir
|
|
browser.bkFile = ChromeData.BookmarkFile
|
|
browser.Stats = new(parsing.Stats)
|
|
browser.NodeTree = &tree.Node{Name: "root", Parent: nil, Type: "root"}
|
|
browser.useFileWatcher = true
|
|
|
|
// Across jobs buffer
|
|
browser.InitBuffer()
|
|
|
|
// Create watch objects, we will watch the basedir for create events
|
|
watchedEvents := []fsnotify.Op{fsnotify.Create}
|
|
w := &Watch{
|
|
Path: browser.baseDir,
|
|
EventTypes: watchedEvents,
|
|
EventNames: []string{path.Join(browser.baseDir, browser.bkFile)},
|
|
ResetWatch: true,
|
|
}
|
|
browser.SetupFileWatcher(w)
|
|
|
|
return browser
|
|
}
|
|
|
|
func (bw *ChromeBrowser) Watch() bool {
|
|
if !bw.isWatching {
|
|
go watch.WatcherThread(bw)
|
|
bw.isWatching = true
|
|
log.Infof("<%s> Watching %s", bw.name, bw.GetBookmarksPath())
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func (bw *ChromeBrowser) Load() {
|
|
|
|
// BaseBrowser load method
|
|
bw.BaseBrowser.Load()
|
|
bw.Run()
|
|
}
|
|
|
|
func (bw *ChromeBrowser) Run() {
|
|
startRun := time.Now()
|
|
|
|
// Rebuild node tree
|
|
bw.RebuildNodeTree()
|
|
|
|
// Load bookmark file
|
|
bookmarkPath := path.Join(bw.baseDir, bw.bkFile)
|
|
f, err := ioutil.ReadFile(bookmarkPath)
|
|
if err != nil {
|
|
log.Critical(err)
|
|
}
|
|
|
|
var parseChildren ParseChildFunc
|
|
var jsonParseRecursive RecursiveParseFunc
|
|
|
|
parseChildren = func(childVal []byte, dataType jsonparser.ValueType, offset int, err error) {
|
|
if err != nil {
|
|
log.Panic(err)
|
|
}
|
|
|
|
jsonParseRecursive(nil, childVal, dataType, offset)
|
|
}
|
|
|
|
// Needed to store the parent of each child node
|
|
var parentNodes []*tree.Node
|
|
|
|
jsonParseRoots := func(key []byte, node []byte, dataType jsonparser.ValueType, offset int) error {
|
|
|
|
// If node type is string ignore (needed for sync_transaction_version)
|
|
if dataType == jsonparser.String {
|
|
return nil
|
|
}
|
|
|
|
bw.Stats.CurrentNodeCount++
|
|
rawNode := new(RawNode)
|
|
rawNode.parseItems(node)
|
|
//log.Debugf("Parsing root folder %s", rawNode.name)
|
|
|
|
currentNode := rawNode.getNode()
|
|
|
|
// Process this node as parent node later
|
|
parentNodes = append(parentNodes, currentNode)
|
|
|
|
// add the root node as parent to this node
|
|
currentNode.Parent = bw.NodeTree
|
|
|
|
// Add this root node as a child of the root node
|
|
bw.NodeTree.Children = append(bw.NodeTree.Children, currentNode)
|
|
|
|
// Call recursive parsing of this node which must
|
|
// a root folder node
|
|
jsonparser.ArrayEach(node, parseChildren, jsonNodePaths.Children)
|
|
|
|
// Finished parsing this root, it is not anymore a parent
|
|
_, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1]
|
|
|
|
//log.Debugf("Parsed root %s folder", rawNode.name)
|
|
|
|
return nil
|
|
}
|
|
|
|
// Main recursive parsing function that parses underneath
|
|
// each root folder
|
|
jsonParseRecursive = func(key []byte, node []byte, dataType jsonparser.ValueType, offset int) error {
|
|
|
|
// If node type is string ignore (needed for sync_transaction_version)
|
|
if dataType == jsonparser.String {
|
|
return nil
|
|
}
|
|
|
|
bw.Stats.CurrentNodeCount++
|
|
|
|
rawNode := new(RawNode)
|
|
rawNode.parseItems(node)
|
|
|
|
currentNode := rawNode.getNode()
|
|
//log.Debugf("parsing node %s", currentNode.Name)
|
|
|
|
// if parents array is not empty
|
|
if len(parentNodes) != 0 {
|
|
parent := parentNodes[len(parentNodes)-1]
|
|
//log.Debugf("Adding current node to parent %s", parent.Name)
|
|
|
|
// Add current node to closest parent
|
|
currentNode.Parent = parent
|
|
|
|
// Add current node as child to parent
|
|
currentNode.Parent.Children = append(currentNode.Parent.Children, currentNode)
|
|
}
|
|
|
|
// if node is a folder with children
|
|
if rawNode.childrenType == jsonparser.Array && len(rawNode.children) > 2 { // if len(children) > len("[]")
|
|
|
|
//log.Debugf("Started folder %s", rawNode.name)
|
|
parentNodes = append(parentNodes, currentNode)
|
|
|
|
// Process recursively all child nodes of this folder node
|
|
jsonparser.ArrayEach(node, parseChildren, jsonNodePaths.Children)
|
|
|
|
//log.Debugf("Finished folder %s", rawNode.name)
|
|
_, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1]
|
|
|
|
}
|
|
|
|
// if node is url(leaf), handle the url
|
|
if utils.S(rawNode.nType) == jsonNodeTypes.URL {
|
|
|
|
currentNode.URL = utils.S(rawNode.url)
|
|
bw.Stats.CurrentUrlCount++
|
|
// Check if url-node already in index
|
|
var nodeVal *tree.Node
|
|
iVal, found := bw.URLIndex.Get(currentNode.URL)
|
|
|
|
nameHash := xxhash.ChecksumString64(currentNode.Name)
|
|
// If node url not in index, add it to index
|
|
if !found {
|
|
//log.Debugf("Not found")
|
|
|
|
// store hash(name)
|
|
currentNode.NameHash = nameHash
|
|
|
|
// The value in the index will be a
|
|
// pointer to currentNode
|
|
//log.Debugf("Inserting url %s to index", nodeURL)
|
|
bw.URLIndex.Insert(currentNode.URL, currentNode)
|
|
|
|
// Run tag parsing hooks
|
|
bw.RunParseHooks(currentNode)
|
|
|
|
// If we find the node already in index
|
|
// we check if the hash(name) changed meaning
|
|
// the data changed
|
|
} else {
|
|
//log.Debugf("URL Found in index")
|
|
nodeVal = iVal.(*tree.Node)
|
|
|
|
// hash(name) is different meaning new commands/tags could
|
|
// be added, we need to process the parsing hoos
|
|
if nodeVal.NameHash != nameHash {
|
|
//log.Debugf("URL name changed !")
|
|
|
|
// Run parse hooks on node
|
|
bw.RunParseHooks(currentNode)
|
|
}
|
|
|
|
// Else we do nothing, the node will not
|
|
// change
|
|
}
|
|
|
|
//If parent is folder, add it as tag and add current node as child
|
|
//And add this link as child
|
|
if currentNode.Parent.Type == jsonNodeTypes.Folder {
|
|
//log.Debug("Parent is folder, parsing as tag ...")
|
|
currentNode.Tags = append(currentNode.Tags, currentNode.Parent.Name)
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
rootsData, _, _, _ := jsonparser.Get(f, "roots")
|
|
|
|
// Start a new node tree building job
|
|
jsonparser.ObjectEach(rootsData, jsonParseRoots)
|
|
bw.Stats.LastFullTreeParseTime = time.Since(startRun)
|
|
log.Debugf("<%s> parsed tree in %s", bw.name, bw.Stats.LastFullTreeParseTime)
|
|
// Finished node tree building job
|
|
|
|
// Debug walk tree
|
|
//go PrintTree(bw.NodeTree)
|
|
|
|
// Reset the index to represent the nodetree
|
|
bw.RebuildIndex()
|
|
|
|
// Finished parsing
|
|
log.Debugf("<%s> parsed %d bookmarks and %d nodes", bw.name, bw.Stats.CurrentUrlCount, bw.Stats.CurrentNodeCount)
|
|
// Reset parser counter
|
|
bw.ResetStats()
|
|
|
|
//Add nodeTree to Cache
|
|
//log.Debugf("<%s> buffer content", bw.name)
|
|
//bw.BufferDB.Print()
|
|
|
|
log.Debugf("<%s> syncing to buffer", bw.name)
|
|
database.SyncTreeToBuffer(bw.NodeTree, bw.BufferDB)
|
|
log.Debugf("<%s> tree synced to buffer", bw.name)
|
|
|
|
//bw.BufferDB.Print()
|
|
|
|
// cacheDB represents bookmarks across all browsers
|
|
// From browsers it should support: add/update
|
|
// Delete method should only be possible through admin interface
|
|
// We could have an @ignore command to ignore a bookmark
|
|
|
|
// URLIndex is a hashmap index of all URLS representing current state
|
|
// of the browser
|
|
|
|
// nodeTree is current state of the browser as tree
|
|
|
|
// Buffer is the current state of the browser represetned by
|
|
// URLIndex and nodeTree
|
|
|
|
// If cacheDB is empty just copy buffer to cacheDB
|
|
// until local db is already populated and preloaded
|
|
//debugPrint("%d", BufferDB.Count())
|
|
if empty, err := CacheDB.IsEmpty(); empty {
|
|
if err != nil {
|
|
log.Error(err)
|
|
}
|
|
log.Info("cache empty: loading buffer to Cachedb")
|
|
|
|
bw.BufferDB.CopyTo(CacheDB)
|
|
|
|
log.Debugf("syncing <%s> to disk", CacheDB.Name)
|
|
} else {
|
|
bw.BufferDB.SyncTo(CacheDB)
|
|
}
|
|
|
|
go CacheDB.SyncToDisk(database.GetDBFullPath())
|
|
bw.Stats.LastWatchRunTime = time.Since(startRun)
|
|
}
|