debug and print trees, wip firefox
This commit is contained in:
parent
09dea0b655
commit
dab486fefd
2
Makefile
2
Makefile
@ -3,7 +3,7 @@
|
|||||||
TARGET=gomark
|
TARGET=gomark
|
||||||
|
|
||||||
|
|
||||||
all: build docs
|
all: build
|
||||||
|
|
||||||
run:
|
run:
|
||||||
@go run *.go
|
@go run *.go
|
||||||
|
35
chrome.go
35
chrome.go
@ -81,7 +81,7 @@ func NewChromeBrowser() IBrowser {
|
|||||||
browser.baseDir = ChromeData.BookmarkDir
|
browser.baseDir = ChromeData.BookmarkDir
|
||||||
browser.bkFile = ChromeData.BookmarkFile
|
browser.bkFile = ChromeData.BookmarkFile
|
||||||
browser.Stats = &ParserStats{}
|
browser.Stats = &ParserStats{}
|
||||||
browser.NodeTree = &Node{Name: "root", Parent: nil}
|
browser.NodeTree = &Node{Name: "root", Parent: nil, Type: "root"}
|
||||||
|
|
||||||
// Across jobs buffer
|
// Across jobs buffer
|
||||||
browser.InitBuffer()
|
browser.InitBuffer()
|
||||||
@ -112,7 +112,7 @@ func (bw *ChromeBrowser) Load() {
|
|||||||
func (bw *ChromeBrowser) Run() {
|
func (bw *ChromeBrowser) Run() {
|
||||||
|
|
||||||
// Rebuild node tree
|
// Rebuild node tree
|
||||||
bw.NodeTree = &Node{Name: "root", Parent: nil}
|
bw.NodeTree = &Node{Name: "root", Parent: nil, Type: "root"}
|
||||||
|
|
||||||
// Load bookmark file
|
// Load bookmark file
|
||||||
bookmarkPath := path.Join(bw.baseDir, bw.bkFile)
|
bookmarkPath := path.Join(bw.baseDir, bw.bkFile)
|
||||||
@ -144,7 +144,7 @@ func (bw *ChromeBrowser) Run() {
|
|||||||
|
|
||||||
rawNode := new(RawNode)
|
rawNode := new(RawNode)
|
||||||
rawNode.parseItems(node)
|
rawNode.parseItems(node)
|
||||||
log.Debugf("Parsing root folder %s", rawNode.name)
|
//log.Debugf("Parsing root folder %s", rawNode.name)
|
||||||
|
|
||||||
currentNode := rawNode.getNode()
|
currentNode := rawNode.getNode()
|
||||||
|
|
||||||
@ -164,7 +164,7 @@ func (bw *ChromeBrowser) Run() {
|
|||||||
// Finished parsing this root, it is not anymore a parent
|
// Finished parsing this root, it is not anymore a parent
|
||||||
_, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1]
|
_, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1]
|
||||||
|
|
||||||
log.Debugf("Parsed root %s folder", rawNode.name)
|
//log.Debugf("Parsed root %s folder", rawNode.name)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@ -184,12 +184,12 @@ func (bw *ChromeBrowser) Run() {
|
|||||||
rawNode.parseItems(node)
|
rawNode.parseItems(node)
|
||||||
|
|
||||||
currentNode := rawNode.getNode()
|
currentNode := rawNode.getNode()
|
||||||
log.Debugf("parsing node %s", currentNode.Name)
|
//log.Debugf("parsing node %s", currentNode.Name)
|
||||||
|
|
||||||
// if parents array is not empty
|
// if parents array is not empty
|
||||||
if len(parentNodes) != 0 {
|
if len(parentNodes) != 0 {
|
||||||
parent := parentNodes[len(parentNodes)-1]
|
parent := parentNodes[len(parentNodes)-1]
|
||||||
log.Debugf("Adding current node to parent %s", parent.Name)
|
//log.Debugf("Adding current node to parent %s", parent.Name)
|
||||||
|
|
||||||
// Add current node to closest parent
|
// Add current node to closest parent
|
||||||
currentNode.Parent = parent
|
currentNode.Parent = parent
|
||||||
@ -201,13 +201,13 @@ func (bw *ChromeBrowser) Run() {
|
|||||||
// if node is a folder with children
|
// if node is a folder with children
|
||||||
if rawNode.childrenType == jsonparser.Array && len(rawNode.children) > 2 { // if len(children) > len("[]")
|
if rawNode.childrenType == jsonparser.Array && len(rawNode.children) > 2 { // if len(children) > len("[]")
|
||||||
|
|
||||||
log.Debugf("Started folder %s", rawNode.name)
|
//log.Debugf("Started folder %s", rawNode.name)
|
||||||
parentNodes = append(parentNodes, currentNode)
|
parentNodes = append(parentNodes, currentNode)
|
||||||
|
|
||||||
// Process recursively all child nodes of this folder node
|
// Process recursively all child nodes of this folder node
|
||||||
jsonparser.ArrayEach(node, parseChildren, jsonNodePaths.Children)
|
jsonparser.ArrayEach(node, parseChildren, jsonNodePaths.Children)
|
||||||
|
|
||||||
log.Debugf("Finished folder %s", rawNode.name)
|
//log.Debugf("Finished folder %s", rawNode.name)
|
||||||
_, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1]
|
_, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1]
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -243,7 +243,7 @@ func (bw *ChromeBrowser) Run() {
|
|||||||
// we check if the hash(name) changed meaning
|
// we check if the hash(name) changed meaning
|
||||||
// the data changed
|
// the data changed
|
||||||
} else {
|
} else {
|
||||||
log.Debugf("URL Found in index")
|
//log.Debugf("URL Found in index")
|
||||||
nodeVal = iVal.(*Node)
|
nodeVal = iVal.(*Node)
|
||||||
|
|
||||||
// hash(name) is different meaning new commands/tags could
|
// hash(name) is different meaning new commands/tags could
|
||||||
@ -266,7 +266,7 @@ func (bw *ChromeBrowser) Run() {
|
|||||||
//If parent is folder, add it as tag and add current node as child
|
//If parent is folder, add it as tag and add current node as child
|
||||||
//And add this link as child
|
//And add this link as child
|
||||||
if currentNode.Parent.Type == jsonNodeTypes.Folder {
|
if currentNode.Parent.Type == jsonNodeTypes.Folder {
|
||||||
log.Debug("Parent is folder, parsing as tag ...")
|
//log.Debug("Parent is folder, parsing as tag ...")
|
||||||
currentNode.Tags = append(currentNode.Tags, currentNode.Parent.Name)
|
currentNode.Tags = append(currentNode.Tags, currentNode.Parent.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -281,11 +281,12 @@ func (bw *ChromeBrowser) Run() {
|
|||||||
start := time.Now()
|
start := time.Now()
|
||||||
jsonparser.ObjectEach(rootsData, jsonParseRoots)
|
jsonparser.ObjectEach(rootsData, jsonParseRoots)
|
||||||
bw.Stats.lastParseTime = time.Since(start)
|
bw.Stats.lastParseTime = time.Since(start)
|
||||||
log.Debugf("Parsed tree in %s", bw.Stats.lastParseTime)
|
log.Debugf("<%s> parsed tree in %s", bw.name, bw.Stats.lastParseTime)
|
||||||
// Finished node tree building job
|
// Finished node tree building job
|
||||||
|
|
||||||
// Debug walk tree
|
// Debug walk tree
|
||||||
//go WalkNode(bw.nodeTree)
|
//go WalkNode(bw.NodeTree)
|
||||||
|
go PrintTree(bw.NodeTree)
|
||||||
|
|
||||||
// Reset the index to represent the nodetree
|
// Reset the index to represent the nodetree
|
||||||
bw.RebuildIndex()
|
bw.RebuildIndex()
|
||||||
@ -297,14 +298,14 @@ func (bw *ChromeBrowser) Run() {
|
|||||||
bw.ResetStats()
|
bw.ResetStats()
|
||||||
|
|
||||||
//Add nodeTree to Cache
|
//Add nodeTree to Cache
|
||||||
log.Debugf("Buffer content")
|
//log.Debugf("<%s> buffer content", bw.name)
|
||||||
bw.BufferDB.Print()
|
//bw.BufferDB.Print()
|
||||||
|
|
||||||
log.Debugf("syncing to buffer")
|
log.Debugf("<%s> syncing to buffer", bw.name)
|
||||||
syncTreeToBuffer(bw.NodeTree, bw.BufferDB)
|
syncTreeToBuffer(bw.NodeTree, bw.BufferDB)
|
||||||
log.Debugf("Tree synced to buffer")
|
log.Debugf("<%s> tree synced to buffer", bw.name)
|
||||||
|
|
||||||
bw.BufferDB.Print()
|
//bw.BufferDB.Print()
|
||||||
|
|
||||||
// cacheDB represents bookmarks across all browsers
|
// cacheDB represents bookmarks across all browsers
|
||||||
// From browsers it should support: add/update
|
// From browsers it should support: add/update
|
||||||
|
30
firefox.go
30
firefox.go
@ -2,8 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"path"
|
"path"
|
||||||
|
"time"
|
||||||
"github.com/OneOfOne/xxhash"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var Firefox = BrowserPaths{
|
var Firefox = BrowserPaths{
|
||||||
@ -34,7 +33,7 @@ func NewFFBrowser() IBrowser {
|
|||||||
browser.baseDir = Firefox.BookmarkDir
|
browser.baseDir = Firefox.BookmarkDir
|
||||||
browser.bkFile = Firefox.BookmarkFile
|
browser.bkFile = Firefox.BookmarkFile
|
||||||
browser.Stats = &ParserStats{}
|
browser.Stats = &ParserStats{}
|
||||||
browser.NodeTree = &Node{Name: "root", Parent: nil}
|
browser.NodeTree = &Node{Name: "root", Parent: nil, Type: "root"}
|
||||||
|
|
||||||
// Across jobs buffer
|
// Across jobs buffer
|
||||||
browser.InitBuffer()
|
browser.InitBuffer()
|
||||||
@ -135,8 +134,8 @@ func getFFBookmarks(bw *FFBrowser) {
|
|||||||
urlNode.Parent = tagMap[tagId]
|
urlNode.Parent = tagMap[tagId]
|
||||||
tagMap[tagId].Children = append(tagMap[tagId].Children, urlNode)
|
tagMap[tagId].Children = append(tagMap[tagId].Children, urlNode)
|
||||||
|
|
||||||
// Check if url already in index
|
// Check if url already in index TODO: should be done in new pass
|
||||||
iVal, found := bw.URLIndex.Get(urlNode.URL)
|
//iVal, found := bw.URLIndex.Get(urlNode.URL)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* The fields where tags may change are hashed together
|
* The fields where tags may change are hashed together
|
||||||
@ -145,7 +144,20 @@ func getFFBookmarks(bw *FFBrowser) {
|
|||||||
* (tags) for this url then hash their concatenation
|
* (tags) for this url then hash their concatenation
|
||||||
*/
|
*/
|
||||||
|
|
||||||
nameHash := xxhash.ChecksumString64(urlNode.Name)
|
//nameHash := xxhash.ChecksumString64(urlNode.Name)
|
||||||
|
// TODO: No guarantee we finished gathering tags !!
|
||||||
|
// We should check again against index in a new pass
|
||||||
|
// This pass needs to finish until we have full count
|
||||||
|
// of tags for each bookmark
|
||||||
|
//parents := urlNode.GetParentTags()
|
||||||
|
//if len(parents) > 4 {
|
||||||
|
//tags := make([]string, 0)
|
||||||
|
//for _, v := range parents {
|
||||||
|
//tags = append(tags, v.Name)
|
||||||
|
//}
|
||||||
|
////log.Debugf("<%s> --> [%s]", urlNode.URL, strings.Join(tags, "|"))
|
||||||
|
|
||||||
|
//}
|
||||||
|
|
||||||
bw.Stats.currentUrlCount++
|
bw.Stats.currentUrlCount++
|
||||||
bw.Stats.currentNodeCount++
|
bw.Stats.currentNodeCount++
|
||||||
@ -202,10 +214,16 @@ func (bw *FFBrowser) Run() {
|
|||||||
bw._places = placesDB
|
bw._places = placesDB
|
||||||
|
|
||||||
// Parse bookmarks to a flat tree (for compatibility with tree system)
|
// Parse bookmarks to a flat tree (for compatibility with tree system)
|
||||||
|
start := time.Now()
|
||||||
getFFBookmarks(bw)
|
getFFBookmarks(bw)
|
||||||
|
|
||||||
// Finished parsing
|
// Finished parsing
|
||||||
|
bw.Stats.lastParseTime = time.Since(start)
|
||||||
log.Debugf("<%s> parsed %d bookmarks and %d nodes", bw.name, bw.Stats.currentUrlCount, bw.Stats.currentNodeCount)
|
log.Debugf("<%s> parsed %d bookmarks and %d nodes", bw.name, bw.Stats.currentUrlCount, bw.Stats.currentNodeCount)
|
||||||
|
log.Debugf("<%s> parsed tree in %s", bw.name, bw.Stats.lastParseTime)
|
||||||
|
|
||||||
|
//go PrintTree(bw.NodeTree)
|
||||||
|
|
||||||
bw.ResetStats()
|
bw.ResetStats()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
2
parse.go
2
parse.go
@ -43,7 +43,7 @@ func ParseTags(node *Node) {
|
|||||||
//res := regex.FindAllStringSubmatch(bk.Metadata, -1)
|
//res := regex.FindAllStringSubmatch(bk.Metadata, -1)
|
||||||
|
|
||||||
if len(node.Tags) > 0 {
|
if len(node.Tags) > 0 {
|
||||||
log.Debugf("[Title] found following tags: %s", node.Tags)
|
log.Debugf("[in title] found following tags: %s", node.Tags)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
75
tree.go
75
tree.go
@ -1,5 +1,11 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/xlab/treeprint"
|
||||||
|
)
|
||||||
|
|
||||||
type Node struct {
|
type Node struct {
|
||||||
Name string
|
Name string
|
||||||
Type string // folder, tag, url
|
Type string // folder, tag, url
|
||||||
@ -22,6 +28,75 @@ func (node *Node) GetBookmark() *Bookmark {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (node *Node) GetRoot() *Node {
|
||||||
|
nodePtr := node
|
||||||
|
|
||||||
|
for {
|
||||||
|
if nodePtr.Name == "root" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
nodePtr = nodePtr.Parent
|
||||||
|
}
|
||||||
|
|
||||||
|
return nodePtr
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns all parent tags for URL nodes
|
||||||
|
func (node *Node) GetParentTags() []*Node {
|
||||||
|
var parents []*Node
|
||||||
|
var walk func(node *Node)
|
||||||
|
var nodePtr *Node
|
||||||
|
|
||||||
|
root := node.GetRoot()
|
||||||
|
|
||||||
|
walk = func(n *Node) {
|
||||||
|
nodePtr = n
|
||||||
|
|
||||||
|
//log.Debugf("type of %s --> %s", nodePtr.Type, nodePtr.Name)
|
||||||
|
if nodePtr.Type == "url" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(nodePtr.Children) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, v := range nodePtr.Children {
|
||||||
|
if v.URL == node.URL &&
|
||||||
|
nodePtr.Type == "tag" {
|
||||||
|
parents = append(parents, nodePtr)
|
||||||
|
}
|
||||||
|
walk(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
walk(root)
|
||||||
|
return parents
|
||||||
|
}
|
||||||
|
|
||||||
|
func PrintTree(root *Node) {
|
||||||
|
var walk func(node *Node, tree treeprint.Tree)
|
||||||
|
tree := treeprint.New()
|
||||||
|
|
||||||
|
walk = func(node *Node, t treeprint.Tree) {
|
||||||
|
|
||||||
|
if len(node.Children) > 0 {
|
||||||
|
t = t.AddBranch(fmt.Sprintf("%s <%s>", node.Type, node.Name))
|
||||||
|
|
||||||
|
for _, child := range node.Children {
|
||||||
|
walk(child, t)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
t.AddNode(fmt.Sprintf("%s <%s>", node.Type, node.URL))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
walk(root, tree)
|
||||||
|
log.Debug(tree.String())
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
// Debuggin bookmark node tree
|
// Debuggin bookmark node tree
|
||||||
// TODO: Better usage of node trees
|
// TODO: Better usage of node trees
|
||||||
func WalkNode(node *Node) {
|
func WalkNode(node *Node) {
|
||||||
|
Loading…
Reference in New Issue
Block a user