From dab486fefd332adf92939b33ef98e37db558ffb1 Mon Sep 17 00:00:00 2001 From: Chakib Ben Ziane Date: Thu, 25 Oct 2018 18:09:03 +0200 Subject: [PATCH] debug and print trees, wip firefox --- Makefile | 2 +- chrome.go | 35 ++++++++++++------------- firefox.go | 30 +++++++++++++++++----- parse.go | 2 +- tree.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 6cbaa5f..f891df8 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ TARGET=gomark -all: build docs +all: build run: @go run *.go diff --git a/chrome.go b/chrome.go index 8f9e4d5..0c3a66b 100644 --- a/chrome.go +++ b/chrome.go @@ -81,7 +81,7 @@ func NewChromeBrowser() IBrowser { browser.baseDir = ChromeData.BookmarkDir browser.bkFile = ChromeData.BookmarkFile browser.Stats = &ParserStats{} - browser.NodeTree = &Node{Name: "root", Parent: nil} + browser.NodeTree = &Node{Name: "root", Parent: nil, Type: "root"} // Across jobs buffer browser.InitBuffer() @@ -112,7 +112,7 @@ func (bw *ChromeBrowser) Load() { func (bw *ChromeBrowser) Run() { // Rebuild node tree - bw.NodeTree = &Node{Name: "root", Parent: nil} + bw.NodeTree = &Node{Name: "root", Parent: nil, Type: "root"} // Load bookmark file bookmarkPath := path.Join(bw.baseDir, bw.bkFile) @@ -144,7 +144,7 @@ func (bw *ChromeBrowser) Run() { rawNode := new(RawNode) rawNode.parseItems(node) - log.Debugf("Parsing root folder %s", rawNode.name) + //log.Debugf("Parsing root folder %s", rawNode.name) currentNode := rawNode.getNode() @@ -164,7 +164,7 @@ func (bw *ChromeBrowser) Run() { // Finished parsing this root, it is not anymore a parent _, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1] - log.Debugf("Parsed root %s folder", rawNode.name) + //log.Debugf("Parsed root %s folder", rawNode.name) return nil } @@ -184,12 +184,12 @@ func (bw *ChromeBrowser) Run() { rawNode.parseItems(node) currentNode := rawNode.getNode() - log.Debugf("parsing node %s", currentNode.Name) + //log.Debugf("parsing node %s", currentNode.Name) // if parents array is not empty if len(parentNodes) != 0 { parent := parentNodes[len(parentNodes)-1] - log.Debugf("Adding current node to parent %s", parent.Name) + //log.Debugf("Adding current node to parent %s", parent.Name) // Add current node to closest parent currentNode.Parent = parent @@ -201,13 +201,13 @@ func (bw *ChromeBrowser) Run() { // if node is a folder with children if rawNode.childrenType == jsonparser.Array && len(rawNode.children) > 2 { // if len(children) > len("[]") - log.Debugf("Started folder %s", rawNode.name) + //log.Debugf("Started folder %s", rawNode.name) parentNodes = append(parentNodes, currentNode) // Process recursively all child nodes of this folder node jsonparser.ArrayEach(node, parseChildren, jsonNodePaths.Children) - log.Debugf("Finished folder %s", rawNode.name) + //log.Debugf("Finished folder %s", rawNode.name) _, parentNodes = parentNodes[len(parentNodes)-1], parentNodes[:len(parentNodes)-1] } @@ -243,7 +243,7 @@ func (bw *ChromeBrowser) Run() { // we check if the hash(name) changed meaning // the data changed } else { - log.Debugf("URL Found in index") + //log.Debugf("URL Found in index") nodeVal = iVal.(*Node) // hash(name) is different meaning new commands/tags could @@ -266,7 +266,7 @@ func (bw *ChromeBrowser) Run() { //If parent is folder, add it as tag and add current node as child //And add this link as child if currentNode.Parent.Type == jsonNodeTypes.Folder { - log.Debug("Parent is folder, parsing as tag ...") + //log.Debug("Parent is folder, parsing as tag ...") currentNode.Tags = append(currentNode.Tags, currentNode.Parent.Name) } @@ -281,11 +281,12 @@ func (bw *ChromeBrowser) Run() { start := time.Now() jsonparser.ObjectEach(rootsData, jsonParseRoots) bw.Stats.lastParseTime = time.Since(start) - log.Debugf("Parsed tree in %s", bw.Stats.lastParseTime) + log.Debugf("<%s> parsed tree in %s", bw.name, bw.Stats.lastParseTime) // Finished node tree building job // Debug walk tree - //go WalkNode(bw.nodeTree) + //go WalkNode(bw.NodeTree) + go PrintTree(bw.NodeTree) // Reset the index to represent the nodetree bw.RebuildIndex() @@ -297,14 +298,14 @@ func (bw *ChromeBrowser) Run() { bw.ResetStats() //Add nodeTree to Cache - log.Debugf("Buffer content") - bw.BufferDB.Print() + //log.Debugf("<%s> buffer content", bw.name) + //bw.BufferDB.Print() - log.Debugf("syncing to buffer") + log.Debugf("<%s> syncing to buffer", bw.name) syncTreeToBuffer(bw.NodeTree, bw.BufferDB) - log.Debugf("Tree synced to buffer") + log.Debugf("<%s> tree synced to buffer", bw.name) - bw.BufferDB.Print() + //bw.BufferDB.Print() // cacheDB represents bookmarks across all browsers // From browsers it should support: add/update diff --git a/firefox.go b/firefox.go index f4d0ae3..371ccb4 100644 --- a/firefox.go +++ b/firefox.go @@ -2,8 +2,7 @@ package main import ( "path" - - "github.com/OneOfOne/xxhash" + "time" ) var Firefox = BrowserPaths{ @@ -34,7 +33,7 @@ func NewFFBrowser() IBrowser { browser.baseDir = Firefox.BookmarkDir browser.bkFile = Firefox.BookmarkFile browser.Stats = &ParserStats{} - browser.NodeTree = &Node{Name: "root", Parent: nil} + browser.NodeTree = &Node{Name: "root", Parent: nil, Type: "root"} // Across jobs buffer browser.InitBuffer() @@ -135,8 +134,8 @@ func getFFBookmarks(bw *FFBrowser) { urlNode.Parent = tagMap[tagId] tagMap[tagId].Children = append(tagMap[tagId].Children, urlNode) - // Check if url already in index - iVal, found := bw.URLIndex.Get(urlNode.URL) + // Check if url already in index TODO: should be done in new pass + //iVal, found := bw.URLIndex.Get(urlNode.URL) /* * The fields where tags may change are hashed together @@ -145,7 +144,20 @@ func getFFBookmarks(bw *FFBrowser) { * (tags) for this url then hash their concatenation */ - nameHash := xxhash.ChecksumString64(urlNode.Name) + //nameHash := xxhash.ChecksumString64(urlNode.Name) + // TODO: No guarantee we finished gathering tags !! + // We should check again against index in a new pass + // This pass needs to finish until we have full count + // of tags for each bookmark + //parents := urlNode.GetParentTags() + //if len(parents) > 4 { + //tags := make([]string, 0) + //for _, v := range parents { + //tags = append(tags, v.Name) + //} + ////log.Debugf("<%s> --> [%s]", urlNode.URL, strings.Join(tags, "|")) + + //} bw.Stats.currentUrlCount++ bw.Stats.currentNodeCount++ @@ -202,10 +214,16 @@ func (bw *FFBrowser) Run() { bw._places = placesDB // Parse bookmarks to a flat tree (for compatibility with tree system) + start := time.Now() getFFBookmarks(bw) // Finished parsing + bw.Stats.lastParseTime = time.Since(start) log.Debugf("<%s> parsed %d bookmarks and %d nodes", bw.name, bw.Stats.currentUrlCount, bw.Stats.currentNodeCount) + log.Debugf("<%s> parsed tree in %s", bw.name, bw.Stats.lastParseTime) + + //go PrintTree(bw.NodeTree) + bw.ResetStats() } diff --git a/parse.go b/parse.go index 3214189..a26067f 100644 --- a/parse.go +++ b/parse.go @@ -43,7 +43,7 @@ func ParseTags(node *Node) { //res := regex.FindAllStringSubmatch(bk.Metadata, -1) if len(node.Tags) > 0 { - log.Debugf("[Title] found following tags: %s", node.Tags) + log.Debugf("[in title] found following tags: %s", node.Tags) } } diff --git a/tree.go b/tree.go index a4767f1..c741900 100644 --- a/tree.go +++ b/tree.go @@ -1,5 +1,11 @@ package main +import ( + "fmt" + + "github.com/xlab/treeprint" +) + type Node struct { Name string Type string // folder, tag, url @@ -22,6 +28,75 @@ func (node *Node) GetBookmark() *Bookmark { } } +func (node *Node) GetRoot() *Node { + nodePtr := node + + for { + if nodePtr.Name == "root" { + break + } + + nodePtr = nodePtr.Parent + } + + return nodePtr +} + +// Returns all parent tags for URL nodes +func (node *Node) GetParentTags() []*Node { + var parents []*Node + var walk func(node *Node) + var nodePtr *Node + + root := node.GetRoot() + + walk = func(n *Node) { + nodePtr = n + + //log.Debugf("type of %s --> %s", nodePtr.Type, nodePtr.Name) + if nodePtr.Type == "url" { + return + } + + if len(nodePtr.Children) == 0 { + return + } + + for _, v := range nodePtr.Children { + if v.URL == node.URL && + nodePtr.Type == "tag" { + parents = append(parents, nodePtr) + } + walk(v) + } + } + + walk(root) + return parents +} + +func PrintTree(root *Node) { + var walk func(node *Node, tree treeprint.Tree) + tree := treeprint.New() + + walk = func(node *Node, t treeprint.Tree) { + + if len(node.Children) > 0 { + t = t.AddBranch(fmt.Sprintf("%s <%s>", node.Type, node.Name)) + + for _, child := range node.Children { + walk(child, t) + } + } else { + t.AddNode(fmt.Sprintf("%s <%s>", node.Type, node.URL)) + } + } + + walk(root, tree) + log.Debug(tree.String()) + +} + // Debuggin bookmark node tree // TODO: Better usage of node trees func WalkNode(node *Node) {