gosuki/firefox/firefox_test.go

package firefox

import (
	"fmt"
	"os"
	"strings"
	"testing"

	"git.sp4ke.xyz/sp4ke/gomark/database"
	"git.sp4ke.xyz/sp4ke/gomark/index"
	"git.sp4ke.xyz/sp4ke/gomark/logging"
	"git.sp4ke.xyz/sp4ke/gomark/modules"
	"git.sp4ke.xyz/sp4ke/gomark/mozilla"
	"git.sp4ke.xyz/sp4ke/gomark/parsing"
	"git.sp4ke.xyz/sp4ke/gomark/profiles"
	"git.sp4ke.xyz/sp4ke/gomark/tree"
	"git.sp4ke.xyz/sp4ke/gomark/utils"
	"github.com/chenhg5/collection"
	"github.com/stretchr/testify/assert"
)

var ff Firefox

func TestMain(m *testing.M) {
	setupFirefox()

	exitVal := m.Run()
	os.Exit(exitVal)
}

func setupFirefox() {
	ff = Firefox{
		FirefoxConfig: &FirefoxConfig{
			BrowserConfig: &modules.BrowserConfig{
				Name:     "firefox",
				Type:     modules.TFirefox,
				BkFile:   mozilla.PlacesFile,
				BkDir:    "../mozilla/testdata",
				BufferDB: &database.DB{},
				URLIndex: index.NewIndex(),
				NodeTree: &tree.Node{Name: mozilla.RootName, Parent: nil, Type: tree.RootNode},
				Stats:    &parsing.Stats{},
			},
		},
		tagMap:    map[string]*tree.Node{},
		folderMap: map[sqlid]*tree.Node{},
	}
}

func runPlacesTest(name string, t *testing.T, test func(t *testing.T)) {

	bkPath, err := ff.BookmarkPath()
	if err != nil {
		t.Error(err)
	}

	ff.places, err = database.NewDB("places", bkPath, database.DBTypeFileDSN,
		FFConfig.PlacesDSN).Init()

	if err != nil {
		t.Error(err)
	}

	defer func() {
		err = ff.places.Handle.Close()
		if err != nil {
			t.Error(err)
		}
		// Run the wal_checkpoint command to clean up the WAL file
		ff.places.Handle.Exec("PRAGMA wal_checkpoint(TRUNCATE)")

	}()

	t.Run(name, test)
}

func Test_addUrlNode(t *testing.T) {

	testUrl := struct {
		url   string
		id    sqlid
		title string
		desc  string
	}{
		url:   "http://test-url.gomark",
		id:    24,
		title: "test url",
		desc:  "desc of test url",
	}

	// fetch url changes into places and bookmarks
	// for each urlId/place
	// if urlNode does not exists create it
	// if urlNode exists find fetch it
	// if urlNode exists put tag node as parent to this url

	testNewUrl := "new urlNode: url is not yet in URLIndex"

	t.Run(testNewUrl, func(t *testing.T) {
		ok, urlNode := ff.addURLNode(testUrl.url, testUrl.title, testUrl.desc)
		if !ok {
			t.Fatalf("expected %v, got %v", true, false)
		}
		if urlNode == nil {
			t.Fatal("url node was not returned", testNewUrl)
		}

		_, ok = ff.URLIndex.Get(testUrl.url)
		if !ok {
			t.Fatal("url was not added to url index")
		}

		if !utils.InList(ff.URLIndexList, testUrl.url) {
			t.Fatal("url was not added to url index list")
		}

	})

	testUrlExists := "return existing urlNode found in URLIndex"
	t.Run(testUrlExists, func(t *testing.T) {
		_, origNode := ff.addURLNode(testUrl.url, testUrl.title, testUrl.desc)
		ok, urlNode := ff.addURLNode(testUrl.url, testUrl.title, testUrl.desc)
		if ok {
			t.Fatalf("expected %v, got %v", false, true)
		}

		if urlNode == nil {
			t.Fatal("existing url node was not returned from index")
		}

		if urlNode != origNode {
			t.Fatal("existing node does not match retrieved node from url index")
		}

		_, ok = ff.URLIndex.Get(testUrl.url)
		if !ok {
			t.Fatal("url was not added to url index")
		}

		if !utils.InList(ff.URLIndexList, testUrl.url) {
			t.Fatal("url was not added to url index list")
		}

	})

}

func Test_addFolderNode(t *testing.T) {

	// Test cases
	// 1. Adding a new folder under a root mozilla folder (parent = 2,3,5,6)
	// 2. Adding a child folder
	// 3. Adding a folder that we already saw before

	t.Run("adding firefox root folder", func(t *testing.T) {
		testRootFolder := MozFolder{
			Id:     3,
			Parent: 1,
			Title:  "toolbar",
		}

		created, fNode := ff.addFolderNode(testRootFolder)

		assert.True(t, created)

		// root folder should have appropriate title
		assert.Equal(t, fNode.Name, mozilla.RootFolderNames[mozilla.ToolbarID])

		// Should be underneath root folder
		assert.Equal(t, fNode.Parent, ff.NodeTree)

	})

	t.Run("add non existing folder with no parent", func(t *testing.T) {
		testFolder := MozFolder{
			Id:     10,
			Parent: 3, // folder under the Bookmarks Toolbar
			Title:  "Programming",
		}

		folderNodeCreated, folderNode := ff.addFolderNode(testFolder)

		// we should have the following hierarchy
		// -- ROOT
		//  |-- Bookmarks Toolbar
		//   |-- Programming

		// We expect the folder was created
		assert.True(t, folderNodeCreated)

		// If we add the same folder, we should get the same node from
		// the folderMap but no new folderNode is created
		folderAdded, sameFolderNode := ff.addFolderNode(testFolder)
		assert.False(t, folderAdded)
		assert.Equal(t, sameFolderNode, folderNode)

		assert.NotNil(t, folderNode, "folder was not created")

		// Folder should not be added at the root of the tree
		assert.NotEqual(t, folderNode.Parent, ff.NodeTree, "wront parent folder")

		// Name of node should match title of scanned folder
		assert.Equal(t, folderNode.Name, testFolder.Title, "parsing folder name")

        // If we add the same folder with differnt title it should update the folder name
        renamedFolder := testFolder
        renamedFolder.Title = "Dev"
        folderAdded, renamedFolderNode := ff.addFolderNode(renamedFolder)
        assert.Equal(t, folderNode, renamedFolderNode) // same folder node
        assert.False(t, folderAdded) // folder node is not created again
        assert.NotEqual(t, folderNode.Name, testFolder.Title)

	})
}

func Test_addTagNode(t *testing.T) {
    setupFirefox()

	testTag := struct {
		tagName string
		tagType string
	}{
		tagName: "#test_tag",
		tagType: "tag",
	}

	// Should return true with the new node
	testName := "add new tag to root tree"
	t.Run(testName, func(t *testing.T) {
		ok, tagNode := ff.addTagNode(testTag.tagName)
		if !ok {
			t.Errorf("[%s] expected %v ,got %v", testName, true, false)
		}
		if tagNode == nil {
			t.Fatalf("[%s] tag node was not returned", testName)
		}

		// "tags" branch should exist

		// TagNode should be underneath "tags" branch
		if tagNode.Parent.Parent != ff.NodeTree &&
			tagNode.Name != "tags" {
			t.Errorf("[%s] wrong parent root for tag", testName)
		}
		t.Run("should be in tagMap", func(t *testing.T) {
			node, ok := ff.tagMap[testTag.tagName]
			if !ok {
				t.Error("tag node was not found in tagMap")
			}

			if node != tagNode {
				t.Error("tag node different from the one added to tagMap")
			}
		})

		t.Run("increment node count", func(t *testing.T) {
			if ff.CurrentNodeCount != 1 {
				t.Errorf("wrong node count")
			}
		})
	})

	// This should return false with the existing node and not add a new one
	testName = "add existing tag to root tree"
	t.Run(testName, func(t *testing.T) {
		ff.addTagNode(testTag.tagName)
		ok, tagNode := ff.addTagNode(testTag.tagName)
		if tagNode == nil {
			t.Fatalf("[%s] tag node was not returned", testName)
		}
		if tagNode.Parent.Name != TagsBranchName {
			t.Errorf("[%s] wrong parent root for tag", testName)
		}
		if ok {
			t.Errorf("[%s] expected %v ,got %v", testName, false, true)
		}
	})
}

func Test_PlaceBookmarkTimeParsing(t *testing.T) {
	assert := assert.New(t)
	pb := mozilla.MergedPlaceBookmark{
		BkLastModified: 1663878015759000,
	}

	res := pb.Datetime().Format("2006-01-02 15:04:05.000000")
	assert.Equal(res, "2022-09-22 20:20:15.759000", "wrong time in scanned bookmark")
}

func findTagsInNodeTree(urlNode *tree.Node,
                        tags []string, // tags to find in tagMap
                        tagMap map[string]*tree.Node) (bool ,error) {
	var foundTagNodeForUrl bool
	for _, tagName := range tags {
		tagNode, tagNodeExists := ff.tagMap[tagName]
		if !tagNodeExists {
			return false, fmt.Errorf("missing tag <%s>", tagName)
		}
		// Check that the URL node is a direct child of the tag node
		if urlNode.DirectChildOf(tagNode) {
			foundTagNodeForUrl = true
		}
	}

    return foundTagNodeForUrl, nil
}

// TODO!: integration test loading firefox bookmarks
func Test_scanBookmarks(t *testing.T) {
	logging.SetMode(-1)

	// expected data from testdata/places.sqlite
	data := struct {
		tags    []string
		folders []string // list of tags

		bookmarkTags map[string][]string // list of folder names

	}{ // list of urls which are bookmarked
		tags: []string{"golang", "programming", "rust"},

		folders: []string{
			"menu",    // Bookmarks Menu
			"toolbar", // Bookmarks Toolbar
			"tags",    // Tags Virtual Folder
			"unfiled", // Other Bookmarks
			"mobile",  // Mobile Bookmarks
			"cooking",
			"Travel",
			"indian",
			"GomarkMenu",
		},

		bookmarkTags: map[string][]string{
			"https://based.cooking/":                              {"based"},
			"https://go.dev/":                                     {"golang", "programming"},
			"https://www.rust-lang.org/":                          {"programming", "rust", "systems"},
			"https://www.tasteofhome.com/article/indian-cooking/": {},
			"http://gomark.io/":                                   {"gomark"},
			"https://www.budapestinfo.hu/":                        {"budapest"},
			"https://www.fsf.org/":                                {"libre"},
		},
	}

	t.Log("loading firefox bookmarks")

	// First make sure bookmarks are scaned then verify they are loaded
	// in CacheDB

	runPlacesTest("find", t, func(t *testing.T) {
		bookmarks, err := ff.scanBookmarks()
		if err != nil {
			t.Error(err)
		}

		// 1- find all tags defined by user
		t.Run("all urls", func(t *testing.T) {
			var urls []string
			for _, bk := range bookmarks {
				urls = utils.Extends(urls, bk.Url)
			}

			var testUrls []string
			for url, _ := range data.bookmarkTags {
				testUrls = append(testUrls, url)
			}
			testUrls = collection.Collect(testUrls).Unique().ToStringArray()

			assert.ElementsMatch(t, urls, testUrls)


		})

		/*
		   2.find all folders
		*/
		t.Run("all folders", func(t *testing.T) {
			var folders []string
			for _, bk := range bookmarks {
				folderS := strings.Split(bk.Folders, ",")
				for _, f := range folderS {
					folders = utils.Extends(folders, f)
				}
			}
			assert.ElementsMatch(t, folders, data.folders)
		})

		/*
		   3. find all url bookmarks with their corresponding tags
		   - should get any user added bookmark (id > 12)
		*/
		t.Run("all tags", func(t *testing.T) {
			bkTags := map[string][]string{}

			for _, bk := range bookmarks {
				bkTags[bk.Url] = collection.Collect(strings.Split(bk.Tags, ",")).
					Unique().Filter(func(item, val interface{}) bool {
					// Filter out empty ("") strings
					if v, ok := val.(string); ok {
						if v == "" {
							return false
						}
					}
					return true
				}).ToStringArray()
			}

			assert.Equal(t, data.bookmarkTags, bkTags)
			// t.Error("urls with their matching tags")

			t.Run("should find all bookmarks that have tags AND within folders", func(t *testing.T) {
				for _, bk := range bookmarks {
					if bk.Url == "https://www.fsf.org/" {
						// should have `libre` tag and `Mobile Bookmarks` folder
						assert.Equal(t, bk.ParentFolder, "mobile")
					}
				}
			})

		})
	})

	runPlacesTest("load bookmarks in node tree", t, func(t *testing.T) {
		bookmarks, err := ff.scanBookmarks()
		if err != nil {
			t.Error(err)
		}
        ff.loadBookmarksToTree(bookmarks)

		t.Run("find every url in the node tree", func(t *testing.T) {

			for _, bk := range bookmarks {
				node, exists := ff.URLIndex.Get(bk.Url)
				assert.True(t, exists, "url missing in URLIndex")

				assert.True(t, tree.FindNode(node.(*tree.Node), ff.NodeTree), "url node missing from tree")
			}
		})

		t.Run("url node is child of the right tag nodes", func(t *testing.T) {
			// Every URL node should be a child of the right tag node

			// Go through each tag node
			for _, bk := range bookmarks {

				urlNode, urlNodeExists := ff.URLIndex.Get(bk.Url)
				assert.True(t, urlNodeExists, "url missing in URLIndex")

				// only check bookmarks with tags
				if len(bk.Tags) == 0 {
					continue
				}

                tags := strings.Split(bk.Tags, ",")
                foundTagNodeForUrl, err := findTagsInNodeTree(urlNode.(*tree.Node),
                                                              tags, ff.tagMap)
                if err != nil {
                  t.Error(err)
                }
                

				assert.True(t, foundTagNodeForUrl)

			}
		})

		t.Run("url underneath the right folders", func(t *testing.T) {
			for _, bk := range bookmarks {
				// folder, folderScanned := ff.folderScanMap[bk.ParentId]
				//  assert.True(t, folderScanned)

				// Get the folder from tree node
				folderNode, folderExists := ff.folderMap[bk.ParentId]
				assert.True(t, folderExists)

				urlNode, exists := ff.URLIndex.Get(bk.Url)
				assert.True(t, exists, "url missing in URLIndex")

				// check that url node has the right parent folder node

				// If Parent is nil, it means no folder was assigned to this url node
				parentFolder := bk.ParentFolder
				switch parentFolder {
				case "unfiled":
					parentFolder = mozilla.RootFolderNames[mozilla.OtherID]
				case "mobile":
					parentFolder = mozilla.RootFolderNames[mozilla.MobileID]
				}
				if urlNode.(*tree.Node).Parent != nil {
					assert.Equal(t, parentFolder, urlNode.(*tree.Node).Parent.Name, 
						"wrong folder for <%s>", bk.Url)
				}

				assert.True(t, urlNode.(*tree.Node).DirectChildOf(folderNode),
					"missing folder for %s", bk.Url)

			}
		})
	})
}

func Test_scanFolders(t *testing.T) {

	folders := []string{
		"menu",    // Bookmarks Menu
		"toolbar", // Bookmarks Toolbar
		"tags",    // Tags Virtual Folder
		"unfiled", // Other Bookmarks
		"mobile",  // Mobile Bookmarks
		"Mozilla Firefox",
		"cooking",
		"Travel",
		"indian",
		"GomarkMenu",
	}

	runPlacesTest("scan all folders", t, func(t *testing.T) {

		// query all folders
		scannedFolders, err := ff.scanFolders(0)
		if err != nil {
			t.Error(err)
		}

		// test that we loaded all folders
		folderS := []string{}
		for _, f := range scannedFolders {
			folderS = utils.Extends(folderS, f.Title)
		}
		assert.ElementsMatch(t, folders, folderS)

		// testing the tree

		// folderMap should have 9 entries (id=4 is reserved for tags)
		assert.Equal(t, len(ff.folderMap), 9, "not all nodes present in folderMap")

		// test that folders are loaded into tree
		// All folders can reach the root ancestor
		for _, f := range ff.folderMap {
			assert.Equal(t, ff.NodeTree, tree.Ancestor(f), "all folders attached to root")

			//every folder in folderMap has a corresponding node in the tree
			assert.True(t, tree.FindNode(f, ff.NodeTree), "folder nodes are attached to tree")
		}

	})

}

func Test_FindModifiedBookmarks(t *testing.T) {
	//NOTE: use a separate test places db that includes changes vs the main test db
	// Test scenarios
	// 1. Modify an existing bookmark
	//  a. Add / Remove tag ( find new tags )
	//  b. Move to folder ( find new folder)
	//  TODO: c. DELETE bookmark
	// 2. Find new bookmarks
	// 2. Find new created tags
	// 3. Find new created folders (even if empty)

    type bkTestData struct {
        tags []string
        folders []string
    }

    modifiedBookmarks := map[string]bkTestData{
        "https://go.dev/": {
            tags:    []string{"language"},
            folders: []string{mozilla.RootFolderNames[mozilla.OtherID]}, // unfiled folder
        },
    }


    newBookmarks := map[string]bkTestData{
        "https://bitcoinwhitepaper.co/": {
        tags:    []string{"bitcoin"},
        folders: []string{
            "Cryptocurrencies",
            mozilla.RootFolderNames[mozilla.OtherID],
            mozilla.RootFolderNames[mozilla.ToolbarID],
        },

        },
        "https://lightning.network/": {
        tags:    []string{"bitcoin", "lightning"},
        folders: []string{mozilla.RootFolderNames[mozilla.OtherID]},
        },
    }

    newFolders := []string{"Cryptocurrencies", "NewFolder"}

    //TODO!: modified folders

	// Setup the appropriate test db
	ff.BkFile = "places-modified.sqlite"

	runPlacesTest("loading changes to node tree", t, func(t *testing.T) {
		scanModifiedSince := 1672688367910000
		bookmarks, err := ff.scanModifiedBookmarks(int64(scanModifiedSince))
		if err != nil {
			t.Error(err)
		}
        ff.loadBookmarksToTree(bookmarks)

        t.Run("modified bookmarks", func(t *testing.T){
            // test that each modified bookmark is is loaded
            // in the node tree
            for modUrl, modBk := range modifiedBookmarks {
                node, exists := ff.URLIndex.Get(modUrl)      
				assert.True(t, exists, "url missing in URLIndex")
                urlNode := node.(*tree.Node)

				assert.True(t, tree.FindNode(node.(*tree.Node), ff.NodeTree), "url node missing from tree")


                // matching tag nodes
                found, err := findTagsInNodeTree(node.(*tree.Node), modBk.tags, ff.tagMap)
                if err != nil {
                  t.Error(err)
                }
                
                assert.True(t, found, "missing tag")


                // matching folders
                parentFolders := tree.FindParents(ff.NodeTree, urlNode, tree.FolderNode)
                pFolderNames := utils.Map(func(node *tree.Node) string{
                    return node.Name
                }, parentFolders)

                assert.ElementsMatch(t, pFolderNames, modBk.folders)


            }
        })

        t.Run("new bookmarks", func(t *testing.T){
            for newUrl, newBk := range newBookmarks {

                node, exists := ff.URLIndex.Get(newUrl)      
                urlNode := node.(*tree.Node)
				assert.True(t, exists, "url missing in URLIndex")
				assert.True(t, tree.FindNode(urlNode, ff.NodeTree), "url node missing from tree")

                // matching tag nodes
                found, err := findTagsInNodeTree(urlNode, newBk.tags, ff.tagMap)
                if err != nil {
                  t.Error(err)
                }

                assert.True(t, found, "missing tag")


                parentFolders := tree.FindParents(ff.NodeTree, urlNode, tree.FolderNode)
                pFolderNames := utils.Map(func(node *tree.Node) string{
                    return node.Name
                }, parentFolders)

                assert.ElementsMatch(t, pFolderNames, newBk.folders, fmt.Sprintf("mismatch folder for <%s>", urlNode.URL))
            }

        })

        t.Run("find new folders", func(t *testing.T){
            // Make sure the new folders exist in the folder node map
            var folderNodes []*tree.Node
            for _, fNode := range ff.folderMap {
                folderNodes = append(folderNodes, fNode)
            }

            // Get all folder names in tree
            folderNames := utils.Map(func(node *tree.Node) string{
                return node.Name
            }, folderNodes)

            assert.Subset(t, folderNames, newFolders)
        })
	})
}

func Test_FindModifiedFolders(t *testing.T) {
   t.Error("modified folder names should change the corresponding bookmark tags") 
}


func TestBrowserImplProfileManager(t *testing.T) {
	assert.Implements(t, (*profiles.ProfileManager)(nil), NewFirefox())
}