gosuki/firefox.go

package main

import (
	"database/sql"
	"fmt"
	"gomark/database"
	"gomark/parsing"
	"gomark/tools"
	"gomark/watch"
	"path"
	"time"

	"github.com/fsnotify/fsnotify"
)

var Firefox = BrowserPaths{
	BookmarkFile: "places.sqlite",
	BookmarkDir:  "/home/spike/.mozilla/firefox/p1rrgord.default/",
}

const (
	MozMinJobInterval = 500 * time.Millisecond
)

type FFBrowser struct {
	BaseBrowser //embedding
	places      *DB
	// TODO: Use URLIndex instead
	URLIndexList []string  // All elements stored in URLIndex
	qChanges     *sql.Stmt // Last changes query
	lastRunTime  time.Time
}

type FFBkType int

const (
	_ = iota
	BkTypeURL
	BkTypeTagFolder
)

type FFBookmarkParent int

const (
	_ = iota
	ffBkRoot
	ffBkMenu
	ffBkToolbar
	ffBkTags
	ffBkOther
	ffBkMobile
)

type FFTag struct {
	id    int
	title string
}

type FFBookmark struct {
	id    int
	title string
	url   string
	btype FFBkType
}

func FFPlacesUpdateHook(op int, db string, table string, rowid int64) {
	fflog.Debug(op)
}

func NewFFBrowser() IBrowser {
	browser := new(FFBrowser)
	browser.name = "firefox"
	browser.bType = TFirefox
	browser.baseDir = Firefox.BookmarkDir
	browser.bkFile = Firefox.BookmarkFile
	browser.useFileWatcher = true
	browser.Stats = &parsing.Stats{}
	browser.NodeTree = &Node{Name: "root", Parent: nil, Type: "root"}

	// Initialize `places.sqlite`
	bookmarkPath := path.Join(browser.baseDir, browser.bkFile)
	browser.places = DB{}.New("Places", bookmarkPath)
	browser.places.InitRO()

	// Buffer that lives accross Run() jobs
	browser.InitBuffer()

	// Setup watcher

	w := &Watch{
		Path:       path.Join(browser.baseDir),
		EventTypes: []fsnotify.Op{fsnotify.Write},
		EventNames: []string{path.Join(browser.baseDir, "places.sqlite-wal")},
		ResetWatch: false,
	}

	browser.SetupFileWatcher(w)

	/*
	 *Run reducer to avoid duplicate running of jobs
	 *when a batch of events is received
	 */

	browser.eventsChan = make(chan fsnotify.Event, EventsChanLen)

	go tools.ReduceEvents(MozMinJobInterval, browser.eventsChan, browser)
	// Prepare sql statements
	// Check changed urls in DB
	// Firefox records time UTC and microseconds
	// Sqlite converts automatically from utc to local
	QPlacesDelta := `
	SELECT * from moz_bookmarks
	WHERE(lastModified > ?
		AND lastModified < strftime('%s', 'now') * 1000 * 1000
		AND NOT id IN (%d,%d)
		)
	`
	stmt, err := browser.places.Handle.Prepare(
		fmt.Sprintf(QPlacesDelta, "%s", ffBkRoot, ffBkOther),
	)
	if err != nil {
		fflog.Error(err)
	}
	browser.qChanges = stmt
	return browser
}

func (bw *FFBrowser) Shutdown() {

	fflog.Debugf("shutting down ... ")
	err := bw.qChanges.Close()
	if err != nil {
		fflog.Critical(err)
	}

	err = bw.BaseBrowser.Close()
	if err != nil {
		fflog.Critical(err)
	}

	err = bw.places.Close()
	if err != nil {
		fflog.Critical(err)
	}
}

func (bw *FFBrowser) Watch() bool {

	fflog.Debugf("TODO ... ")

	if !bw.isWatching {
		go watch.WatcherThread(bw)
		bw.isWatching = true
		fflog.Infof("Watching %s", bw.GetPath())
		return true
	}

	return false
}

func (bw *FFBrowser) Load() {
	bw.BaseBrowser.Load()

	// Parse bookmarks to a flat tree (for compatibility with tree system)
	start := time.Now()
	getFFBookmarks(bw)
	bw.Stats.LastParseTime = time.Since(start)
	bw.lastRunTime = time.Now().UTC()

	// Finished parsing
	//go PrintTree(bw.NodeTree) // debugging
	fflog.Debugf("parsed %d bookmarks and %d nodes in %s",
		bw.Stats.CurrentUrlCount,
		bw.Stats.CurrentNodeCount,
		bw.Stats.LastParseTime)
	bw.ResetStats()

	// Sync the URLIndex to the buffer
	// We do not use the NodeTree here as firefox tags are represented
	// as a flat tree which is not efficient, we use the go hashmap instead

	database.SyncURLIndexToBuffer(bw.URLIndexList, bw.URLIndex, bw.BufferDB)
	bw.BufferDB.SyncTo(CacheDB)
}

func getFFBookmarks(bw *FFBrowser) {

	QGetBookmarks := `WITH bookmarks AS

	(SELECT moz_places.url AS url,
			moz_places.description as desc,
			moz_places.title as urlTitle,
			moz_bookmarks.parent AS tagId
		FROM moz_places LEFT OUTER JOIN moz_bookmarks
		ON moz_places.id = moz_bookmarks.fk
		WHERE moz_bookmarks.parent
		IN (SELECT id FROM moz_bookmarks WHERE parent = ? ))

	SELECT url, IFNULL(urlTitle, ''), IFNULL(desc,''),
			tagId, moz_bookmarks.title AS tagTitle

	FROM bookmarks LEFT OUTER JOIN moz_bookmarks
	ON tagId = moz_bookmarks.id
	ORDER BY url`

	//QGetTags := "SELECT id,title from moz_bookmarks WHERE parent = %d"

	rows, err := bw.places.Handle.Query(QGetBookmarks, ffBkTags)
	if err != nil {
		fflog.Error(err)
	}

	tagMap := make(map[int]*Node)

	// Rebuild node tree
	// Note: the node tree is build only for compatilibity
	// pruposes with tree based bookmark parsing.
	// For efficiency reading after the initial parse from places.sqlite,
	// reading should be done in loops in instead of tree parsing.
	rootNode := bw.NodeTree

	/*
	 *This pass is used only for fetching bookmarks from firefox.
	 *Checking against the URLIndex should not be done here
	 */
	for rows.Next() {
		var url, title, tagTitle, desc string
		var tagId int
		err = rows.Scan(&url, &title, &desc, &tagId, &tagTitle)
		//fflog.Debugf("%s|%s|%s|%d|%s", url, title, desc, tagId, tagTitle)
		if err != nil {
			fflog.Error(err)
		}

		/*
		 * If this is the first time we see this tag
		 * add it to the tagMap and create its node
		 */
		tagNode, tagNodeExists := tagMap[tagId]
		if !tagNodeExists {
			// Add the tag as a node
			tagNode = new(Node)
			tagNode.Type = "tag"
			tagNode.Name = tagTitle
			tagNode.Parent = rootNode
			rootNode.Children = append(rootNode.Children, tagNode)
			tagMap[tagId] = tagNode
			bw.Stats.CurrentNodeCount++
		}

		// Add the url to the tag
		var urlNode *Node
		iUrlNode, urlNodeExists := bw.URLIndex.Get(url)
		if !urlNodeExists {
			urlNode = new(Node)
			urlNode.Type = "url"
			urlNode.URL = url
			urlNode.Name = title
			urlNode.Desc = desc
			bw.URLIndex.Insert(url, urlNode)
			bw.URLIndexList = append(bw.URLIndexList, url)

		} else {
			urlNode = iUrlNode.(*Node)
		}

		// Add tag to urlnode tags
		urlNode.Tags = append(urlNode.Tags, tagNode.Name)

		// Set tag as parent to urlnode
		urlNode.Parent = tagMap[tagId]

		// Add urlnode as child to tag node
		tagMap[tagId].Children = append(tagMap[tagId].Children, urlNode)

		bw.Stats.CurrentUrlCount++
		bw.Stats.CurrentNodeCount++
	}

	/*
	 *Build tags for each url then check against URLIndex
	 *for changes
	 */

	// Check if url already in index TODO: should be done in new pass
	//iVal, found := bw.URLIndex.Get(urlNode.URL)

	/*
	 * The fields where tags may change are hashed together
	 * to detect changes in futre parses
	 * To handle tag changes we need to get all parent nodes
	 *  (tags) for this url then hash their concatenation
	 */

	//nameHash := xxhash.ChecksumString64(urlNode.Name)

}

func (bw *FFBrowser) Run() {

	//fflog.Debugf("%d", bw.lastRunTime.Unix())
	//var _time string
	//row := bw.places.handle.QueryRow("SELECT strftime('%s', 'now') AS now")
	//row.Scan(&_time)
	//fflog.Debug(_time)

	fflog.Debugf("Checking changes since %s",
		bw.lastRunTime.Format("Mon Jan 2 15:04:05 MST 2006"))
	start := time.Now()
	rows, err := bw.qChanges.Query(bw.lastRunTime.UnixNano() / 1000)
	if err != nil {
		fflog.Error(err)
	}
	defer rows.Close()
	elapsed := time.Since(start)
	fflog.Debugf("Places test query in %s", elapsed)

	// Found new results in places db since last time we had changes
	database.DebugPrintRows(rows)
	if rows.Next() {
		bw.lastRunTime = time.Now().UTC()
		fflog.Debugf("CHANGE ! Time: %s",
			bw.lastRunTime.Format("Mon Jan 2 15:04:05 MST 2006"))
		// Get the change

	} else {
		fflog.Debugf("no change")
	}

	//TODO: change logger for more granular debugging
	// candidates:  glg

}