zk/core/note/index.go

package note

import (
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strings"
	"time"

	"github.com/mickael-menu/zk/core/zk"
	"github.com/mickael-menu/zk/util"
	"github.com/mickael-menu/zk/util/errors"
	"github.com/mickael-menu/zk/util/paths"
	strutil "github.com/mickael-menu/zk/util/strings"
	"gopkg.in/djherbis/times.v1"
)

// Metadata holds information about a particular note.
type Metadata struct {
	Path       string
	Title      string
	Lead       string
	Body       string
	RawContent string
	WordCount  int
	Links      []Link
	Created    time.Time
	Modified   time.Time
	Checksum   string
}

// IndexingStats holds metrics about an indexing process.
type IndexingStats struct {
	SourceCount   int
	AddedCount    int
	ModifiedCount int
	RemovedCount  int
	Duration      time.Duration
}

// String implements Stringer
func (s IndexingStats) String() string {
	return fmt.Sprintf(`Indexed %d %v in %v
  + %d added
  ~ %d modified
  - %d removed`,
		s.SourceCount,
		strutil.Pluralize("note", s.SourceCount),
		s.Duration.Round(500*time.Millisecond),
		s.AddedCount, s.ModifiedCount, s.RemovedCount,
	)
}

// Indexer persists the notes index.
type Indexer interface {
	// Indexed returns the list of indexed note file metadata.
	Indexed() (<-chan paths.Metadata, error)
	// Add indexes a new note from its metadata.
	Add(metadata Metadata) (int64, error)
	// Update updates the metadata of an already indexed note.
	Update(metadata Metadata) error
	// Remove deletes a note from the index.
	Remove(path string) error
}

// Index indexes the content of the notes in the given slip box.
func Index(zk *zk.Zk, force bool, parser Parser, indexer Indexer, logger util.Logger, callback func(change paths.DiffChange)) (IndexingStats, error) {
	wrap := errors.Wrapper("indexing failed")

	stats := IndexingStats{}
	startTime := time.Now()

	source := paths.Walk(zk.Path, zk.Config.Extension, logger)
	target, err := indexer.Indexed()
	if err != nil {
		return stats, wrap(err)
	}

	count, err := paths.Diff(source, target, force, func(change paths.DiffChange) error {
		callback(change)

		switch change.Kind {
		case paths.DiffAdded:
			stats.AddedCount += 1
			metadata, err := metadata(change.Path, zk, parser)
			if err == nil {
				_, err = indexer.Add(metadata)
			}
			logger.Err(err)

		case paths.DiffModified:
			stats.ModifiedCount += 1
			metadata, err := metadata(change.Path, zk, parser)
			if err == nil {
				err = indexer.Update(metadata)
			}
			logger.Err(err)

		case paths.DiffRemoved:
			stats.RemovedCount += 1
			err := indexer.Remove(change.Path)
			logger.Err(err)
		}
		return nil
	})

	stats.SourceCount = count
	stats.Duration = time.Since(startTime)

	return stats, wrap(err)
}

// metadata retrieves note metadata for the given file.
func metadata(path string, zk *zk.Zk, parser Parser) (Metadata, error) {
	metadata := Metadata{
		Path:  path,
		Links: make([]Link, 0),
	}

	absPath := filepath.Join(zk.Path, path)
	content, err := ioutil.ReadFile(absPath)
	if err != nil {
		return metadata, err
	}
	contentStr := string(content)
	contentParts, err := parser.Parse(contentStr)
	if err != nil {
		return metadata, err
	}
	metadata.Title = contentParts.Title.String()
	metadata.Lead = contentParts.Lead.String()
	metadata.Body = contentParts.Body.String()
	metadata.RawContent = contentStr
	metadata.WordCount = len(strings.Fields(contentStr))
	metadata.Links = make([]Link, 0)
	metadata.Checksum = fmt.Sprintf("%x", sha256.Sum256(content))

	for _, link := range contentParts.Links {
		if !strutil.IsURL(link.Href) {
			// Make the href relative to the slip box root.
			href := filepath.Join(filepath.Dir(absPath), link.Href)
			link.Href, err = zk.RelPath(href)
			if err != nil {
				return metadata, err
			}
		}
		metadata.Links = append(metadata.Links, link)
	}

	times, err := times.Stat(absPath)
	if err != nil {
		return metadata, err
	}

	metadata.Modified = times.ModTime().UTC()
	if times.HasBirthTime() {
		metadata.Created = times.BirthTime().UTC()
	} else {
		metadata.Created = time.Now().UTC()
	}

	return metadata, nil
}