zk/internal/core/note_index.go

package core

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/bmatcuk/doublestar/v4"
	"github.com/zk-org/zk/internal/util"
	"github.com/zk-org/zk/internal/util/errors"
	"github.com/zk-org/zk/internal/util/paths"
	strutil "github.com/zk-org/zk/internal/util/strings"
)

// NoteIndex persists and grants access to indexed information about the notes.
type NoteIndex interface {
	// Find retrieves the notes matching the given filtering and sorting criteria.
	Find(opts NoteFindOpts) ([]ContextualNote, error)
	// FindMinimal retrieves lightweight metadata for the notes matching the
	// given filtering and sorting criteria.
	FindMinimal(opts NoteFindOpts) ([]MinimalNote, error)

	// Find link match returns the best note match for a given link href,
	// relative to baseDir.
	FindLinkMatch(baseDir string, href string, linkType LinkType) (NoteID, error)

	// FindLinksBetweenNotes retrieves the links between the given notes.
	FindLinksBetweenNotes(ids []NoteID) ([]ResolvedLink, error)

	// FindCollections retrieves all the collections of the given kind.
	FindCollections(kind CollectionKind, sorters []CollectionSorter) ([]Collection, error)

	// Indexed returns the list of indexed note file metadata.
	IndexedPaths() (<-chan paths.Metadata, error)
	// Add indexes a new note.
	Add(note Note) (NoteID, error)
	// Update resets the metadata of an already indexed note.
	Update(note Note) error
	// Remove deletes a note from the index.
	Remove(path string) error

	// Commit performs a set of operations atomically.
	Commit(transaction func(idx NoteIndex) error) error

	// NeedsReindexing returns whether all notes should be reindexed.
	NeedsReindexing() (bool, error)
	// SetNeedsReindexing indicates whether all notes should be reindexed.
	SetNeedsReindexing(needsReindexing bool) error
}

// NoteIndexingStats holds statistics about a notebook indexing process.
type NoteIndexingStats struct {
	// Number of notes in the source.
	SourceCount int `json:"sourceCount"`
	// Number of newly indexed notes.
	AddedCount int `json:"addedCount"`
	// Number of notes modified since last indexing.
	ModifiedCount int `json:"modifiedCount"`
	// Number of notes removed since last indexing.
	RemovedCount int `json:"removedCount"`
	// Duration of the indexing process.
	Duration time.Duration `json:"duration"`
}

// String implements Stringer
func (s NoteIndexingStats) String() string {
	return fmt.Sprintf(`Indexed %d %v in %v
  + %d added
  ~ %d modified
  - %d removed`,
		s.SourceCount,
		strutil.Pluralize("note", s.SourceCount),
		s.Duration.Round(500*time.Millisecond),
		s.AddedCount, s.ModifiedCount, s.RemovedCount,
	)
}

// NoteIndexOpts holds the options for the indexing process.
type NoteIndexOpts struct {
	// When true, existing notes will be reindexed.
	Force   bool
	Verbose bool
}

// indexTask indexes the notes in the given directory with the NoteIndex.
type indexTask struct {
	path    string
	config  Config
	force   bool
	verbose bool
	index   NoteIndex
	parser  NoteParser
	logger  util.Logger
}

func (t *indexTask) execute(callback func(change paths.DiffChange)) (NoteIndexingStats, error) {
	wrap := errors.Wrapper("indexing failed")

	stats := NoteIndexingStats{}
	startTime := time.Now()

	needsReindexing, err := t.index.NeedsReindexing()
	if err != nil {
		return stats, wrap(err)
	}

	print := func(message string) {
		if t.verbose {
			fmt.Println(message)
		}
	}

	force := t.force || needsReindexing

	type IgnoredFile struct {
		Path   string
		Reason string
	}
	ignoredFiles := []IgnoredFile{}

	shouldIgnorePath := func(path string) (bool, error) {
		notifyIgnored := func(reason string) {
			ignoredFiles = append(ignoredFiles, IgnoredFile{
				Path:   path,
				Reason: reason,
			})
		}

		group, err := t.config.GroupConfigForPath(path)
		if err != nil {
			return true, err
		}

		if filepath.Ext(path) != "."+group.Note.Extension {
			notifyIgnored("expected extension \"" + group.Note.Extension + "\"")
			return true, nil
		}

		for _, ignoreGlob := range group.ExcludeGlobs() {
			matches, err := doublestar.PathMatch(ignoreGlob, path)
			if err != nil {
				return true, errors.Wrapf(err, "failed to match exclude glob %s to %s", ignoreGlob, path)
			}
			if matches {
				notifyIgnored("matched exclude glob \"" + ignoreGlob + "\"")
				return true, nil
			}
		}

		return false, nil
	}

	source := paths.Walk(t.path, t.logger, shouldIgnorePath)

	target, err := t.index.IndexedPaths()
	if err != nil {
		return stats, wrap(err)
	}

	// FIXME: Use the FS?
	count, err := paths.Diff(source, target, force, func(change paths.DiffChange) error {
		callback(change)
		print("- " + change.Kind.String() + " " + change.Path)
		absPath := filepath.Join(t.path, change.Path)

		switch change.Kind {
		case paths.DiffAdded:
			stats.AddedCount += 1
			note, err := t.parser.ParseNoteAt(absPath)
			if note != nil {
				_, err = t.index.Add(*note)
			}
			t.logger.Err(err)

		case paths.DiffModified:
			stats.ModifiedCount += 1
			note, err := t.parser.ParseNoteAt(absPath)
			if note != nil {
				err = t.index.Update(*note)
			}
			t.logger.Err(err)

		case paths.DiffRemoved:
			stats.RemovedCount += 1
			err := t.index.Remove(change.Path)
			t.logger.Err(err)
		}

		return nil
	})

	for _, ignored := range ignoredFiles {
		print("- ignored " + ignored.Path + ": " + ignored.Reason)
	}

	stats.SourceCount = count
	stats.Duration = time.Since(startTime)

	if needsReindexing {
		err = t.index.SetNeedsReindexing(false)
	}

	print("")
	return stats, wrap(err)
}