From dc27a7dd7c2bb8958d0ed1b0a8bc7db19b90023a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Menu?= Date: Sat, 25 Sep 2021 19:28:29 +0200 Subject: [PATCH] Improve Markdown and wiki links matching and generation (#71) Fallback wiki link resolution by matching on title or path Add new template variables when generating Markdown links Add a {{substring}} template helper --- CHANGELOG.md | 18 ++- docs/note-format.md | 17 ++- docs/template.md | 14 +++ internal/adapter/handlebars/handlebars.go | 1 + .../adapter/handlebars/handlebars_test.go | 12 +- internal/adapter/handlebars/helpers/link.go | 10 +- .../adapter/handlebars/helpers/substring.go | 31 +++++ internal/adapter/lsp/document.go | 12 +- internal/adapter/lsp/server.go | 90 +++++++++----- internal/adapter/markdown/markdown.go | 6 +- internal/adapter/markdown/markdown_test.go | 6 +- internal/adapter/sqlite/note_dao.go | 36 ++++-- internal/adapter/sqlite/note_dao_test.go | 30 +++-- internal/cli/cmd/new.go | 11 +- internal/cli/container.go | 2 +- internal/core/link_format.go | 58 +++++++-- internal/core/link_format_test.go | 96 +++++++++++---- internal/core/note.go | 31 +++-- internal/core/note_find.go | 2 + internal/core/note_format.go | 8 +- internal/core/note_index.go | 114 +++--------------- internal/core/note_new_test.go | 73 +++++++---- internal/core/note_parse.go | 103 +++++++++++++++- internal/core/note_parse_test.go | 18 +++ internal/core/notebook.go | 101 ++++++++++++---- 25 files changed, 629 insertions(+), 271 deletions(-) create mode 100644 internal/adapter/handlebars/helpers/substring.go create mode 100644 internal/core/note_parse_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 22c7ba6..e7a7250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,25 @@ All notable changes to this project will be documented in this file. ### Added +* Support for LSP references to browse the backlinks of the current note, if the caret is not over a link. +* New template variables are available when [generating custom Markdown links with `link-format`](docs/note-format.md). + * `filename`, `path`, `abs-path` and `rel-path` for many path flavors. + * `metadata` to use information (e.g. `id`) from the YAML frontmatter. +* The LSP server is now matching wiki links to any part of a note's path or its title. + * Given the note `book/z5mj Information Graphics.md` with the title "Book Review of Information Graphics", the following wiki links would work from a note located under `journal/2020-09-25.md`: + ```markdown + [[../book/z5mj]] + [[book/z5mj]] + [[z5mj]] + [[book review information]] + [[Information Graphics]] + ``` * Use the `{{abs-path}}` template variable when [formatting notes](docs/template-format.md) to print the absolute path to the note (contributed by [@pstuifzand](https://github.com/mickael-menu/zk/pull/60)). +* A new `{{substring s index length}}` template helper extracts a portion of a given string, e.g.: + * `{{substring 'A full quote' 2 4}}` outputs `full` + * `{{substring 'A full quote' -5 5}` outputs `quote` * Allow setting the `--working-dir` and `--notebook-dir` flags before the `zk` subcommand when using aliases, e.g. `zk -W ~/notes my-alias`. -* Support for LSP references to browse the backlinks of the current note, if the caret is not over a link. + ### Fixed diff --git a/docs/note-format.md b/docs/note-format.md index e96f97e..c454709 100644 --- a/docs/note-format.md +++ b/docs/note-format.md @@ -21,9 +21,22 @@ You can set up some features of `zk`'s Markdown parser from your [configuration ### Customizing the Markdown links generated by `zk` -By default, `zk` will generate regular Markdown links for internal links. If you prefer to use `[[Wiki Links]]` instead, set the `link-format` setting to `wiki`. If you want to override completely the link format, you can also set `link-format` to a [custom template](template.md). Two variables `path` and `title` are available in the template, for example to generate a wiki-link with a title: +By default, `zk` will generate regular Markdown links for internal links. If you prefer to use `[[Wiki Links]]` instead, set the `link-format` setting to `wiki`. If you want to override completely the link format, you can also set `link-format` to a [custom template](template.md). For example, to generate a wiki link using an ID from the frontmatter and a title: ```toml [format.markdown] -link-format = "[[{{path}}|{{title}}]]" +link-format = "[[{{metadata.id}}|{{title}}]]" ``` + +The following variables are available in the template: + +| Variable | Type | Description | +|------------|--------|-----------------------------------------------------------| +| `filename` | string | Filename of the note | +| `path` | string | File path to the note, relative to the notebook directory | +| `abs-path` | string | Absolute file path to the note | +| `rel-path` | string | File path to the note, relative to the current directory | +| `title` | string | Note title | +| `metadata` | map | YAML frontmatter metadata, e.g. `metadata.id`1 | + +1. YAML keys are normalized to lower case. diff --git a/docs/template.md b/docs/template.md index dd1199a..732e788 100644 --- a/docs/template.md +++ b/docs/template.md @@ -24,6 +24,20 @@ can generate (depending on the user config): The second parameter `title` is optional. +### String helpers + +There are a couple of template helpers operating on strings. + +#### Concat helper + +The `{{concat s1 s2}}` helper concatenates two strings together. For example `{{concat '> ' 'A quote'}}` produces `> A quote`. + +#### Substring helper + +* The `{{substring s index length}}` helper extracts a portion of the given string. For example: + * `{{substring 'A full quote' 2 4}}` outputs `full` + * `{{substring 'A full quote' -5 5}` outputs `quote` + ### Date helper The `{{date}}` helper formats the given date for display. diff --git a/internal/adapter/handlebars/handlebars.go b/internal/adapter/handlebars/handlebars.go index a2fff8d..5ae031e 100644 --- a/internal/adapter/handlebars/handlebars.go +++ b/internal/adapter/handlebars/handlebars.go @@ -15,6 +15,7 @@ import ( func Init(supportsUTF8 bool, logger util.Logger) { helpers.RegisterConcat() + helpers.RegisterSubstring() helpers.RegisterDate(logger) helpers.RegisterJoin() helpers.RegisterJSON(logger) diff --git a/internal/adapter/handlebars/handlebars_test.go b/internal/adapter/handlebars/handlebars_test.go index bd7d0dc..9d2e8fe 100644 --- a/internal/adapter/handlebars/handlebars_test.go +++ b/internal/adapter/handlebars/handlebars_test.go @@ -137,6 +137,14 @@ func TestConcatHelper(t *testing.T) { testString(t, "{{concat '> ' 'A quote'}}", nil, "> A quote") } +func TestSubstringHelper(t *testing.T) { + testString(t, "{{substring '' 2 4}}", nil, "") + testString(t, "{{substring 'A full quote' 2 4}}", nil, "full") + testString(t, "{{substring 'A full quote' 40 4}}", nil, "") + testString(t, "{{substring 'A full quote' -5 5}}", nil, "quote") + testString(t, "{{substring 'A full quote' -5 6}}", nil, "quote") +} + func TestJoinHelper(t *testing.T) { test := func(items []string, expected string) { context := map[string]interface{}{"items": items} @@ -273,8 +281,8 @@ func testLoader(opts LoaderOpts) *Loader { loader.RegisterHelper("style", helpers.NewStyleHelper(opts.Styler, &util.NullLogger)) loader.RegisterHelper("slug", helpers.NewSlugHelper("en", &util.NullLogger)) - formatter := func(path, title string) (string, error) { - return path + " - " + title, nil + formatter := func(context core.LinkFormatterContext) (string, error) { + return context.Path + " - " + context.Title, nil } loader.RegisterHelper("format-link", helpers.NewLinkHelper(formatter, &util.NullLogger)) diff --git a/internal/adapter/handlebars/helpers/link.go b/internal/adapter/handlebars/helpers/link.go index da370f7..269a840 100644 --- a/internal/adapter/handlebars/helpers/link.go +++ b/internal/adapter/handlebars/helpers/link.go @@ -8,13 +8,19 @@ import ( // NewLinkHelper creates a new template helper to generate an internal link // using a LinkFormatter. // -// {{link "path/to/note.md" "An interesting subject"}} -> (depends on the LinkFormatter) +// {{format-link "path/to/note.md" "An interesting subject"}} -> (depends on the LinkFormatter) // [[path/to/note]] // [An interesting subject](path/to/note) func NewLinkHelper(formatter core.LinkFormatter, logger util.Logger) interface{} { return func(path string, opt interface{}) string { title, _ := opt.(string) - link, err := formatter(path, title) + link, err := formatter(core.LinkFormatterContext{ + Path: path, + RelPath: path, + AbsPath: path, + Title: title, + Metadata: map[string]interface{}{}, + }) if err != nil { logger.Err(err) return "" diff --git a/internal/adapter/handlebars/helpers/substring.go b/internal/adapter/handlebars/helpers/substring.go new file mode 100644 index 0000000..4deb3e3 --- /dev/null +++ b/internal/adapter/handlebars/helpers/substring.go @@ -0,0 +1,31 @@ +package helpers + +import ( + "github.com/aymerick/raymond" +) + +// RegisterSubstring registers a {{substring}} template helper which extracts a +// substring given a starting index and a length. +// +// {{substring 'A full quote' 2 4}} -> "full" +// {{substring 'A full quote' -5 5}} -> "quote" +// +func RegisterSubstring() { + raymond.RegisterHelper("substring", func(str string, index int, length int) string { + if index < 0 { + index = len(str) + index + } + if index >= len(str) { + return "" + } + end := min(index+length, len(str)) + return str[index:end] + }) +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/internal/adapter/lsp/document.go b/internal/adapter/lsp/document.go index 60cef61..8b00561 100644 --- a/internal/adapter/lsp/document.go +++ b/internal/adapter/lsp/document.go @@ -195,7 +195,7 @@ func (d *document) DocumentLinks() ([]documentLink, error) { lines := d.GetLines() for lineIndex, line := range lines { - appendLink := func(href string, start, end int, hasTitle bool) { + appendLink := func(href string, start, end int, hasTitle bool, isWikiLink bool) { if href == "" { return } @@ -212,7 +212,8 @@ func (d *document) DocumentLinks() ([]documentLink, error) { Character: protocol.UInteger(end), }, }, - HasTitle: hasTitle, + HasTitle: hasTitle, + IsWikiLink: isWikiLink, }) } @@ -222,13 +223,13 @@ func (d *document) DocumentLinks() ([]documentLink, error) { if decodedHref, err := url.PathUnescape(href); err == nil { href = decodedHref } - appendLink(href, match[0], match[1], true) + appendLink(href, match[0], match[1], false, false) } for _, match := range wikiLinkRegex.FindAllStringSubmatchIndex(line, -1) { href := line[match[2]:match[3]] hasTitle := match[4] != -1 - appendLink(href, match[0], match[1], hasTitle) + appendLink(href, match[0], match[1], hasTitle, true) } } @@ -241,4 +242,7 @@ type documentLink struct { // HasTitle indicates whether this link has a title information. For // example [[filename]] doesn't but [[filename|title]] does. HasTitle bool + // IsWikiLink indicates whether this link is a [[WikiLink]] instead of a + // regular Markdown link. + IsWikiLink bool } diff --git a/internal/adapter/lsp/server.go b/internal/adapter/lsp/server.go index 035a84c..eb31065 100644 --- a/internal/adapter/lsp/server.go +++ b/internal/adapter/lsp/server.go @@ -241,7 +241,7 @@ func NewServer(opts ServerOpts) *Server { return nil, err } - target, err := server.noteForHref(link.Href, doc, notebook) + target, err := server.noteForLink(*link, doc, notebook) if err != nil || target == nil { return nil, err } @@ -284,7 +284,7 @@ func NewServer(opts ServerOpts) *Server { documentLinks := []protocol.DocumentLink{} for _, link := range links { - target, err := server.noteForHref(link.Href, doc, notebook) + target, err := server.noteForLink(link, doc, notebook) if target == nil || err != nil { continue } @@ -314,7 +314,7 @@ func NewServer(opts ServerOpts) *Server { return nil, err } - target, err := server.noteForHref(link.Href, doc, notebook) + target, err := server.noteForLink(*link, doc, notebook) if link == nil || target == nil || err != nil { return nil, err } @@ -414,7 +414,7 @@ func NewServer(opts ServerOpts) *Server { link = &documentLink{Href: href} } - target, err := server.noteForHref(link.Href, doc, notebook) + target, err := server.noteForLink(*link, doc, notebook) if link == nil || target == nil || err != nil { return nil, err } @@ -546,7 +546,7 @@ func (s *Server) executeCommandNew(context *glsp.Context, args []interface{}) (i return nil, errors.Wrapf(err, "%s, failed to parse the `date` option", opts.Date) } - path, err := notebook.NewNote(core.NewNoteOpts{ + note, err := notebook.NewNote(core.NewNoteOpts{ Title: opt.NewNotEmptyString(opts.Title), Content: opts.Content, Directory: opt.NewNotEmptyString(opts.Dir), @@ -560,11 +560,16 @@ func (s *Server) executeCommandNew(context *glsp.Context, args []interface{}) (i if !errors.As(err, ¬eExists) { return nil, err } - path = noteExists.Path + note, err = notebook.FindNote(core.NoteFindOpts{ + IncludePaths: []string{noteExists.Name}, + }) + if err != nil { + return nil, err + } + } + if note == nil { + return nil, errors.New("zk.new could not generate a new note") } - - // Index the notebook to be able to navigate to the new note. - notebook.Index(false) if opts.InsertLinkAtLocation != nil { doc, ok := s.documents.Get(opts.InsertLinkAtLocation.URI) @@ -576,12 +581,13 @@ func (s *Server) executeCommandNew(context *glsp.Context, args []interface{}) (i return nil, err } - relPath, err := filepath.Rel(filepath.Dir(doc.Path), path) + currentDir := filepath.Dir(doc.Path) + linkFormatterContext, err := core.NewLinkFormatterContext(note.AsMinimalNote(), notebook.Path, currentDir) if err != nil { return nil, err } - link, err := linkFormatter(relPath, opts.Title) + link, err := linkFormatter(linkFormatterContext) if err != nil { return nil, err } @@ -595,22 +601,47 @@ func (s *Server) executeCommandNew(context *glsp.Context, args []interface{}) (i }, nil) } + absPath := filepath.Join(notebook.Path, note.Path) if opts.Edit { go context.Call(protocol.ServerWindowShowDocument, protocol.ShowDocumentParams{ - URI: "file://" + path, + URI: pathToURI(absPath), TakeFocus: boolPtr(true), }, nil) } - return map[string]interface{}{"path": path}, nil + return map[string]interface{}{"path": absPath}, nil } func (s *Server) notebookOf(doc *document) (*core.Notebook, error) { return s.notebooks.Open(doc.Path) } -// noteForHref returns the LSP documentUri for the note at the given HREF. -func (s *Server) noteForHref(href string, doc *document, notebook *core.Notebook) (*Note, error) { +// noteForLink returns the LSP documentUri for the note targeted by the given link. +// +// Match by order of precedence: +// 1. Prefix of relative path +// 2. Find any occurrence of the href in a note path (substring) +// 3. Match the href as a term in the note titles +func (s *Server) noteForLink(link documentLink, doc *document, notebook *core.Notebook) (*Note, error) { + note, err := s.noteForHref(link.Href, doc, notebook) + if note == nil && err == nil && link.IsWikiLink { + // Try to find a partial href match. + note, err = notebook.FindByHref(link.Href, true) + if note == nil && err == nil { + // Fallback on matching the note title. + note, err = s.noteMatchingTitle(link.Href, notebook) + } + } + if note == nil || err != nil { + return nil, err + } + + joined_path := filepath.Join(notebook.Path, note.Path) + return &Note{*note, pathToURI(joined_path)}, nil +} + +// noteForHref returns the LSP documentUri for the note targeted by the given HREF. +func (s *Server) noteForHref(href string, doc *document, notebook *core.Notebook) (*core.MinimalNote, error) { if strutil.IsURL(href) { return nil, nil } @@ -620,17 +651,24 @@ func (s *Server) noteForHref(href string, doc *document, notebook *core.Notebook if err != nil { return nil, errors.Wrapf(err, "failed to resolve href: %s", href) } - note, err := notebook.FindByHref(path) + note, err := notebook.FindByHref(path, false) if err != nil { s.logger.Printf("findByHref(%s): %s", href, err.Error()) - return nil, err } - if note == nil { + return note, err +} + +// noteMatchingTitle returns the LSP documentUri for the note matching the given search terms. +func (s *Server) noteMatchingTitle(terms string, notebook *core.Notebook) (*core.MinimalNote, error) { + if terms == "" { return nil, nil } - joined_path := filepath.Join(notebook.Path, note.Path) - return &Note{*note, pathToURI(joined_path)}, nil + note, err := notebook.FindMatching("title:(" + terms + ")") + if err != nil { + s.logger.Printf("findMatching(title: %s): %s", terms, err.Error()) + } + return note, err } type Note struct { @@ -673,7 +711,7 @@ func (s *Server) refreshDiagnosticsOfDocument(doc *document, notify glsp.NotifyF if strutil.IsURL(link.Href) { continue } - target, err := s.noteForHref(link.Href, doc, notebook) + target, err := s.noteForLink(link, doc, notebook) if err != nil { s.logger.Err(err) continue @@ -813,14 +851,12 @@ func (s *Server) newCompletionItem(notebook *core.Notebook, note core.MinimalNot } func (s *Server) newTextEditForLink(notebook *core.Notebook, note core.MinimalNote, doc *document, pos protocol.Position, linkFormatter core.LinkFormatter) (interface{}, error) { - path := filepath.Join(notebook.Path, note.Path) - path = s.fs.Canonical(path) - path, err := filepath.Rel(filepath.Dir(doc.Path), path) + currentDir := filepath.Dir(doc.Path) + context, err := core.NewLinkFormatterContext(note, notebook.Path, currentDir) if err != nil { - path = note.Path + return nil, err } - - link, err := linkFormatter(path, note.Title) + link, err := linkFormatter(context) if err != nil { return nil, err } diff --git a/internal/adapter/markdown/markdown.go b/internal/adapter/markdown/markdown.go index aae7c0c..2394e3a 100644 --- a/internal/adapter/markdown/markdown.go +++ b/internal/adapter/markdown/markdown.go @@ -60,8 +60,8 @@ func NewParser(options ParserOpts) *Parser { } } -// Parse implements core.NoteParser. -func (p *Parser) Parse(content string) (*core.ParsedNote, error) { +// ParseNoteContent implements core.NoteContentParser. +func (p *Parser) ParseNoteContent(content string) (*core.NoteContent, error) { bytes := []byte(content) context := parser.NewContext() @@ -91,7 +91,7 @@ func (p *Parser) Parse(content string) (*core.ParsedNote, error) { return nil, err } - return &core.ParsedNote{ + return &core.NoteContent{ Title: title, Body: body, Lead: parseLead(body), diff --git a/internal/adapter/markdown/markdown_test.go b/internal/adapter/markdown/markdown_test.go index 1d827a5..7973e92 100644 --- a/internal/adapter/markdown/markdown_test.go +++ b/internal/adapter/markdown/markdown_test.go @@ -575,7 +575,7 @@ Paragraph }) } -func parse(t *testing.T, source string) core.ParsedNote { +func parse(t *testing.T, source string) core.NoteContent { return parseWithOptions(t, source, ParserOpts{ HashtagEnabled: true, MultiWordTagEnabled: true, @@ -583,8 +583,8 @@ func parse(t *testing.T, source string) core.ParsedNote { }) } -func parseWithOptions(t *testing.T, source string, options ParserOpts) core.ParsedNote { - content, err := NewParser(options).Parse(source) +func parseWithOptions(t *testing.T, source string, options ParserOpts) core.NoteContent { + content, err := NewParser(options).ParseNoteContent(source) assert.Nil(t, err) return *content } diff --git a/internal/adapter/sqlite/note_dao.go b/internal/adapter/sqlite/note_dao.go index 2666694..d012f69 100644 --- a/internal/adapter/sqlite/note_dao.go +++ b/internal/adapter/sqlite/note_dao.go @@ -345,21 +345,27 @@ func (d *NoteDAO) FindMinimal(opts core.NoteFindOpts) ([]core.MinimalNote, error func (d *NoteDAO) scanMinimalNote(row RowScanner) (*core.MinimalNote, error) { var ( - id int - path, title string + id int + path, title, metadataJSON string ) - err := row.Scan(&id, &path, &title) + err := row.Scan(&id, &path, &title, &metadataJSON) switch { case err == sql.ErrNoRows: return nil, nil case err != nil: return nil, err default: + metadata, err := unmarshalMetadata(metadataJSON) + if err != nil { + d.logger.Err(errors.Wrap(err, path)) + } + return &core.MinimalNote{ - ID: core.NoteID(id), - Path: path, - Title: title, + ID: core.NoteID(id), + Path: path, + Title: title, + Metadata: metadata, }, nil } } @@ -403,8 +409,8 @@ func (d *NoteDAO) scanNote(row RowScanner) (*core.ContextualNote, error) { ) err := row.Scan( - &id, &path, &title, &lead, &body, &rawContent, &wordCount, - &created, &modified, &metadataJSON, &checksum, &tags, &snippets, + &id, &path, &title, &metadataJSON, &lead, &body, &rawContent, + &wordCount, &created, &modified, &checksum, &tags, &snippets, ) switch { case err == sql.ErrNoRows: @@ -598,7 +604,10 @@ func (d *NoteDAO) findRows(opts core.NoteFindOpts, minimal bool) (*sql.Rows, err regexes := make([]string, 0) for _, path := range opts.IncludePaths { regexes = append(regexes, "n.path REGEXP ?") - args = append(args, pathRegex(path)) + if !opts.EnablePathRegexes { + path = pathRegex(path) + } + args = append(args, path) } whereExprs = append(whereExprs, strings.Join(regexes, " OR ")) } @@ -607,7 +616,10 @@ func (d *NoteDAO) findRows(opts core.NoteFindOpts, minimal bool) (*sql.Rows, err regexes := make([]string, 0) for _, path := range opts.ExcludePaths { regexes = append(regexes, "n.path NOT REGEXP ?") - args = append(args, pathRegex(path)) + if !opts.EnablePathRegexes { + path = pathRegex(path) + } + args = append(args, path) } whereExprs = append(whereExprs, strings.Join(regexes, " AND ")) } @@ -771,9 +783,9 @@ WHERE collection_id IN (SELECT id FROM collections t WHERE kind = '%s' AND (%s)) query += "\n)\n" } - query += "SELECT n.id, n.path, n.title" + query += "SELECT n.id, n.path, n.title, n.metadata" if !minimal { - query += fmt.Sprintf(", n.lead, n.body, n.raw_content, n.word_count, n.created, n.modified, n.metadata, n.checksum, n.tags, %s AS snippet", snippetCol) + query += fmt.Sprintf(", n.lead, n.body, n.raw_content, n.word_count, n.created, n.modified, n.checksum, n.tags, %s AS snippet", snippetCol) } query += "\nFROM notes_with_metadata n\n" diff --git a/internal/adapter/sqlite/note_dao_test.go b/internal/adapter/sqlite/note_dao_test.go index d1ccc85..2fff020 100644 --- a/internal/adapter/sqlite/note_dao_test.go +++ b/internal/adapter/sqlite/note_dao_test.go @@ -397,13 +397,19 @@ func TestNoteDAOFindMinimalAll(t *testing.T) { assert.Nil(t, err) assert.Equal(t, notes, []core.MinimalNote{ - {ID: 5, Path: "ref/test/b.md", Title: "A nested note"}, - {ID: 4, Path: "f39c8.md", Title: "An interesting note"}, - {ID: 6, Path: "ref/test/a.md", Title: "Another nested note"}, - {ID: 1, Path: "log/2021-01-03.md", Title: "Daily note"}, - {ID: 7, Path: "log/2021-02-04.md", Title: "February 4, 2021"}, - {ID: 3, Path: "index.md", Title: "Index"}, - {ID: 2, Path: "log/2021-01-04.md", Title: "January 4, 2021"}, + {ID: 5, Path: "ref/test/b.md", Title: "A nested note", Metadata: map[string]interface{}{}}, + {ID: 4, Path: "f39c8.md", Title: "An interesting note", Metadata: map[string]interface{}{}}, + {ID: 6, Path: "ref/test/a.md", Title: "Another nested note", Metadata: map[string]interface{}{ + "alias": "a.md", + }}, + {ID: 1, Path: "log/2021-01-03.md", Title: "Daily note", Metadata: map[string]interface{}{ + "author": "Dom", + }}, + {ID: 7, Path: "log/2021-02-04.md", Title: "February 4, 2021", Metadata: map[string]interface{}{}}, + {ID: 3, Path: "index.md", Title: "Index", Metadata: map[string]interface{}{ + "aliases": []interface{}{"First page"}, + }}, + {ID: 2, Path: "log/2021-01-04.md", Title: "January 4, 2021", Metadata: map[string]interface{}{}}, }) }) } @@ -418,9 +424,13 @@ func TestNoteDAOFindMinimalWithFilter(t *testing.T) { assert.Nil(t, err) assert.Equal(t, notes, []core.MinimalNote{ - {ID: 1, Path: "log/2021-01-03.md", Title: "Daily note"}, - {ID: 3, Path: "index.md", Title: "Index"}, - {ID: 7, Path: "log/2021-02-04.md", Title: "February 4, 2021"}, + {ID: 1, Path: "log/2021-01-03.md", Title: "Daily note", Metadata: map[string]interface{}{ + "author": "Dom", + }}, + {ID: 3, Path: "index.md", Title: "Index", Metadata: map[string]interface{}{ + "aliases": []interface{}{"First page"}, + }}, + {ID: 7, Path: "log/2021-02-04.md", Title: "February 4, 2021", Metadata: map[string]interface{}{}}, }) }) } diff --git a/internal/cli/cmd/new.go b/internal/cli/cmd/new.go index a3e71c6..0126474 100644 --- a/internal/cli/cmd/new.go +++ b/internal/cli/cmd/new.go @@ -3,6 +3,7 @@ package cmd import ( "errors" "fmt" + "path/filepath" "time" "github.com/mickael-menu/zk/internal/cli" @@ -32,7 +33,7 @@ func (cmd *New) Run(container *cli.Container) error { return err } - path, err := notebook.NewNote(core.NewNoteOpts{ + note, err := notebook.NewNote(core.NewNoteOpts{ Title: opt.NewNotEmptyString(cmd.Title), Content: content.Unwrap(), Directory: opt.NewNotEmptyString(cmd.Directory), @@ -41,19 +42,15 @@ func (cmd *New) Run(container *cli.Container) error { Extra: cmd.Extra, Date: time.Now(), }) + path := filepath.Join(notebook.Path, note.Path) if err != nil { var noteExists core.ErrNoteExists if !errors.As(err, ¬eExists) { return err } - relPath, err := notebook.RelPath(path) - if err != nil { - return err - } - if confirmed, _ := container.Terminal.Confirm( - fmt.Sprintf("%s already exists, do you want to edit this note instead?", relPath), + fmt.Sprintf("%s already exists, do you want to edit this note instead?", note.Path), true, ); !confirmed { // abort... diff --git a/internal/cli/container.go b/internal/cli/container.go index 11f0b48..610ab4c 100644 --- a/internal/cli/container.go +++ b/internal/cli/container.go @@ -89,7 +89,7 @@ func NewContainer(version string) (*Container, error) { notebook := core.NewNotebook(path, config, core.NotebookPorts{ NoteIndex: sqlite.NewNoteIndex(db, logger), - NoteParser: markdown.NewParser(markdown.ParserOpts{ + NoteContentParser: markdown.NewParser(markdown.ParserOpts{ HashtagEnabled: config.Format.Markdown.Hashtags, MultiWordTagEnabled: config.Format.Markdown.MultiwordTags, ColontagEnabled: config.Format.Markdown.ColonTags, diff --git a/internal/core/link_format.go b/internal/core/link_format.go index edad8a9..d93396a 100644 --- a/internal/core/link_format.go +++ b/internal/core/link_format.go @@ -3,14 +3,47 @@ package core import ( "fmt" "net/url" + "path/filepath" "strings" "github.com/mickael-menu/zk/internal/util/errors" "github.com/mickael-menu/zk/internal/util/paths" ) +// Metadata used to generate a link. +type LinkFormatterContext struct { + // Filename of the note + Filename string + // File path to the note, relative to the notebook root. + Path string + // Absolute file path to the note. + AbsPath string `handlebars:"abs-path"` + // File path to the note, relative to the current directory. + RelPath string `handlebars:"rel-path"` + // Title of the note. + Title string + // Metadata extracted from the YAML frontmatter. + Metadata map[string]interface{} +} + +func NewLinkFormatterContext(note MinimalNote, notebookDir string, currentDir string) (LinkFormatterContext, error) { + absPath := filepath.Join(notebookDir, note.Path) + relPath, err := filepath.Rel(currentDir, absPath) + if err != nil { + return LinkFormatterContext{}, err + } + return LinkFormatterContext{ + Filename: filepath.Base(note.Path), + Path: note.Path, + AbsPath: absPath, + RelPath: relPath, + Title: note.Title, + Metadata: note.Metadata, + }, nil +} + // LinkFormatter formats internal links according to user configuration. -type LinkFormatter func(path string, title string) (string, error) +type LinkFormatter func(context LinkFormatterContext) (string, error) // NewLinkFormatter generates a new LinkFormatter from the user Markdown // configuration. @@ -26,8 +59,8 @@ func NewLinkFormatter(config MarkdownConfig, templateLoader TemplateLoader) (Lin } func NewMarkdownLinkFormatter(config MarkdownConfig, onlyHref bool) (LinkFormatter, error) { - return func(path, title string) (string, error) { - path = formatPath(path, config) + return func(context LinkFormatterContext) (string, error) { + path := formatPath(context.RelPath, config) if !config.LinkEncodePath { path = strings.ReplaceAll(path, `\`, `\\`) path = strings.ReplaceAll(path, `)`, `\)`) @@ -35,6 +68,7 @@ func NewMarkdownLinkFormatter(config MarkdownConfig, onlyHref bool) (LinkFormatt if onlyHref { return fmt.Sprintf("(%s)", path), nil } else { + title := context.Title title = strings.ReplaceAll(title, `\`, `\\`) title = strings.ReplaceAll(title, `]`, `\]`) return fmt.Sprintf("[%s](%s)", title, path), nil @@ -43,8 +77,8 @@ func NewMarkdownLinkFormatter(config MarkdownConfig, onlyHref bool) (LinkFormatt } func NewWikiLinkFormatter(config MarkdownConfig) (LinkFormatter, error) { - return func(path, title string) (string, error) { - path = formatPath(path, config) + return func(context LinkFormatterContext) (string, error) { + path := formatPath(context.Path, config) if !config.LinkEncodePath { path = strings.ReplaceAll(path, `\`, `\\`) path = strings.ReplaceAll(path, `]]`, `\]]`) @@ -60,17 +94,15 @@ func NewCustomLinkFormatter(config MarkdownConfig, templateLoader TemplateLoader return nil, wrap(err) } - return func(path, title string) (string, error) { - path = formatPath(path, config) - return template.Render(customLinkRenderContext{Path: path, Title: title}) + return func(context LinkFormatterContext) (string, error) { + context.Filename = formatPath(context.Filename, config) + context.Path = formatPath(context.Path, config) + context.RelPath = formatPath(context.RelPath, config) + context.AbsPath = formatPath(context.AbsPath, config) + return template.Render(context) }, nil } -type customLinkRenderContext struct { - Path string - Title string -} - func formatPath(path string, config MarkdownConfig) string { if config.LinkDropExtension { path = paths.DropExt(path) diff --git a/internal/core/link_format_test.go b/internal/core/link_format_test.go index 83c6dbc..e8b974c 100644 --- a/internal/core/link_format_test.go +++ b/internal/core/link_format_test.go @@ -1,6 +1,7 @@ package core import ( + "path/filepath" "testing" "github.com/mickael-menu/zk/internal/util/test/assert" @@ -16,7 +17,13 @@ func TestMarkdownLinkFormatter(t *testing.T) { assert.Nil(t, err) return func(path, title, expected string) { - actual, err := formatter(path, title) + actual, err := formatter(LinkFormatterContext{ + Filename: "filename", + Path: "path", + RelPath: path, + AbsPath: "abs-path", + Title: title, + }) assert.Nil(t, err) assert.Equal(t, actual, expected) } @@ -46,7 +53,13 @@ func TestMarkdownLinkFormatterOnlyHref(t *testing.T) { assert.Nil(t, err) return func(path, expected string) { - actual, err := formatter(path, "") + actual, err := formatter(LinkFormatterContext{ + Filename: "filename", + Path: "path", + RelPath: path, + AbsPath: "abs-path", + Title: "title", + }) assert.Nil(t, err) assert.Equal(t, actual, expected) } @@ -76,7 +89,13 @@ func TestWikiLinkFormatter(t *testing.T) { assert.Nil(t, err) return func(path, title, expected string) { - actual, err := formatter(path, title) + actual, err := formatter(LinkFormatterContext{ + Filename: "filename", + Path: path, + RelPath: "rel-path", + AbsPath: "abs-path", + Title: "title", + }) assert.Nil(t, err) assert.Equal(t, actual, expected) } @@ -98,8 +117,8 @@ func TestWikiLinkFormatter(t *testing.T) { } func TestCustomLinkFormatter(t *testing.T) { - newTester := func(encodePath, dropExtension bool) func(path, title string, expected customLinkRenderContext) { - return func(path, title string, expected customLinkRenderContext) { + newTester := func(encodePath, dropExtension bool) func(path, title string, expected LinkFormatterContext) { + return func(path, title string, expected LinkFormatterContext) { loader := newTemplateLoaderMock() template := loader.SpyString("custom") @@ -110,7 +129,13 @@ func TestCustomLinkFormatter(t *testing.T) { }, loader) assert.Nil(t, err) - actual, err := formatter(path, title) + actual, err := formatter(LinkFormatterContext{ + Filename: filepath.Base(path), + Path: path, + AbsPath: "/" + path, + RelPath: "../" + path, + Title: title, + }) assert.Nil(t, err) assert.Equal(t, actual, "custom") assert.Equal(t, template.Contexts, []interface{}{expected}) @@ -118,29 +143,54 @@ func TestCustomLinkFormatter(t *testing.T) { } test := newTester(false, false) - test("path/to note.md", "", customLinkRenderContext{Path: "path/to note.md"}) - test("", "", customLinkRenderContext{}) - test("path/to note.md", "An interesting subject", customLinkRenderContext{ - Title: "An interesting subject", - Path: "path/to note.md", + test("path/to note.md", "", LinkFormatterContext{ + Filename: "to note.md", + Path: "path/to note.md", + AbsPath: "/path/to note.md", + RelPath: "../path/to note.md", }) - test(`path/(no\te).md`, `An [interesting] \subject`, customLinkRenderContext{ - Title: `An [interesting] \subject`, - Path: `path/(no\te).md`, + test("", "", LinkFormatterContext{ + Filename: ".", + Path: "", + AbsPath: "/", + RelPath: "../", + }) + test("path/to note.md", "An interesting subject", LinkFormatterContext{ + Filename: "to note.md", + Path: "path/to note.md", + AbsPath: "/path/to note.md", + RelPath: "../path/to note.md", + Title: "An interesting subject", + }) + test(`path/(no\te).md`, `An [interesting] \subject`, LinkFormatterContext{ + Filename: `(no\te).md`, + Path: `path/(no\te).md`, + AbsPath: `/path/(no\te).md`, + RelPath: `../path/(no\te).md`, + Title: `An [interesting] \subject`, }) test = newTester(true, false) - test("path/to note.md", "An interesting subject", customLinkRenderContext{ - Title: "An interesting subject", - Path: "path/to%20note.md", + test("path/to note.md", "An interesting subject", LinkFormatterContext{ + Filename: "to%20note.md", + Path: "path/to%20note.md", + AbsPath: "/path/to%20note.md", + RelPath: "../path/to%20note.md", + Title: "An interesting subject", }) test = newTester(false, true) - test("path/to note.md", "An interesting subject", customLinkRenderContext{ - Title: "An interesting subject", - Path: "path/to note", + test("path/to note.md", "An interesting subject", LinkFormatterContext{ + Filename: "to note", + Path: "path/to note", + AbsPath: "/path/to note", + RelPath: "../path/to note", + Title: "An interesting subject", }) test = newTester(true, true) - test("path/to note.md", "An interesting subject", customLinkRenderContext{ - Title: "An interesting subject", - Path: "path/to%20note", + test("path/to note.md", "An interesting subject", LinkFormatterContext{ + Filename: "to%20note", + Path: "path/to%20note", + AbsPath: "/path/to%20note", + RelPath: "../path/to%20note", + Title: "An interesting subject", }) } diff --git a/internal/core/note.go b/internal/core/note.go index 5a31922..c62bb4c 100644 --- a/internal/core/note.go +++ b/internal/core/note.go @@ -12,6 +12,18 @@ func (id NoteID) IsValid() bool { return id > 0 } +// MinimalNote holds a Note's title and path information, for display purposes. +type MinimalNote struct { + // Unique ID of this note in a notebook. + ID NoteID + // Path relative to the root of the notebook. + Path string + // Title of the note. + Title string + // JSON dictionary of raw metadata extracted from the frontmatter. + Metadata map[string]interface{} +} + // Note holds the metadata and content of a single note. type Note struct { // Unique ID of this note in a NoteRepository. @@ -42,6 +54,15 @@ type Note struct { Checksum string } +func (n Note) AsMinimalNote() MinimalNote { + return MinimalNote{ + ID: n.ID, + Path: n.Path, + Title: n.Title, + Metadata: n.Metadata, + } +} + // ContextualNote holds a Note and context-sensitive content snippets. // // This is used for example: @@ -52,13 +73,3 @@ type ContextualNote struct { // List of context-sensitive excerpts from the note. Snippets []string } - -// MinimalNote holds a Note's title and path information, for display purposes. -type MinimalNote struct { - // Unique ID of this note in a notebook. - ID NoteID - // Path relative to the root of the notebook. - Path string - // Title of the note. - Title string -} diff --git a/internal/core/note_find.go b/internal/core/note_find.go index df767e6..d9a4656 100644 --- a/internal/core/note_find.go +++ b/internal/core/note_find.go @@ -19,6 +19,8 @@ type NoteFindOpts struct { IncludePaths []string // Filter excluding notes at the given paths. ExcludePaths []string + // Indicates whether IncludePaths and ExcludePaths are using regexes. + EnablePathRegexes bool // Filter excluding notes with the given IDs. ExcludeIDs []NoteID // Filter by tags found in the notes. diff --git a/internal/core/note_format.go b/internal/core/note_format.go index b5f0d49..c769a58 100644 --- a/internal/core/note_format.go +++ b/internal/core/note_format.go @@ -38,7 +38,13 @@ func newNoteFormatter(basePath string, template Template, linkFormatter LinkForm AbsPath: absPath, Title: note.Title, Link: newLazyStringer(func() string { - link, _ := linkFormatter(path, note.Title) + link, _ := linkFormatter(LinkFormatterContext{ + Path: note.Path, + RelPath: path, + AbsPath: absPath, + Title: note.Title, + Metadata: note.Metadata, + }) return link }), Lead: note.Lead, diff --git a/internal/core/note_index.go b/internal/core/note_index.go index 429d06c..63c27fa 100644 --- a/internal/core/note_index.go +++ b/internal/core/note_index.go @@ -1,24 +1,18 @@ package core import ( - "crypto/sha256" "fmt" - "io/ioutil" "path/filepath" - "strings" "time" "github.com/mickael-menu/zk/internal/util" "github.com/mickael-menu/zk/internal/util/errors" "github.com/mickael-menu/zk/internal/util/paths" strutil "github.com/mickael-menu/zk/internal/util/strings" - "github.com/relvacode/iso8601" - "gopkg.in/djherbis/times.v1" ) // NoteIndex persists and grants access to indexed information about the notes. type NoteIndex interface { - // Find retrieves the notes matching the given filtering and sorting criteria. Find(opts NoteFindOpts) ([]ContextualNote, error) // FindMinimal retrieves lightweight metadata for the notes matching the @@ -30,7 +24,7 @@ type NoteIndex interface { // Indexed returns the list of indexed note file metadata. IndexedPaths() (<-chan paths.Metadata, error) - // Add indexes a new note from its metadata. + // Add indexes a new note. Add(note Note) (NoteID, error) // Update resets the metadata of an already indexed note. Update(note Note) error @@ -75,11 +69,12 @@ func (s NoteIndexingStats) String() string { // indexTask indexes the notes in the given directory with the NoteIndex. type indexTask struct { - notebook *Notebook - force bool - index NoteIndex - parser NoteParser - logger util.Logger + path string + config Config + force bool + index NoteIndex + parser NoteParser + logger util.Logger } func (t *indexTask) execute(callback func(change paths.DiffChange)) (NoteIndexingStats, error) { @@ -96,7 +91,7 @@ func (t *indexTask) execute(callback func(change paths.DiffChange)) (NoteIndexin force := t.force || needsReindexing shouldIgnorePath := func(path string) (bool, error) { - group, err := t.notebook.Config.GroupConfigForPath(path) + group, err := t.config.GroupConfigForPath(path) if err != nil { return true, err } @@ -118,7 +113,7 @@ func (t *indexTask) execute(callback func(change paths.DiffChange)) (NoteIndexin return false, nil } - source := paths.Walk(t.notebook.Path, t.logger, shouldIgnorePath) + source := paths.Walk(t.path, t.logger, shouldIgnorePath) target, err := t.index.IndexedPaths() if err != nil { @@ -128,21 +123,22 @@ func (t *indexTask) execute(callback func(change paths.DiffChange)) (NoteIndexin // FIXME: Use the FS? count, err := paths.Diff(source, target, force, func(change paths.DiffChange) error { callback(change) + absPath := filepath.Join(change.Path) switch change.Kind { case paths.DiffAdded: stats.AddedCount += 1 - note, err := t.noteAt(change.Path) - if err == nil { - _, err = t.index.Add(note) + note, err := t.parser.ParseNoteAt(absPath) + if note != nil { + _, err = t.index.Add(*note) } t.logger.Err(err) case paths.DiffModified: stats.ModifiedCount += 1 - note, err := t.noteAt(change.Path) - if err == nil { - err = t.index.Update(note) + note, err := t.parser.ParseNoteAt(absPath) + if note != nil { + err = t.index.Update(*note) } t.logger.Err(err) @@ -163,81 +159,3 @@ func (t *indexTask) execute(callback func(change paths.DiffChange)) (NoteIndexin return stats, wrap(err) } - -// noteAt parses a Note at the given path. -func (t *indexTask) noteAt(path string) (Note, error) { - wrap := errors.Wrapper(path) - - note := Note{ - Path: path, - Links: []Link{}, - Tags: []string{}, - } - - absPath := filepath.Join(t.notebook.Path, path) - content, err := ioutil.ReadFile(absPath) - if err != nil { - return note, wrap(err) - } - contentStr := string(content) - contentParts, err := t.parser.Parse(contentStr) - if err != nil { - return note, wrap(err) - } - note.Title = contentParts.Title.String() - note.Lead = contentParts.Lead.String() - note.Body = contentParts.Body.String() - note.RawContent = contentStr - note.WordCount = len(strings.Fields(contentStr)) - note.Links = make([]Link, 0) - note.Tags = contentParts.Tags - note.Metadata = contentParts.Metadata - note.Checksum = fmt.Sprintf("%x", sha256.Sum256(content)) - - for _, link := range contentParts.Links { - if !strutil.IsURL(link.Href) { - // Make the href relative to the notebook root. - href := filepath.Join(filepath.Dir(absPath), link.Href) - link.Href, err = t.notebook.RelPath(href) - if err != nil { - t.logger.Err(err) - continue - } - } - note.Links = append(note.Links, link) - } - - times, err := times.Stat(absPath) - if err != nil { - return note, wrap(err) - } - - note.Modified = times.ModTime().UTC() - note.Created = t.creationDateFrom(note.Metadata, times) - - return note, nil -} - -func (t *indexTask) creationDateFrom(metadata map[string]interface{}, times times.Timespec) time.Time { - // Read the creation date from the YAML frontmatter `date` key. - if dateVal, ok := metadata["date"]; ok { - if dateStr, ok := dateVal.(string); ok { - if time, err := iso8601.ParseString(dateStr); err == nil { - return time - } - // Omitting the `T` is common - if time, err := time.Parse("2006-01-02 15:04:05", dateStr); err == nil { - return time - } - if time, err := time.Parse("2006-01-02 15:04", dateStr); err == nil { - return time - } - } - } - - if times.HasBirthTime() { - return times.BirthTime().UTC() - } - - return time.Now().UTC() -} diff --git a/internal/core/note_new_test.go b/internal/core/note_new_test.go index a005053..fcbb1d3 100644 --- a/internal/core/note_new_test.go +++ b/internal/core/note_new_test.go @@ -7,6 +7,7 @@ import ( "github.com/mickael-menu/zk/internal/util" "github.com/mickael-menu/zk/internal/util/opt" + "github.com/mickael-menu/zk/internal/util/paths" "github.com/mickael-menu/zk/internal/util/test/assert" ) @@ -16,7 +17,7 @@ func TestNotebookNewNote(t *testing.T) { } test.setup() - path, err := test.run(NewNoteOpts{ + note, err := test.run(NewNoteOpts{ Title: opt.NewString("Note title"), Content: "Note content", Extra: map[string]string{ @@ -25,11 +26,12 @@ func TestNotebookNewNote(t *testing.T) { Date: now, }) + assert.NotNil(t, note) assert.Nil(t, err) - assert.Equal(t, path, "/notebook/filename.ext") + assert.Equal(t, note.Path, "filename.ext") // Check created note. - assert.Equal(t, test.fs.files[path], "body") + assert.Equal(t, test.fs.files["/notebook/filename.ext"], "body") assert.Equal(t, test.receivedLang, test.config.Note.Lang) assert.Equal(t, test.receivedIDOpts, test.config.Note.IDOptions) @@ -105,17 +107,17 @@ func TestNotebookNewNoteInDir(t *testing.T) { } test.setup() - path, err := test.run(NewNoteOpts{ + note, err := test.run(NewNoteOpts{ Title: opt.NewString("Note title"), Directory: opt.NewString("a-dir"), Date: now, }) assert.Nil(t, err) - assert.Equal(t, path, "/notebook/a-dir/filename.ext") + assert.Equal(t, note.Path, "a-dir/filename.ext") // Check created note. - assert.Equal(t, test.fs.files[path], "body") + assert.Equal(t, test.fs.files["/notebook/a-dir/filename.ext"], "body") // Check that the templates received the proper render contexts. assert.Equal(t, test.filenameTemplate.Contexts, []interface{}{ @@ -180,16 +182,15 @@ func TestNotebookNewNoteInDirWithGroup(t *testing.T) { filenameTemplate := test.templateLoader.SpyString("group-filename.group-ext") bodyTemplate := test.templateLoader.SpyFile("group-body", "group template body") - path, err := test.run(NewNoteOpts{ + note, err := test.run(NewNoteOpts{ Directory: opt.NewString("a-dir"), Date: now, }) assert.Nil(t, err) - assert.Equal(t, path, "/notebook/a-dir/group-filename.group-ext") + assert.Equal(t, note.Path, "a-dir/group-filename.group-ext") - // Check created note. - assert.Equal(t, test.fs.files[path], "group template body") + assert.Equal(t, test.fs.files["/notebook/a-dir/group-filename.group-ext"], "group template body") assert.Equal(t, test.receivedLang, groupConfig.Note.Lang) assert.Equal(t, test.receivedIDOpts, groupConfig.Note.IDOptions) @@ -255,16 +256,16 @@ func TestNotebookNewNoteWithGroup(t *testing.T) { filenameTemplate := test.templateLoader.SpyString("group-filename.group-ext") bodyTemplate := test.templateLoader.SpyFile("group-body", "group template body") - path, err := test.run(NewNoteOpts{ + note, err := test.run(NewNoteOpts{ Group: opt.NewString("group-a"), Date: now, }) assert.Nil(t, err) - assert.Equal(t, path, "/notebook/group-filename.group-ext") + assert.Equal(t, note.Path, "group-filename.group-ext") // Check created note. - assert.Equal(t, test.fs.files[path], "group template body") + assert.Equal(t, test.fs.files["/notebook/group-filename.group-ext"], "group template body") assert.Equal(t, test.receivedLang, groupConfig.Note.Lang) assert.Equal(t, test.receivedIDOpts, groupConfig.Note.IDOptions) @@ -319,13 +320,13 @@ func TestNotebookNewNoteWithCustomTemplate(t *testing.T) { test.setup() test.templateLoader.SpyFile("custom-body", "custom body template") - path, err := test.run(NewNoteOpts{ + note, err := test.run(NewNoteOpts{ Template: opt.NewString("custom-body"), Date: now, }) assert.Nil(t, err) - assert.Equal(t, test.fs.files[path], "custom body template") + assert.Equal(t, test.fs.files["/notebook/"+note.Path], "custom body template") } // Tries to generate a filename until one is free. @@ -344,15 +345,15 @@ func TestNotebookNewNoteTriesUntilFreePath(t *testing.T) { } test.setup() - path, err := test.run(NewNoteOpts{ + note, err := test.run(NewNoteOpts{ Date: now, }) assert.Nil(t, err) - assert.Equal(t, path, "/notebook/filename4.ext") + assert.Equal(t, note.Path, "filename4.ext") // Check created note. - assert.Equal(t, test.fs.files[path], "body") + assert.Equal(t, test.fs.files["/notebook/filename4.ext"], "body") } func TestNotebookNewNoteErrorWhenNoFreePath(t *testing.T) { @@ -386,6 +387,8 @@ type newNoteTest struct { files map[string]string dirs []string fs *fileStorageMock + index *noteIndexAddMock + parser *noteContentParserMock config Config groups map[string]GroupConfig templateLoader *templateLoaderMock @@ -412,6 +415,9 @@ func (t *newNoteTest) setup() { t.fs.files = t.files } + t.index = ¬eIndexAddMock{ReturnedID: 42} + t.parser = newNoteContentParserMock(map[string]*NoteContent{}) + t.templateLoader = newTemplateLoaderMock() if t.filenameTemplateRender != nil { t.filenameTemplate = t.templateLoader.Spy("filename.ext", func(context interface{}) string { @@ -459,7 +465,11 @@ func (t *newNoteTest) setup() { } } -func (t *newNoteTest) run(opts NewNoteOpts) (string, error) { +func (t *newNoteTest) parseContentAsNote(content string, note *NoteContent) { + t.parser.results[content] = note +} + +func (t *newNoteTest) run(opts NewNoteOpts) (*Note, error) { notebook := NewNotebook(t.rootDir, t.config, NotebookPorts{ TemplateLoaderFactory: func(language string) (TemplateLoader, error) { t.receivedLang = language @@ -469,9 +479,11 @@ func (t *newNoteTest) run(opts NewNoteOpts) (string, error) { t.receivedIDOpts = opts return t.idGeneratorFactory(opts) }, - FS: t.fs, - Logger: &util.NullLogger, - OSEnv: func() map[string]string { return t.osEnv }, + FS: t.fs, + NoteIndex: t.index, + NoteContentParser: t.parser, + Logger: &util.NullLogger, + OSEnv: func() map[string]string { return t.osEnv }, }) return notebook.NewNote(opts) @@ -485,3 +497,20 @@ func incrementingID(opts IDOptions) func() string { return fmt.Sprintf("%d", i) } } + +type noteIndexAddMock struct { + ReturnedID NoteID +} + +func (m *noteIndexAddMock) Find(opts NoteFindOpts) ([]ContextualNote, error) { return nil, nil } +func (m *noteIndexAddMock) FindMinimal(opts NoteFindOpts) ([]MinimalNote, error) { return nil, nil } +func (m *noteIndexAddMock) FindCollections(kind CollectionKind) ([]Collection, error) { + return nil, nil +} +func (m *noteIndexAddMock) IndexedPaths() (<-chan paths.Metadata, error) { return nil, nil } +func (m *noteIndexAddMock) Add(note Note) (NoteID, error) { return m.ReturnedID, nil } +func (m *noteIndexAddMock) Update(note Note) error { return nil } +func (m *noteIndexAddMock) Remove(path string) error { return nil } +func (m *noteIndexAddMock) Commit(transaction func(idx NoteIndex) error) error { return nil } +func (m *noteIndexAddMock) NeedsReindexing() (bool, error) { return false, nil } +func (m *noteIndexAddMock) SetNeedsReindexing(needsReindexing bool) error { return nil } diff --git a/internal/core/note_parse.go b/internal/core/note_parse.go index fbcf091..cfb1a95 100644 --- a/internal/core/note_parse.go +++ b/internal/core/note_parse.go @@ -1,16 +1,31 @@ package core import ( + "crypto/sha256" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/mickael-menu/zk/internal/util/errors" "github.com/mickael-menu/zk/internal/util/opt" + strutil "github.com/mickael-menu/zk/internal/util/strings" + "github.com/relvacode/iso8601" + "gopkg.in/djherbis/times.v1" ) -// NoteParser parses a note's raw content into its components. +// NoteParser parses a note on the file system into a Note model. type NoteParser interface { - Parse(content string) (*ParsedNote, error) + ParseNoteAt(absPath string) (*Note, error) +} + +// NoteContentParser parses a note's raw content into its components. +type NoteContentParser interface { + ParseNoteContent(content string) (*NoteContent, error) } -// ParsedNote holds the data parsed from the note content. -type ParsedNote struct { +// NoteContent holds the data parsed from the note content. +type NoteContent struct { // Title is the heading of the note. Title opt.String // Lead is the opening paragraph or section of the note. @@ -24,3 +39,83 @@ type ParsedNote struct { // Additional metadata. For example, extracted from a YAML frontmatter. Metadata map[string]interface{} } + +// ParseNoteAt implements NoteParser. +func (n *Notebook) ParseNoteAt(absPath string) (*Note, error) { + wrap := errors.Wrapper(absPath) + + relPath, err := n.RelPath(absPath) + if err != nil { + return nil, wrap(err) + } + + content, err := n.fs.Read(absPath) + if err != nil { + return nil, wrap(err) + } + contentStr := string(content) + contentParts, err := n.parser.ParseNoteContent(contentStr) + if err != nil { + return nil, wrap(err) + } + + note := Note{ + Path: relPath, + Title: contentParts.Title.String(), + Lead: contentParts.Lead.String(), + Body: contentParts.Body.String(), + RawContent: contentStr, + WordCount: len(strings.Fields(contentStr)), + Links: make([]Link, 0), + Tags: contentParts.Tags, + Metadata: contentParts.Metadata, + Checksum: fmt.Sprintf("%x", sha256.Sum256(content)), + } + + for _, link := range contentParts.Links { + if !strutil.IsURL(link.Href) { + // Make the href relative to the notebook root. + href := filepath.Join(filepath.Dir(absPath), link.Href) + link.Href, err = n.RelPath(href) + if err != nil { + n.logger.Err(err) + continue + } + } + note.Links = append(note.Links, link) + } + + times, err := times.Stat(absPath) + if err != nil { + n.logger.Err(err) + } else { + note.Modified = times.ModTime().UTC() + note.Created = creationDateFrom(note.Metadata, times) + } + + return ¬e, nil +} + +func creationDateFrom(metadata map[string]interface{}, times times.Timespec) time.Time { + // Read the creation date from the YAML frontmatter `date` key. + if dateVal, ok := metadata["date"]; ok { + if dateStr, ok := dateVal.(string); ok { + if time, err := iso8601.ParseString(dateStr); err == nil { + return time + } + // Omitting the `T` is common + if time, err := time.Parse("2006-01-02 15:04:05", dateStr); err == nil { + return time + } + if time, err := time.Parse("2006-01-02 15:04", dateStr); err == nil { + return time + } + } + } + + if times.HasBirthTime() { + return times.BirthTime().UTC() + } + + return time.Now().UTC() +} diff --git a/internal/core/note_parse_test.go b/internal/core/note_parse_test.go new file mode 100644 index 0000000..63edf67 --- /dev/null +++ b/internal/core/note_parse_test.go @@ -0,0 +1,18 @@ +package core + +type noteContentParserMock struct { + results map[string]*NoteContent +} + +func newNoteContentParserMock(results map[string]*NoteContent) *noteContentParserMock { + return ¬eContentParserMock{ + results: results, + } +} + +func (p *noteContentParserMock) ParseNoteContent(content string) (*NoteContent, error) { + if note, ok := p.results[content]; ok { + return note, nil + } + return &NoteContent{}, nil +} diff --git a/internal/core/notebook.go b/internal/core/notebook.go index 59a2363..289c31e 100644 --- a/internal/core/notebook.go +++ b/internal/core/notebook.go @@ -9,6 +9,7 @@ import ( "github.com/mickael-menu/zk/internal/util" "github.com/mickael-menu/zk/internal/util/errors" + "github.com/mickael-menu/zk/internal/util/icu" "github.com/mickael-menu/zk/internal/util/opt" "github.com/mickael-menu/zk/internal/util/paths" "github.com/schollz/progressbar/v3" @@ -20,7 +21,7 @@ type Notebook struct { Config Config index NoteIndex - parser NoteParser + parser NoteContentParser templateLoaderFactory TemplateLoaderFactory idGeneratorFactory IDGeneratorFactory fs FileStorage @@ -38,7 +39,7 @@ func NewNotebook( Path: path, Config: config, index: ports.NoteIndex, - parser: ports.NoteParser, + parser: ports.NoteContentParser, templateLoaderFactory: ports.TemplateLoaderFactory, idGeneratorFactory: ports.IDGeneratorFactory, fs: ports.FS, @@ -49,7 +50,7 @@ func NewNotebook( type NotebookPorts struct { NoteIndex NoteIndex - NoteParser NoteParser + NoteContentParser NoteContentParser TemplateLoaderFactory TemplateLoaderFactory IDGeneratorFactory IDGeneratorFactory FS FileStorage @@ -72,11 +73,12 @@ func (n *Notebook) Index(force bool) (stats NoteIndexingStats, err error) { err = n.index.Commit(func(index NoteIndex) error { task := indexTask{ - notebook: n, - force: force, - index: index, - parser: n.parser, - logger: n.logger, + path: n.Path, + config: n.Config, + force: force, + index: index, + parser: n, + logger: n.logger, } stats, err = task.execute(func(change paths.DiffChange) { bar.Add(1) @@ -119,20 +121,20 @@ func (e ErrNoteExists) Error() string { return fmt.Sprintf("%s: note already exists", e.Path) } -// NewNote generates a new note in the notebook and returns its path. +// NewNote generates a new note in the notebook, index and returns it. // // Returns ErrNoteExists if no free filename can be generated for this note. -func (n *Notebook) NewNote(opts NewNoteOpts) (string, error) { +func (n *Notebook) NewNote(opts NewNoteOpts) (*Note, error) { wrap := errors.Wrapper("new note") dir, err := n.RequireDirAt(opts.Directory.OrString(n.Path).Unwrap()) if err != nil { - return "", wrap(err) + return nil, wrap(err) } config, err := n.Config.GroupConfigNamed(opts.Group.OrString(dir.Group).Unwrap()) if err != nil { - return "", wrap(err) + return nil, wrap(err) } extra := config.Extra @@ -142,7 +144,7 @@ func (n *Notebook) NewNote(opts NewNoteOpts) (string, error) { templates, err := n.templateLoaderFactory(config.Note.Lang) if err != nil { - return "", wrap(err) + return nil, wrap(err) } task := newNoteTask{ @@ -159,7 +161,22 @@ func (n *Notebook) NewNote(opts NewNoteOpts) (string, error) { genID: n.idGeneratorFactory(config.Note.IDOptions), } path, err := task.execute() - return path, wrap(err) + if err != nil { + return nil, wrap(err) + } + + note, err := n.ParseNoteAt(path) + if note == nil || err != nil { + return nil, wrap(err) + } + + id, err := n.index.Add(*note) + if err != nil { + return nil, wrap(err) + } + + note.ID = id + return note, nil } // FindNotes retrieves the notes matching the given filtering options. @@ -167,34 +184,66 @@ func (n *Notebook) FindNotes(opts NoteFindOpts) ([]ContextualNote, error) { return n.index.Find(opts) } +// FindNote retrieves the first note matching the given filtering options. +func (n *Notebook) FindNote(opts NoteFindOpts) (*Note, error) { + opts.Limit = 1 + notes, err := n.FindNotes(opts) + switch { + case err != nil: + return nil, err + case len(notes) == 0: + return nil, nil + default: + return ¬es[0].Note, nil + } +} + // FindMinimalNotes retrieves lightweight metadata for the notes matching // the given filtering options. func (n *Notebook) FindMinimalNotes(opts NoteFindOpts) ([]MinimalNote, error) { return n.index.FindMinimal(opts) } +// FindMinimalNotes retrieves lightweight metadata for the first note matching +// the given filtering options. +func (n *Notebook) FindMinimalNote(opts NoteFindOpts) (*MinimalNote, error) { + opts.Limit = 1 + notes, err := n.FindMinimalNotes(opts) + switch { + case err != nil: + return nil, err + case len(notes) == 0: + return nil, nil + default: + return ¬es[0], nil + } +} + // FindByHref retrieves the first note matching the given link href. -func (n *Notebook) FindByHref(href string) (*MinimalNote, error) { +// If allowPartialMatch is true, the href can match any unique sub portion of a note path. +func (n *Notebook) FindByHref(href string, allowPartialMatch bool) (*MinimalNote, error) { // Remove any anchor at the end of the HREF, since it's most likely // matching a sub-section in the note. href = strings.SplitN(href, "#", 2)[0] - notes, err := n.FindMinimalNotes(NoteFindOpts{ - IncludePaths: []string{href}, - Limit: 1, + if allowPartialMatch { + href = "(.*)" + icu.EscapePattern(href) + "(.*)" + } + + return n.FindMinimalNote(NoteFindOpts{ + IncludePaths: []string{href}, + EnablePathRegexes: allowPartialMatch, // To find the best match possible, we sort by path length. // See https://github.com/mickael-menu/zk/issues/23 Sorters: []NoteSorter{{Field: NoteSortPathLength, Ascending: true}}, }) +} - switch { - case err != nil: - return nil, err - case len(notes) == 0: - return nil, nil - default: - return ¬es[0], nil - } +// FindMatching retrieves the first note matching the given search terms. +func (n *Notebook) FindMatching(terms string) (*MinimalNote, error) { + return n.FindMinimalNote(NoteFindOpts{ + Match: opt.NewNotEmptyString(terms), + }) } // FindCollections retrieves all the collections of the given kind.