Add the --mentioned-by filtering option (#15)

Find every note whose title is mentioned in the note you are working on with `--mentioned-by file.md`
This commit is contained in:
Mickaël Menu 2021-03-20 19:17:46 +01:00 committed by GitHub
parent ec574ff519
commit 52434f8618
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 212 additions and 92 deletions

View File

@ -15,6 +15,9 @@ All notable changes to this project will be documented in this file.
* This allows running `zk` without being in a notebook.
* By setting `ZK_NOTEBOOK_DIR` in your shell configuration file (e.g. `~/.profile`), you are declaring a default global notebook which will be used when `zk` is not in a notebook.
* When the notebook directory is set explicitly, any path given as argument will be relative to it instead of the actual working directory.
* Find every note whose title is mentioned in the note you are working on with `--mentioned-by file.md`.
* To refer to a note using several names, you can use the [YAML frontmatter key `aliases`](https://publish.obsidian.md/help/How+to/Add+aliases+to+note). For example the note titled "Artificial Intelligence" might have: `aliases: [AI, robot]`
* To find only unlinked mentions, pair it with `--no-linked-by`, e.g. `--mentioned-by file.md --no-linked-by file.md`.
### Fixed

View File

@ -3,11 +3,23 @@ package sqlite
import (
"database/sql"
_ "github.com/mattn/go-sqlite3"
sqlite "github.com/mattn/go-sqlite3"
"github.com/mickael-menu/zk/core/note"
"github.com/mickael-menu/zk/util/errors"
)
func init() {
// Register custom SQLite functions.
sql.Register("sqlite3_custom", &sqlite.SQLiteDriver{
ConnectHook: func(conn *sqlite.SQLiteConn) error {
if err := conn.RegisterFunc("mention_query", buildMentionQuery, true); err != nil {
return err
}
return nil
},
})
}
// DB holds the connections to a SQLite database.
type DB struct {
db *sql.DB
@ -26,7 +38,7 @@ func OpenInMemory() (*DB, error) {
func open(uri string) (*DB, error) {
wrap := errors.Wrapper("failed to open the database")
db, err := sql.Open("sqlite3", uri)
db, err := sql.Open("sqlite3_custom", uri)
if err != nil {
return nil, wrap(err)
}

View File

@ -271,6 +271,10 @@ func (d *NoteDAO) findIdsByPathPrefixes(paths []string) ([]core.NoteId, error) {
ids = append(ids, id)
}
}
if len(ids) == 0 {
return ids, fmt.Errorf("could not find notes at: " + strings.Join(paths, ", "))
}
return ids, nil
}
@ -329,7 +333,7 @@ func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) {
continue
}
metadata, err := d.unmarshalMetadata(metadataJSON)
metadata, err := unmarshalMetadata(metadataJSON)
if err != nil {
d.logger.Err(errors.Wrap(err, path))
}
@ -369,8 +373,6 @@ func parseListFromNullString(str sql.NullString) []string {
// expandMentionsIntoMatch finds the titles associated with the notes in opts.Mention to
// expand them into the opts.Match predicate.
func (d *NoteDAO) expandMentionsIntoMatch(opts note.FinderOpts) (note.FinderOpts, error) {
notFoundErr := fmt.Errorf("could not find notes at: " + strings.Join(opts.Mention, ","))
if opts.Mention == nil {
return opts, nil
}
@ -380,18 +382,9 @@ func (d *NoteDAO) expandMentionsIntoMatch(opts note.FinderOpts) (note.FinderOpts
if err != nil {
return opts, err
}
if len(ids) == 0 {
return opts, notFoundErr
}
// Exclude the mentioned notes from the results.
if opts.ExcludeIds == nil {
opts.ExcludeIds = ids
} else {
for _, id := range ids {
opts.ExcludeIds = append(opts.ExcludeIds, id)
}
}
opts = opts.ExcludingIds(ids...)
// Find their titles.
titlesQuery := "SELECT title, metadata FROM notes WHERE id IN (" + d.joinIds(ids, ",") + ")"
@ -401,11 +394,7 @@ func (d *NoteDAO) expandMentionsIntoMatch(opts note.FinderOpts) (note.FinderOpts
}
defer rows.Close()
titles := []string{}
appendTitle := func(t string) {
titles = append(titles, `"`+strings.ReplaceAll(t, `"`, "")+`"`)
}
mentionQueries := []string{}
for rows.Next() {
var title, metadataJSON string
@ -414,34 +403,16 @@ func (d *NoteDAO) expandMentionsIntoMatch(opts note.FinderOpts) (note.FinderOpts
return opts, err
}
appendTitle(title)
// Support `aliases` key in the YAML frontmatter, like Obsidian:
// https://publish.obsidian.md/help/How+to/Add+aliases+to+note
metadata, err := d.unmarshalMetadata(metadataJSON)
if err != nil {
d.logger.Err(err)
} else {
if aliases, ok := metadata["aliases"]; ok {
switch aliases := aliases.(type) {
case []interface{}:
for _, alias := range aliases {
appendTitle(fmt.Sprint(alias))
}
case string:
appendTitle(aliases)
}
}
}
mentionQueries = append(mentionQueries, buildMentionQuery(title, metadataJSON))
}
if len(titles) == 0 {
return opts, notFoundErr
if len(mentionQueries) == 0 {
return opts, nil
}
// Expand the titles in the match predicate.
// Expand the mention queries in the match predicate.
match := opts.Match.String()
match += " (" + strings.Join(titles, " OR ") + ")"
match += " " + strings.Join(mentionQueries, " OR ")
opts.Match = opt.NewString(match)
return opts, nil
@ -528,10 +499,10 @@ func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
}
if !opts.Match.IsNull() {
snippetCol = `snippet(notes_fts, 2, '<zk:match>', '</zk:match>', '…', 20)`
joinClauses = append(joinClauses, "JOIN notes_fts ON n.id = notes_fts.rowid")
additionalOrderTerms = append(additionalOrderTerms, `bm25(notes_fts, 1000.0, 500.0, 1.0)`)
whereExprs = append(whereExprs, "notes_fts MATCH ?")
snippetCol = `snippet(fts_match.notes_fts, 2, '<zk:match>', '</zk:match>', '…', 20)`
joinClauses = append(joinClauses, "JOIN notes_fts fts_match ON n.id = fts_match.rowid")
additionalOrderTerms = append(additionalOrderTerms, `bm25(fts_match.notes_fts, 1000.0, 500.0, 1.0)`)
whereExprs = append(whereExprs, "fts_match.notes_fts MATCH ?")
args = append(args, fts5.ConvertQuery(opts.Match.String()))
}
@ -553,10 +524,6 @@ func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
whereExprs = append(whereExprs, strings.Join(regexes, " AND "))
}
if opts.ExcludeIds != nil {
whereExprs = append(whereExprs, "n.id NOT IN ("+d.joinIds(opts.ExcludeIds, ",")+")")
}
if opts.Tags != nil {
separatorRegex := regexp.MustCompile(`(\ OR\ )|\|`)
for _, tagsArg := range opts.Tags {
@ -605,6 +572,19 @@ WHERE collection_id IN (SELECT id FROM collections t WHERE kind = '%s' AND (%s))
}
}
if opts.MentionedBy != nil {
ids, err := d.findIdsByPathPrefixes(opts.MentionedBy)
if err != nil {
return nil, err
}
// Exclude the mentioning notes from the results.
opts = opts.ExcludingIds(ids...)
snippetCol = `snippet(nsrc.notes_fts, 2, '<zk:match>', '</zk:match>', '…', 20)`
joinClauses = append(joinClauses, "JOIN notes_fts nsrc ON nsrc.rowid IN ("+d.joinIds(ids, ",")+") AND nsrc.notes_fts MATCH mention_query(n.title, n.metadata)")
}
if opts.LinkedBy != nil {
filter := opts.LinkedBy
maxDistance = filter.MaxDistance
@ -658,6 +638,10 @@ WHERE collection_id IN (SELECT id FROM collections t WHERE kind = '%s' AND (%s))
args = append(args, opts.ModifiedEnd)
}
if opts.ExcludeIds != nil {
whereExprs = append(whereExprs, "n.id NOT IN ("+d.joinIds(opts.ExcludeIds, ",")+")")
}
orderTerms := []string{}
for _, sorter := range opts.Sorters {
orderTerms = append(orderTerms, orderTerm(sorter))
@ -771,8 +755,50 @@ func (d *NoteDAO) joinIds(ids []core.NoteId, delimiter string) string {
return strings.Join(strs, delimiter)
}
func (d *NoteDAO) unmarshalMetadata(metadataJSON string) (metadata map[string]interface{}, err error) {
func unmarshalMetadata(metadataJSON string) (metadata map[string]interface{}, err error) {
err = json.Unmarshal([]byte(metadataJSON), &metadata)
err = errors.Wrapf(err, "cannot parse note metadata from JSON: %s", metadataJSON)
return
}
// buildMentionQuery creates an FTS5 predicate to match the given note's title
// (or aliases from the metadata) in the content of another note.
//
// It is exposed as a custom SQLite function as `mention_query()`.
func buildMentionQuery(title, metadataJSON string) string {
titles := []string{}
appendTitle := func(t string) {
t = strings.TrimSpace(t)
if t != "" {
// Remove double quotes in the title to avoid tripping the FTS5 parser.
titles = append(titles, `"`+strings.ReplaceAll(t, `"`, "")+`"`)
}
}
appendTitle(title)
// Support `aliases` key in the YAML frontmatter, like Obsidian:
// https://publish.obsidian.md/help/How+to/Add+aliases+to+note
metadata, err := unmarshalMetadata(metadataJSON)
if err == nil {
if aliases, ok := metadata["aliases"]; ok {
switch aliases := aliases.(type) {
case []interface{}:
for _, alias := range aliases {
appendTitle(fmt.Sprint(alias))
}
case string:
appendTitle(aliases)
}
}
}
if len(titles) == 0 {
// Return an arbitrary search term otherwise MATCH will find every note.
// Not proud of this hack but it does the job.
return "8b80252291ee418289cfc9968eb2961c"
}
return "(" + strings.Join(titles, " OR ") + ")"
}

View File

@ -649,7 +649,7 @@ func TestNoteDAOFindUnlinkedMentions(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
Mention: []string{"log/2021-01-03.md", "index.md"},
LinkTo: &note.LinkToFilter{
LinkTo: &note.LinkFilter{
Paths: []string{"log/2021-01-03.md", "index.md"},
Negate: true,
},
@ -658,10 +658,72 @@ func TestNoteDAOFindUnlinkedMentions(t *testing.T) {
)
}
func TestNoteDAOFindMentionedBy(t *testing.T) {
testNoteDAOFind(t,
note.FinderOpts{MentionedBy: []string{"ref/test/b.md", "log/2021-01-04.md"}},
[]note.Match{
{
Metadata: note.Metadata{
Path: "log/2021-01-03.md",
Title: "Daily note",
Lead: "A daily note",
Body: "A daily note\n\nWith lot of content",
RawContent: "# A daily note\nA daily note\n\nWith lot of content",
WordCount: 3,
Links: []note.Link{},
Tags: []string{"fiction", "adventure"},
Metadata: map[string]interface{}{
"author": "Dom",
},
Created: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
Modified: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
Checksum: "qwfpgj",
},
Snippets: []string{"A second <zk:match>daily note</zk:match>"},
},
{
Metadata: note.Metadata{
Path: "index.md",
Title: "Index",
Lead: "Index of the Zettelkasten",
Body: "Index of the Zettelkasten",
RawContent: "# Index\nIndex of the Zettelkasten",
WordCount: 4,
Links: []note.Link{},
Tags: []string{},
Metadata: map[string]interface{}{
"aliases": []interface{}{
"First page",
},
},
Created: time.Date(2019, 12, 4, 11, 59, 11, 0, time.UTC),
Modified: time.Date(2019, 12, 4, 12, 17, 21, 0, time.UTC),
Checksum: "iaefhv",
},
Snippets: []string{"This one is in a sub sub directory, not the <zk:match>first page</zk:match>"},
},
},
)
}
// Common use case: `--mentioned-by x --no-linked-by x`
func TestNoteDAOFindUnlinkedMentionedBy(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
MentionedBy: []string{"ref/test/b.md", "log/2021-01-04.md"},
LinkedBy: &note.LinkFilter{
Paths: []string{"ref/test/b.md", "log/2021-01-04.md"},
Negate: true,
},
},
[]string{"log/2021-01-03.md"},
)
}
func TestNoteDAOFindLinkedBy(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
LinkedBy: &note.LinkedByFilter{
LinkedBy: &note.LinkFilter{
Paths: []string{"f39c8.md", "log/2021-01-03"},
Negate: false,
Recursive: false,
@ -674,7 +736,7 @@ func TestNoteDAOFindLinkedBy(t *testing.T) {
func TestNoteDAOFindLinkedByRecursive(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
LinkedBy: &note.LinkedByFilter{
LinkedBy: &note.LinkFilter{
Paths: []string{"log/2021-01-04.md"},
Negate: false,
Recursive: true,
@ -687,7 +749,7 @@ func TestNoteDAOFindLinkedByRecursive(t *testing.T) {
func TestNoteDAOFindLinkedByRecursiveWithMaxDistance(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
LinkedBy: &note.LinkedByFilter{
LinkedBy: &note.LinkFilter{
Paths: []string{"log/2021-01-04.md"},
Negate: false,
Recursive: true,
@ -701,7 +763,7 @@ func TestNoteDAOFindLinkedByRecursiveWithMaxDistance(t *testing.T) {
func TestNoteDAOFindLinkedByWithSnippets(t *testing.T) {
testNoteDAOFind(t,
note.FinderOpts{
LinkedBy: &note.LinkedByFilter{Paths: []string{"f39c8.md"}},
LinkedBy: &note.LinkFilter{Paths: []string{"f39c8.md"}},
},
[]note.Match{
{
@ -754,7 +816,7 @@ func TestNoteDAOFindLinkedByWithSnippets(t *testing.T) {
func TestNoteDAOFindNotLinkedBy(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
LinkedBy: &note.LinkedByFilter{
LinkedBy: &note.LinkFilter{
Paths: []string{"f39c8.md", "log/2021-01-03"},
Negate: true,
Recursive: false,
@ -767,7 +829,7 @@ func TestNoteDAOFindNotLinkedBy(t *testing.T) {
func TestNoteDAOFindLinkTo(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
LinkTo: &note.LinkToFilter{
LinkTo: &note.LinkFilter{
Paths: []string{"log/2021-01-04", "ref/test/a.md"},
Negate: false,
Recursive: false,
@ -780,7 +842,7 @@ func TestNoteDAOFindLinkTo(t *testing.T) {
func TestNoteDAOFindLinkToRecursive(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
LinkTo: &note.LinkToFilter{
LinkTo: &note.LinkFilter{
Paths: []string{"log/2021-01-04.md"},
Negate: false,
Recursive: true,
@ -793,7 +855,7 @@ func TestNoteDAOFindLinkToRecursive(t *testing.T) {
func TestNoteDAOFindLinkToRecursiveWithMaxDistance(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
LinkTo: &note.LinkToFilter{
LinkTo: &note.LinkFilter{
Paths: []string{"log/2021-01-04.md"},
Negate: false,
Recursive: true,
@ -807,7 +869,7 @@ func TestNoteDAOFindLinkToRecursiveWithMaxDistance(t *testing.T) {
func TestNoteDAOFindNotLinkTo(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{
LinkTo: &note.LinkToFilter{Paths: []string{"log/2021-01-04", "ref/test/a.md"}, Negate: true},
LinkTo: &note.LinkFilter{Paths: []string{"log/2021-01-04", "ref/test/a.md"}, Negate: true},
},
[]string{"ref/test/b.md", "ref/test/a.md", "log/2021-02-04.md", "index.md", "log/2021-01-04.md"},
)

View File

@ -19,11 +19,12 @@ type Filtering struct {
Match string `group:filter short:m placeholder:QUERY help:"Terms to search for in the notes."`
Exclude []string `group:filter short:x placeholder:PATH help:"Ignore notes matching the given path, including its descendants."`
Tag []string `group:filter short:t help:"Find notes tagged with the given tags."`
Mention []string `group:filter placeholder:PATH help:"Find notes mentioning the title of the given ones."`
LinkedBy []string `group:filter short:l placeholder:PATH help:"Find notes which are linked by the given ones." xor:link`
NoLinkedBy []string `group:filter placeholder:PATH help:"Find notes which are not linked by the given ones." xor:link`
LinkTo []string `group:filter short:L placeholder:PATH help:"Find notes which are linking to the given ones." xor:link`
Mention []string `group:filter placeholder:PATH help:"Find notes mentioning the title of the given ones." xor:mention`
MentionedBy []string `group:filter placeholder:PATH help:"Find notes whose title is mentioned in the given ones." xor:mention`
LinkTo []string `group:filter short:l placeholder:PATH help:"Find notes which are linking to the given ones." xor:link`
NoLinkTo []string `group:filter placeholder:PATH help:"Find notes which are not linking to the given notes." xor:link`
LinkedBy []string `group:filter short:L placeholder:PATH help:"Find notes which are linked by the given ones." xor:link`
NoLinkedBy []string `group:filter placeholder:PATH help:"Find notes which are not linked by the given ones." xor:link`
Orphan bool `group:filter help:"Find notes which are not linked by any other note." xor:link`
Related []string `group:filter placeholder:PATH help:"Find notes which might be related to the given ones." xor:link`
MaxDistance int `group:filter placeholder:COUNT help:"Maximum distance between two linked notes."`
@ -63,29 +64,33 @@ func NewFinderOpts(zk *zk.Zk, filtering Filtering, sorting Sorting) (*note.Finde
opts.Mention = filtering.Mention
}
if len(filtering.MentionedBy) > 0 {
opts.MentionedBy = filtering.MentionedBy
}
if paths, ok := relPaths(zk, filtering.LinkedBy); ok {
opts.LinkedBy = &note.LinkedByFilter{
opts.LinkedBy = &note.LinkFilter{
Paths: paths,
Negate: false,
Recursive: filtering.Recursive,
MaxDistance: filtering.MaxDistance,
}
} else if paths, ok := relPaths(zk, filtering.NoLinkedBy); ok {
opts.LinkedBy = &note.LinkedByFilter{
opts.LinkedBy = &note.LinkFilter{
Paths: paths,
Negate: true,
}
}
if paths, ok := relPaths(zk, filtering.LinkTo); ok {
opts.LinkTo = &note.LinkToFilter{
opts.LinkTo = &note.LinkFilter{
Paths: paths,
Negate: false,
Recursive: filtering.Recursive,
MaxDistance: filtering.MaxDistance,
}
} else if paths, ok := relPaths(zk, filtering.NoLinkTo); ok {
opts.LinkTo = &note.LinkToFilter{
opts.LinkTo = &note.LinkFilter{
Paths: paths,
Negate: true,
}

View File

@ -33,12 +33,14 @@ type FinderOpts struct {
ExcludeIds []core.NoteId
// Filter by tags found in the notes.
Tags []string
// Filter the notes mentioning the given notes.
// Filter the notes mentioning the given ones.
Mention []string
// Filter the notes mentioned by the given ones.
MentionedBy []string
// Filter to select notes being linked by another one.
LinkedBy *LinkedByFilter
LinkedBy *LinkFilter
// Filter to select notes linking to another one.
LinkTo *LinkToFilter
LinkTo *LinkFilter
// Filter to select notes which could might be related to the given notes paths.
Related []string
// Filter to select notes having no other notes linking to them.
@ -59,6 +61,17 @@ type FinderOpts struct {
Sorters []Sorter
}
// ExcludingIds creates a new FinderOpts after adding the given ids to the list
// of excluded note ids.
func (o FinderOpts) ExcludingIds(ids ...core.NoteId) FinderOpts {
if o.ExcludeIds == nil {
o.ExcludeIds = []core.NoteId{}
}
o.ExcludeIds = append(o.ExcludeIds, ids...)
return o
}
// Match holds information about a note matching the find options.
type Match struct {
Metadata
@ -66,16 +79,8 @@ type Match struct {
Snippets []string
}
// LinkedByFilter is a note filter used to select notes being linked by another one.
type LinkedByFilter struct {
Paths []string
Negate bool
Recursive bool
MaxDistance int
}
// LinkToFilter is a note filter used to select notes linking to another one.
type LinkToFilter struct {
// LinkFilter is a note filter used to select notes linking to other ones.
type LinkFilter struct {
Paths []string
Negate bool
Recursive bool

View File

@ -157,14 +157,14 @@ This is such a useful command, that an alias might be helpful.
bl = "zk list --link-to $@"
```
### Locate unlinked mentions of a note
### Locate unlinked mentions in a note
This alias can help you look for potential new links to establish, by listing every mention of a note in your notebook which is not already linked to it.
This alias can help you look for potential new links to establish, by listing every note whose title is mentioned in the note you are working on but which are not already linked to it.
Note that we are using a single argument `$1` which is repeated for both options.
```toml
unlinked-mentions = "zk list --mention $1 --no-link-to $1"
unlinked-mentions = "zk list --mentioned-by $1 --no-linked-by $1"
```
### Browse the Git history of selected notes

View File

@ -158,7 +158,7 @@ You can filter by range instead, using `--created-before`, `--created-after`, `-
You can use the following options to explore the web of links spanning your [notebook](notebook.md).
`--linked-by <path>` (or `-l`) finds the notes linked by the given one, while `--link-to <path>` (or `-L`) searches the notes having a link to it (also known as *backlinks*).
`--linked-by <path>` (or `-L`) finds the notes linked by the given one, while `--link-to <path>` (or `-l`) searches the notes having a link to it (also known as *backlinks*).
```
--linked-by 200911172034
@ -181,15 +181,15 @@ Part of writing a great notebook is to establish links between related notes. Th
--related 200911172034
```
## Locate mentions in other notes
## Locate mentions of other notes
Another great way to look for potential new links is to find every mention of a note in your notebook.
Another great way to look for potential new links is to find every mention of other notes in the note you are currently working on.
```
--mention 200911172034
--mentioned-by 200911172034
```
This option will locate the notes containing the note's title. To refer to a note using several names, you can use the [YAML frontmatter](note-frontmatter.md) to declare additional aliases. For example, a note titled "Artificial Intelligence" might have for aliases "AI" and "robot". This method is compatible with [Obsidian](https://publish.obsidian.md/help/How+to/Add+aliases+to+note).
This option will find every note whose title is mentioned in the given note. To refer to a note using several names, you can use the [YAML frontmatter](note-frontmatter.md) to declare additional aliases. For example, a note titled "Artificial Intelligence" might have for aliases "AI" and "robot". This method is compatible with [Obsidian](https://publish.obsidian.md/help/How+to/Add+aliases+to+note).
```
---
@ -198,9 +198,16 @@ aliases: [AI, robot]
---
```
To find only unlinked mentions, pair the `--mention` option with `--no-link-to` to remove notes which are already linked from the results.
Alternatively, find every note mentioning the given note with `--mention`.
```
--mention 200911172034
```
To find only unlinked mentions, pair the `--mentioned-by` and `--mentions` options with `--no-linked-by` (resp. `--no-link-to`) to remove notes which are already linked from the results.
```
--mentioned-by 200911172034 --no-linked-by 200911172034
--mention 200911172034 --no-link-to 200911172034
```