Archives
/
zk
mirror of https://github.com/mickael-menu/zk
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Extract contextual snippets for link filters

Browse Source
pull/6/head
Mickaël Menu 3 years ago
parent fb78993fbc
commit b011d6adc5
No known key found for this signature in database
GPG Key ID: 53D73664CD359895
6 changed files with 157 additions and 74 deletions
Show Stats Download Patch File Download Diff File

8
adapter/sqlite/fixtures/default/links.yml
Unescape Escape View File

@ -37,3 +37,11 @@
href: "ref/test/a"
external: false
snippet: "[[Link from 4 to 6]]"
- id: 6
source_id: 4
target_id: 6
title: "Duplicated link"
href: "ref/test/a"
external: false
snippet: "[[Duplicated link]]"

106
adapter/sqlite/note_dao.go
Unescape Escape View File

@ -295,20 +295,26 @@ func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) {
for rows.Next() {
var (
id, wordCount int
title, lead, body, rawContent, snippet string
path, checksum string
created, modified time.Time
id, wordCount int
title, lead, body, rawContent string
nullableSnippets sql.NullString
path, checksum string
created, modified time.Time
)
err := rows.Scan(&id, &path, &title, &lead, &body, &rawContent, &wordCount, &created, &modified, &checksum, &snippet)
err := rows.Scan(&id, &path, &title, &lead, &body, &rawContent, &wordCount, &created, &modified, &checksum, &nullableSnippets)
if err != nil {
d.logger.Err(err)
continue
}
snippets := make([]string, 0)
if nullableSnippets.Valid && nullableSnippets.String != "" {
snippets = strings.Split(nullableSnippets.String, "\x01")
}
matches = append(matches, note.Match{
Snippet: snippet,
Snippets: snippets,
Metadata: note.Metadata{
Path: path,
Title: title,
@ -326,24 +332,53 @@ func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) {
return matches, nil
}
type findQuery struct {
SnippetCol string
WhereExprs []string
OrderTerms []string
Args []interface{}
}
func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
snippetCol := `n.lead`
joinClauses := make([]string, 0)
whereExprs := make([]string, 0)
orderTerms := make([]string, 0)
groupById := false
args := make([]interface{}, 0)
setupLinkFilter := func(paths []string, forward, negate bool) error {
ids, err := d.findIdsByPathPrefixes(paths)
if err != nil {
return err
}
if len(ids) == 0 {
return nil
}
idsList := "(" + strutil.JoinInt64(ids, ",") + ")"
from := "source_id"
to := "target_id"
if !forward {
from, to = to, from
}
if !negate {
groupById = true
joinClauses = append(joinClauses, fmt.Sprintf(`LEFT JOIN links l ON n.id = l.%s AND l.%s IN %v`, from, to, idsList))
snippetCol = "GROUP_CONCAT(REPLACE(l.snippet, l.title, '<zk:match>' || l.title || '</zk:match>'), '\x01') AS snippet"
}
expr := "n.id"
if negate {
expr += " NOT"
}
expr += fmt.Sprintf(" IN (SELECT %v FROM links WHERE target_id IS NOT NULL AND %v IN %v)", from, to, idsList)
whereExprs = append(whereExprs, expr)
return nil
}
for _, filter := range opts.Filters {
switch filter := filter.(type) {
case note.MatchFilter:
snippetCol = `snippet(notes_fts, 2, '<zk:match>', '</zk:match>', '…', 20) as snippet`
joinClauses = append(joinClauses, "JOIN notes_fts ON n.id = notes_fts.rowid")
orderTerms = append(orderTerms, `bm25(notes_fts, 1000.0, 500.0, 1.0)`)
whereExprs = append(whereExprs, "notes_fts MATCH ?")
args = append(args, fts5.ConvertQuery(string(filter)))
@ -371,44 +406,16 @@ func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
whereExprs = append(whereExprs, strings.Join(globs, " AND "))
case note.LinkedByFilter:
ids, err := d.findIdsByPathPrefixes(filter.Paths)
err := setupLinkFilter(filter.Paths, false, filter.Negate)
if err != nil {
return nil, err
}
if len(ids) == 0 {
break
}
expr := "n.id"
if filter.Negate {
expr += " NOT"
}
expr += fmt.Sprintf(
" IN (SELECT target_id FROM links WHERE target_id IS NOT NULL AND source_id IN %v)",
"("+strutil.JoinInt64(ids, ",")+")",
)
whereExprs = append(whereExprs, expr)
case note.LinkingToFilter:
ids, err := d.findIdsByPathPrefixes(filter.Paths)
err := setupLinkFilter(filter.Paths, true, filter.Negate)
if err != nil {
return nil, err
}
if len(ids) == 0 {
break
}
expr := "n.id"
if filter.Negate {
expr += " NOT"
}
expr += fmt.Sprintf(
" IN (SELECT source_id FROM links WHERE target_id IN %v)",
"("+strutil.JoinInt64(ids, ",")+")",
)
whereExprs = append(whereExprs, expr)
case note.OrphanFilter:
whereExprs = append(whereExprs, `n.id NOT IN (
@ -443,15 +450,20 @@ func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
query := "SELECT n.id, n.path, n.title, n.lead, n.body, n.raw_content, n.word_count, n.created, n.modified, n.checksum, " + snippetCol
query += `
FROM notes n
JOIN notes_fts
ON n.id = notes_fts.rowid`
query += "\nFROM notes n"
for _, clause := range joinClauses {
query += "\n" + clause
}
if len(whereExprs) > 0 {
query += "\nWHERE " + strings.Join(whereExprs, "\nAND ")
}
if groupById {
query += "\nGROUP BY n.id"
}
query += "\nORDER BY " + strings.Join(orderTerms, ", ")
if opts.Limit > 0 {

53
adapter/sqlite/note_dao_test.go
Unescape Escape View File

@ -418,7 +418,7 @@ func TestNoteDAOFindMatch(t *testing.T) {
Modified: time.Date(2019, 12, 4, 12, 17, 21, 0, time.UTC),
Checksum: "iaefhv",
},
Snippet: "<zk:match>Index</zk:match> of the Zettelkasten",
Snippets: []string{"<zk:match>Index</zk:match> of the Zettelkasten"},
},
{
Metadata: note.Metadata{
@ -432,7 +432,7 @@ func TestNoteDAOFindMatch(t *testing.T) {
Modified: time.Date(2020, 11, 10, 8, 20, 18, 0, time.UTC),
Checksum: "earkte",
},
Snippet: "A third <zk:match>daily</zk:match> note",
Snippets: []string{"A third <zk:match>daily</zk:match> note"},
},
{
Metadata: note.Metadata{
@ -446,7 +446,7 @@ func TestNoteDAOFindMatch(t *testing.T) {
Modified: time.Date(2020, 11, 29, 8, 20, 18, 0, time.UTC),
Checksum: "arstde",
},
Snippet: "A second <zk:match>daily</zk:match> note",
Snippets: []string{"A second <zk:match>daily</zk:match> note"},
},
{
Metadata: note.Metadata{
@ -460,7 +460,7 @@ func TestNoteDAOFindMatch(t *testing.T) {
Modified: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
Checksum: "qwfpgj",
},
Snippet: "A <zk:match>daily</zk:match> note\n\nWith lot of content",
Snippets: []string{"A <zk:match>daily</zk:match> note\n\nWith lot of content"},
},
},
)
@ -511,6 +511,51 @@ func TestNoteDAOFindLinkedBy(t *testing.T) {
)
}
func TestNoteDAOFindLinkedByWithSnippets(t *testing.T) {
testNoteDAOFind(t,
note.FinderOpts{
Filters: []note.Filter{note.LinkedByFilter{Paths: []string{"f39c8.md"}}},
},
[]note.Match{
{
Metadata: note.Metadata{
Path: "ref/test/a.md",
Title: "Another nested note",
Lead: "It shall appear before b.md",
Body: "It shall appear before b.md",
RawContent: "#Another nested note\nIt shall appear before b.md",
WordCount: 5,
Links: nil,
Created: time.Date(2019, 11, 20, 20, 32, 56, 0, time.UTC),
Modified: time.Date(2019, 11, 20, 20, 34, 6, 0, time.UTC),
Checksum: "iecywst",
},
Snippets: []string{
"[[<zk:match>Link from 4 to 6</zk:match>]]",
"[[<zk:match>Duplicated link</zk:match>]]",
},
},
{
Metadata: note.Metadata{
Path: "log/2021-01-03.md",
Title: "January 3, 2021",
Lead: "A daily note",
Body: "A daily note\n\nWith lot of content",
RawContent: "# A daily note\nA daily note\n\nWith lot of content",
WordCount: 3,
Links: nil,
Created: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
Modified: time.Date(2020, 11, 22, 16, 27, 45, 0, time.UTC),
Checksum: "qwfpgj",
},
Snippets: []string{
"[[<zk:match>Another link</zk:match>]]",
},
},
},
)
}
func TestNoteDAOFindNotLinkedBy(t *testing.T) {
testNoteDAOFindPaths(t,
note.FinderOpts{

4
core/note/find.go
Unescape Escape View File

@ -24,8 +24,8 @@ type FinderOpts struct {
// Match holds information about a note matching the find options.
type Match struct {
Metadata
// Snippet is an excerpt of the note.
Snippet string
// Snippets are relevant excerpts in the note.
Snippets []string
}
// Filter is a sealed interface implemented by Finder filter criteria.

31
core/note/format.go
Unescape Escape View File

@ -63,20 +63,26 @@ var formatTemplates = map[string]string{
"short": `{{style "title" title}} {{style "path" path}} ({{date created "elapsed"}})
{{prepend " " snippet}}
{{#each snippets}}
‣ {{.}}
{{/each}}
`,
"medium": `{{style "title" title}} {{style "path" path}}
Created: {{date created "short"}}
{{prepend " " snippet}}
{{#each snippets}}
‣ {{.}}
{{/each}}
`,
"long": `{{style "title" title}} {{style "path" path}}
Created: {{date created "short"}}
Modified: {{date created "short"}}
{{prepend " " snippet}}
{{#each snippets}}
‣ {{.}}
{{/each}}
`,
"full": `{{style "title" title}} {{style "path" path}}
@ -96,14 +102,17 @@ func (f *Formatter) Format(match Match) (string, error) {
return "", err
}
snippets := make([]string, 0)
for _, snippet := range match.Snippets {
snippets = append(snippets, termRegex.ReplaceAllString(snippet, f.snippetTermReplacement))
}
return f.renderer.Render(formatRenderContext{
Path: path,
Title: match.Title,
Lead: match.Lead,
Body: match.Body,
Snippet: strings.TrimSpace(
termRegex.ReplaceAllString(match.Snippet, f.snippetTermReplacement),
),
Path: path,
Title: match.Title,
Lead: match.Lead,
Body: match.Body,
Snippets: snippets,
RawContent: match.RawContent,
WordCount: match.WordCount,
Created: match.Created,
@ -117,7 +126,7 @@ type formatRenderContext struct {
Title string
Lead string
Body string
Snippet string
Snippets []string
RawContent string `handlebars:"raw-content"`
WordCount int `handlebars:"word-count"`
Created time.Time

29
core/note/format_test.go
Unescape Escape View File

@ -21,7 +21,9 @@ func TestDefaultFormat(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, res, `{{style "title" title}} {{style "path" path}} ({{date created "elapsed"}})
{{prepend " " snippet}}
{{#each snippets}}
‣ {{.}}
{{/each}}
`)
}
@ -40,20 +42,26 @@ func TestFormats(t *testing.T) {
test("short", `{{style "title" title}} {{style "path" path}} ({{date created "elapsed"}})
{{prepend " " snippet}}
{{#each snippets}}
‣ {{.}}
{{/each}}
`)
test("medium", `{{style "title" title}} {{style "path" path}}
Created: {{date created "short"}}
{{prepend " " snippet}}
{{#each snippets}}
‣ {{.}}
{{/each}}
`)
test("long", `{{style "title" title}} {{style "path" path}}
Created: {{date created "short"}}
Modified: {{date created "short"}}
{{prepend " " snippet}}
{{#each snippets}}
‣ {{.}}
{{/each}}
`)
test("full", `{{style "title" title}} {{style "path" path}}
@ -77,7 +85,7 @@ func TestFormatRenderContext(t *testing.T) {
f, templs := newFormatter(t, opt.NewString("path"))
_, err := f.Format(Match{
Snippet: "Note snippet",
Snippets: []string{"Note snippet"},
Metadata: Metadata{
Path: "dir/note.md",
Title: "Note title",
@ -100,7 +108,7 @@ func TestFormatRenderContext(t *testing.T) {
Title: "Note title",
Lead: "Lead paragraph",
Body: "Note body",
Snippet: "Note snippet",
Snippets: []string{"Note snippet"},
RawContent: "Raw content",
WordCount: 42,
Created: Now,
@ -119,7 +127,8 @@ func TestFormatPath(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, templs.Contexts, []interface{}{
formatRenderContext{
Path: expected,
Path: expected,
Snippets: []string{},
},
})
}
@ -139,13 +148,13 @@ func TestFormatStylesSnippetTerm(t *testing.T) {
test := func(snippet string, expected string) {
f, templs := newFormatter(t, opt.NullString)
_, err := f.Format(Match{
Snippet: snippet,
Snippets: []string{snippet},
})
assert.Nil(t, err)
assert.Equal(t, templs.Contexts, []interface{}{
formatRenderContext{
Path: ".",
Snippet: expected,
Path: ".",
Snippets: []string{expected},
},
})
}

Write Preview
Loading…
Cancel
Save