Add recursive link filtering

adapter/sqlite/fixtures/default/links.yml

@@ -45,3 +45,19 @@
   href: "ref/test/a"
   external: false
   snippet: "[[Duplicated link]]"
+
+- id: 7
+  source_id: 2
+  target_id: 3
+  title: "A transition link"
+  href: "index.md"
+  external: false
+  snippet: "[[A transition link]]"
+
+- id: 8
+  source_id: 3
+  target_id: 4
+  title: "Another transition link"
+  href: "f39c8.md"
+  external: false
+  snippet: "[[Another transition link]]"

adapter/sqlite/note_dao.go

@@ -311,6 +311,7 @@ func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) {
 		snippets := make([]string, 0)
 		if nullableSnippets.Valid && nullableSnippets.String != "" {
 			snippets = strings.Split(nullableSnippets.String, "\x01")
+			snippets = strutil.RemoveDuplicates(snippets)
 		}
 
 		matches = append(matches, note.Match{
@@ -334,13 +335,14 @@ func (d *NoteDAO) Find(opts note.FinderOpts) ([]note.Match, error) {
 
 func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
 	snippetCol := `n.lead`
+	cteClauses := make([]string, 0)
 	joinClauses := make([]string, 0)
 	whereExprs := make([]string, 0)
 	orderTerms := make([]string, 0)
 	groupById := false
 	args := make([]interface{}, 0)
 
-	setupLinkFilter := func(paths []string, forward, negate bool) error {
+	setupLinkFilter := func(paths []string, forward, negate bool, recursive bool) error {
 		ids, err := d.findIdsByPathPrefixes(paths)
 		if err != nil {
 			return err
@@ -358,15 +360,38 @@ func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
 			alias += "n"
 		}
 
+		links_src := "links"
 		from := "source_id"
 		to := "target_id"
 		if !forward {
 			from, to = to, from
 		}
 
+		if recursive {
+			// Credit to https://inviqa.com/blog/storing-graphs-database-sql-meets-social-network
+			links_src = "links_transitive_closure"
+			orderTerms = append(orderTerms, alias+".distance")
+			cteClauses = append(cteClauses, `WITH RECURSIVE links_transitive_closure(source_id, target_id, title, snippet, distance, path) AS (
+    SELECT source_id, target_id, title, snippet,
+	       1 AS distance,
+           source_id || '.' || target_id || '.' AS path
+      FROM links
+ 
+     UNION ALL
+ 
+    SELECT tc.source_id, l.target_id, l.title, l.snippet,
+	       tc.distance + 1,
+           tc.path || l.target_id || '.' AS path
+      FROM links AS l
+      JOIN links_transitive_closure AS tc
+        ON l.source_id = tc.target_id
+     WHERE tc.path NOT LIKE '%' || l.target_id || '.%'
+)`)
+		}
+
 		if !negate {
 			groupById = true
-			joinClauses = append(joinClauses, fmt.Sprintf(`LEFT JOIN links %[1]s ON n.id = %[1]s.%[2]s AND %[1]s.%[3]s IN %[4]s`, alias, from, to, idsList))
+			joinClauses = append(joinClauses, fmt.Sprintf(`LEFT JOIN %[1]s %[2]s ON n.id = %[2]s.%[3]s AND %[2]s.%[4]s IN %[5]s`, links_src, alias, from, to, idsList))
 			snippetCol = fmt.Sprintf("GROUP_CONCAT(REPLACE(%[1]s.snippet, %[1]s.title, '<zk:match>' || %[1]s.title || '</zk:match>'), '\x01') AS snippet", alias)
 		}
 
@@ -374,7 +399,7 @@ func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
 		if negate {
 			expr += " NOT"
 		}
-		expr += fmt.Sprintf(" IN (SELECT %s FROM links WHERE target_id IS NOT NULL AND %s IN %s)", from, to, idsList)
+		expr += fmt.Sprintf(" IN (SELECT %[2]s FROM %[1]s WHERE target_id IS NOT NULL AND %[3]s IN %[4]s)", links_src, from, to, idsList)
 
 		whereExprs = append(whereExprs, expr)
 
@@ -414,13 +439,13 @@ func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
 			whereExprs = append(whereExprs, strings.Join(globs, " AND "))
 
 		case note.LinkedByFilter:
-			err := setupLinkFilter(filter.Paths, false, filter.Negate)
+			err := setupLinkFilter(filter.Paths, false, filter.Negate, filter.Recursive)
 			if err != nil {
 				return nil, err
 			}
 
 		case note.LinkingToFilter:
-			err := setupLinkFilter(filter.Paths, true, filter.Negate)
+			err := setupLinkFilter(filter.Paths, true, filter.Negate, filter.Recursive)
 			if err != nil {
 				return nil, err
 			}
@@ -456,26 +481,32 @@ func (d *NoteDAO) findRows(opts note.FinderOpts) (*sql.Rows, error) {
 	}
 	orderTerms = append(orderTerms, `n.title ASC`)
 
-	query := "SELECT n.id, n.path, n.title, n.lead, n.body, n.raw_content, n.word_count, n.created, n.modified, n.checksum, " + snippetCol
+	query := ""
 
-	query += "\nFROM notes n"
+	for _, clause := range cteClauses {
+		query += clause + "\n"
+	}
+
+	query += "SELECT n.id, n.path, n.title, n.lead, n.body, n.raw_content, n.word_count, n.created, n.modified, n.checksum, " + snippetCol + "\n"
+
+	query += "FROM notes n\n"
 
 	for _, clause := range joinClauses {
-		query += "\n" + clause
+		query += clause + "\n"
 	}
 
 	if len(whereExprs) > 0 {
-		query += "\nWHERE " + strings.Join(whereExprs, "\nAND ")
+		query += "WHERE " + strings.Join(whereExprs, "\nAND ") + "\n"
 	}
 
 	if groupById {
-		query += "\nGROUP BY n.id"
+		query += "GROUP BY n.id\n"
 	}
 
-	query += "\nORDER BY " + strings.Join(orderTerms, ", ")
+	query += "ORDER BY " + strings.Join(orderTerms, ", ") + "\n"
 
 	if opts.Limit > 0 {
-		query += fmt.Sprintf("\nLIMIT %d", opts.Limit)
+		query += fmt.Sprintf("LIMIT %d\n", opts.Limit)
 	}
 
 	// fmt.Println(query)

adapter/sqlite/note_dao_test.go

@@ -505,12 +505,29 @@ func TestNoteDAOFindExcludingMultiplePaths(t *testing.T) {
 func TestNoteDAOFindLinkedBy(t *testing.T) {
 	testNoteDAOFindPaths(t,
 		note.FinderOpts{
-			Filters: []note.Filter{note.LinkedByFilter{Paths: []string{"f39c8.md", "log/2021-01-03"}, Negate: false}},
+			Filters: []note.Filter{note.LinkedByFilter{
+				Paths:     []string{"f39c8.md", "log/2021-01-03"},
+				Negate:    false,
+				Recursive: false,
+			}},
 		},
 		[]string{"ref/test/a.md", "log/2021-01-03.md", "log/2021-01-04.md"},
 	)
 }
 
+func TestNoteDAOFindLinkedByRecursive(t *testing.T) {
+	testNoteDAOFindPaths(t,
+		note.FinderOpts{
+			Filters: []note.Filter{note.LinkedByFilter{
+				Paths:     []string{"log/2021-01-04.md"},
+				Negate:    false,
+				Recursive: true,
+			}},
+		},
+		[]string{"index.md", "f39c8.md", "ref/test/a.md", "log/2021-01-03.md"},
+	)
+}
+
 func TestNoteDAOFindLinkedByWithSnippets(t *testing.T) {
 	testNoteDAOFind(t,
 		note.FinderOpts{
@@ -559,7 +576,11 @@ func TestNoteDAOFindLinkedByWithSnippets(t *testing.T) {
 func TestNoteDAOFindNotLinkedBy(t *testing.T) {
 	testNoteDAOFindPaths(t,
 		note.FinderOpts{
-			Filters: []note.Filter{note.LinkedByFilter{Paths: []string{"f39c8.md", "log/2021-01-03"}, Negate: true}},
+			Filters: []note.Filter{note.LinkedByFilter{
+				Paths:     []string{"f39c8.md", "log/2021-01-03"},
+				Negate:    true,
+				Recursive: false,
+			}},
 		},
 		[]string{"ref/test/b.md", "f39c8.md", "log/2021-02-04.md", "index.md"},
 	)
@@ -568,12 +589,29 @@ func TestNoteDAOFindNotLinkedBy(t *testing.T) {
 func TestNoteDAOFindLinkingTo(t *testing.T) {
 	testNoteDAOFindPaths(t,
 		note.FinderOpts{
-			Filters: []note.Filter{note.LinkingToFilter{Paths: []string{"log/2021-01-04", "ref/test/a.md"}, Negate: false}},
+			Filters: []note.Filter{note.LinkingToFilter{
+				Paths:     []string{"log/2021-01-04", "ref/test/a.md"},
+				Negate:    false,
+				Recursive: false,
+			}},
 		},
 		[]string{"f39c8.md", "log/2021-01-03.md"},
 	)
 }
 
+func TestNoteDAOFindLinkingToRecursive(t *testing.T) {
+	testNoteDAOFindPaths(t,
+		note.FinderOpts{
+			Filters: []note.Filter{note.LinkingToFilter{
+				Paths:     []string{"log/2021-01-04.md"},
+				Negate:    false,
+				Recursive: true,
+			}},
+		},
+		[]string{"log/2021-01-03.md", "f39c8.md", "index.md"},
+	)
+}
+
 func TestNoteDAOFindNotLinkingTo(t *testing.T) {
 	testNoteDAOFindPaths(t,
 		note.FinderOpts{
@@ -588,7 +626,7 @@ func TestNoteDAOFindOrphan(t *testing.T) {
 		note.FinderOpts{
 			Filters: []note.Filter{note.OrphanFilter{}},
 		},
-		[]string{"ref/test/b.md", "f39c8.md", "log/2021-02-04.md", "index.md"},
+		[]string{"ref/test/b.md", "log/2021-02-04.md"},
 	)
 }
 

cmd/finder_opts.go

@@ -26,6 +26,7 @@ type Filtering struct {
 	NotLinkingTo   []string `help:"Only the notes not linking to the given notes" placeholder:"<path>"`
 	Orphan         bool     `help:"Only the notes which don't have any other note linking to them"`
 	Exclude        []string `help:"Excludes notes matching the given file path pattern from the list" short:"x" placeholder:"<glob>"`
+	Recursive      bool     `help:"Follow links recursively" short:"r"`
 	Interactive    bool     `help:"Further filter the list of notes interactively" short:"i"`
 }
 
@@ -127,16 +128,18 @@ func NewFinderOpts(zk *zk.Zk, filtering Filtering, sorting Sorting) (*note.Finde
 	linkedByPaths, ok := relPaths(zk, filtering.LinkedBy)
 	if ok {
 		filters = append(filters, note.LinkedByFilter{
-			Paths:  linkedByPaths,
-			Negate: false,
+			Paths:     linkedByPaths,
+			Negate:    false,
+			Recursive: filtering.Recursive,
 		})
 	}
 
 	linkingToPaths, ok := relPaths(zk, filtering.LinkingTo)
 	if ok {
 		filters = append(filters, note.LinkingToFilter{
-			Paths:  linkingToPaths,
-			Negate: false,
+			Paths:     linkingToPaths,
+			Negate:    false,
+			Recursive: filtering.Recursive,
 		})
 	}
 

core/note/find.go

@@ -42,14 +42,16 @@ type ExcludePathFilter []string
 
 // LinkedByFilter is a note filter used to select notes being linked by another one.
 type LinkedByFilter struct {
-	Paths  []string
-	Negate bool
+	Paths     []string
+	Negate    bool
+	Recursive bool
 }
 
 // LinkingToFilter is a note filter used to select notes being linked by another one.
 type LinkingToFilter struct {
-	Paths  []string
-	Negate bool
+	Paths     []string
+	Negate    bool
+	Recursive bool
 }
 
 // OrphanFilter is a note filter used to select notes having no other notes linking to them.

util/strings/strings.go

@@ -72,3 +72,18 @@ func IsURL(s string) bool {
 
 	return true
 }
+
+// RemoveDuplicates keeps only unique strings in the source.
+func RemoveDuplicates(strings []string) []string {
+	check := make(map[string]bool)
+	res := make([]string, 0)
+	for _, val := range strings {
+		if _, ok := check[val]; ok {
+			continue
+		}
+		check[val] = true
+		res = append(res, val)
+	}
+
+	return res
+}

util/strings/strings_test.go

@@ -80,3 +80,16 @@ func TestIsURL(t *testing.T) {
 	test("http://example.com/dir", true)
 	test("ftp://example.com/", true)
 }
+
+func TestRemoveDuplicates(t *testing.T) {
+	test := func(items []string, expected []string) {
+		assert.Equal(t, RemoveDuplicates(items), expected)
+	}
+
+	test([]string{}, []string{})
+	test([]string{"One"}, []string{"One"})
+	test([]string{"One", "Two"}, []string{"One", "Two"})
+	test([]string{"One", "Two", "One"}, []string{"One", "Two"})
+	test([]string{"Two", "One", "Two", "One"}, []string{"Two", "One"})
+	test([]string{"One", "Two", "OneTwo"}, []string{"One", "Two", "OneTwo"})
+}