mirror of https://github.com/hwchase17/langchain
Description: Adds tags and dataview fields to ObsidianLoader doc metadata. - Issue: #9800, #4991 - Dependencies: none - Tag maintainer: My best guess is @hwchase17 looking through the git logs - Twitter handle: I don't use twitter, sorry!pull/9791/head
parent
ce47124e8f
commit
7d1b0fbe79
@ -0,0 +1,9 @@
|
||||
---
|
||||
anArray:
|
||||
one
|
||||
- two
|
||||
- three
|
||||
tags: 'onetag', 'twotag' ]
|
||||
---
|
||||
|
||||
A document with frontmatter that isn't valid.
|
@ -0,0 +1,5 @@
|
||||
---
|
||||
tags: journal/entry, obsidian
|
||||
---
|
||||
|
||||
No other content than the frontmatter.
|
@ -0,0 +1 @@
|
||||
A markdown document with no additional metadata.
|
@ -0,0 +1,35 @@
|
||||
---
|
||||
aFloat: 13.12345
|
||||
anInt: 15
|
||||
aBool: true
|
||||
aString: string value
|
||||
anArray:
|
||||
- one
|
||||
- two
|
||||
- three
|
||||
aDict:
|
||||
dictId1: '58417'
|
||||
dictId2: 1500
|
||||
tags: [ 'onetag', 'twotag' ]
|
||||
---
|
||||
|
||||
# Tags
|
||||
|
||||
()#notatag
|
||||
#12345
|
||||
#read
|
||||
something #tagWithCases
|
||||
- #tag-with-dash
|
||||
#tag_with_underscore #tag/with/nesting
|
||||
|
||||
# Dataview
|
||||
|
||||
Here is some data in a [dataview1:: a value] line.
|
||||
Here is even more data in a (dataview2:: another value) line.
|
||||
dataview3:: more data
|
||||
notdataview4: this is not a field
|
||||
notdataview5: this is not a field
|
||||
|
||||
# Text content
|
||||
|
||||
https://example.com/blog/#not-a-tag
|
@ -0,0 +1,128 @@
|
||||
from pathlib import Path
|
||||
|
||||
from langchain.document_loaders.obsidian import ObsidianLoader
|
||||
|
||||
OBSIDIAN_EXAMPLE_PATH = Path(__file__).parent / "sample_documents" / "obsidian"
|
||||
STANDARD_METADATA_FIELDS = {
|
||||
"created",
|
||||
"path",
|
||||
"source",
|
||||
"last_accessed",
|
||||
"last_modified",
|
||||
}
|
||||
|
||||
loader = ObsidianLoader(str(OBSIDIAN_EXAMPLE_PATH))
|
||||
docs = loader.load()
|
||||
|
||||
|
||||
def test_page_content_loaded() -> None:
|
||||
"""Verify that all docs have page_content"""
|
||||
assert len(docs) == 5
|
||||
assert all(doc.page_content for doc in docs)
|
||||
|
||||
|
||||
def test_disable_collect_metadata() -> None:
|
||||
"""If collect_metadata is False, no additional metadata should be collected."""
|
||||
loader_without_metadata = ObsidianLoader(
|
||||
str(OBSIDIAN_EXAMPLE_PATH), collect_metadata=False
|
||||
)
|
||||
docs_wo = loader_without_metadata.load()
|
||||
assert len(docs_wo) == 5
|
||||
assert all(doc.page_content for doc in docs_wo)
|
||||
assert all(set(doc.metadata) == STANDARD_METADATA_FIELDS for doc in docs_wo)
|
||||
|
||||
|
||||
def test_metadata_without_frontmatter() -> None:
|
||||
"""Verify docs without frontmatter, still have basic metadata."""
|
||||
doc = next(doc for doc in docs if doc.metadata["source"] == "no_metadata.md")
|
||||
assert set(doc.metadata) == STANDARD_METADATA_FIELDS
|
||||
|
||||
|
||||
def test_metadata_with_frontmatter() -> None:
|
||||
"""Verify a standard frontmatter field is loaded."""
|
||||
doc = next(doc for doc in docs if doc.metadata["source"] == "frontmatter.md")
|
||||
assert set(doc.metadata) == STANDARD_METADATA_FIELDS | {"tags"}
|
||||
assert set(doc.metadata["tags"].split(",")) == {"journal/entry", "obsidian"}
|
||||
|
||||
|
||||
def test_metadata_with_bad_frontmatter() -> None:
|
||||
"""Verify a doc with non-yaml frontmatter."""
|
||||
doc = next(doc for doc in docs if doc.metadata["source"] == "bad_frontmatter.md")
|
||||
assert set(doc.metadata) == STANDARD_METADATA_FIELDS
|
||||
|
||||
|
||||
def test_metadata_with_tags_and_frontmatter() -> None:
|
||||
"""Verify a doc with frontmatter and tags/dataview tags are all added to
|
||||
metadata."""
|
||||
doc = next(
|
||||
doc for doc in docs if doc.metadata["source"] == "tags_and_frontmatter.md"
|
||||
)
|
||||
|
||||
FRONTMATTER_FIELDS = {
|
||||
"aBool",
|
||||
"aFloat",
|
||||
"anInt",
|
||||
"anArray",
|
||||
"aString",
|
||||
"aDict",
|
||||
"tags",
|
||||
}
|
||||
DATAVIEW_FIELDS = {"dataview1", "dataview2", "dataview3"}
|
||||
assert (
|
||||
set(doc.metadata)
|
||||
== STANDARD_METADATA_FIELDS | FRONTMATTER_FIELDS | DATAVIEW_FIELDS
|
||||
)
|
||||
|
||||
|
||||
def test_tags_in_page_content() -> None:
|
||||
"""Verify a doc with tags are included in the metadata"""
|
||||
doc = next(doc for doc in docs if doc.metadata["source"] == "no_frontmatter.md")
|
||||
assert set(doc.metadata) == STANDARD_METADATA_FIELDS | {"tags"}
|
||||
|
||||
|
||||
def test_boolean_metadata() -> None:
|
||||
"""Verify boolean metadata is loaded correctly"""
|
||||
doc = next(
|
||||
doc for doc in docs if doc.metadata["source"] == "tags_and_frontmatter.md"
|
||||
)
|
||||
assert doc.metadata["aBool"]
|
||||
|
||||
|
||||
def test_float_metadata() -> None:
|
||||
"""Verify float metadata is loaded correctly"""
|
||||
doc = next(
|
||||
doc for doc in docs if doc.metadata["source"] == "tags_and_frontmatter.md"
|
||||
)
|
||||
assert doc.metadata["aFloat"] == 13.12345
|
||||
|
||||
|
||||
def test_int_metadata() -> None:
|
||||
"""Verify int metadata is loaded correctly"""
|
||||
doc = next(
|
||||
doc for doc in docs if doc.metadata["source"] == "tags_and_frontmatter.md"
|
||||
)
|
||||
assert doc.metadata["anInt"] == 15
|
||||
|
||||
|
||||
def test_string_metadata() -> None:
|
||||
"""Verify string metadata is loaded correctly"""
|
||||
doc = next(
|
||||
doc for doc in docs if doc.metadata["source"] == "tags_and_frontmatter.md"
|
||||
)
|
||||
assert doc.metadata["aString"] == "string value"
|
||||
|
||||
|
||||
def test_array_metadata() -> None:
|
||||
"""Verify array metadata is loaded as a string"""
|
||||
doc = next(
|
||||
doc for doc in docs if doc.metadata["source"] == "tags_and_frontmatter.md"
|
||||
)
|
||||
assert doc.metadata["anArray"] == "['one', 'two', 'three']"
|
||||
|
||||
|
||||
def test_dict_metadata() -> None:
|
||||
"""Verify dict metadata is stored as a string"""
|
||||
doc = next(
|
||||
doc for doc in docs if doc.metadata["source"] == "tags_and_frontmatter.md"
|
||||
)
|
||||
assert doc.metadata["aDict"] == "{'dictId1': '58417', 'dictId2': 1500}"
|
Loading…
Reference in New Issue