mirror of https://github.com/hwchase17/langchain
Add MimeType based parser (#4376)
# Add MimeType Based Parser This PR adds a MimeType Based Parser. The parser inspects the mime-type of the blob it is parsing and based on the mime-type can delegate to the sub parser. ## Before submitting Waiting on adding notebooks until more implementations are landed. ## Who can review? Community members can review the PR once tests pass. Tag maintainers/contributors who might be interested: @hwchase17 @vowelparrotpull/4408/head
parent
c485e7ab59
commit
ae0c3382dd
@ -0,0 +1,68 @@
|
||||
"""Code for generic / auxiliary parsers.
|
||||
|
||||
This module contains some logic to help assemble more sophisticated parsers.
|
||||
"""
|
||||
from typing import Iterator, Mapping, Optional
|
||||
|
||||
from langchain.document_loaders.base import BaseBlobParser
|
||||
from langchain.document_loaders.blob_loaders.schema import Blob
|
||||
from langchain.schema import Document
|
||||
|
||||
|
||||
class MimeTypeBasedParser(BaseBlobParser):
|
||||
"""A parser that uses mime-types to determine how to parse a blob.
|
||||
|
||||
This parser is useful for simple pipelines where the mime-type is sufficient
|
||||
to determine how to parse a blob.
|
||||
|
||||
To use, configure handlers based on mime-types and pass them to the initializer.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from langchain.document_loaders.parsers.generic import MimeTypeBasedParser
|
||||
|
||||
parser = MimeTypeBasedParser(
|
||||
handlers={
|
||||
"application/pdf": ...,
|
||||
},
|
||||
fallback_parser=...,
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
handlers: Mapping[str, BaseBlobParser],
|
||||
fallback_parser: Optional[BaseBlobParser] = None,
|
||||
) -> None:
|
||||
"""Define a parser that uses mime-types to determine how to parse a blob.
|
||||
|
||||
Args:
|
||||
handlers: A mapping from mime-types to functions that take a blob, parse it
|
||||
and return a document.
|
||||
fallback_parser: A fallback_parser parser to use if the mime-type is not
|
||||
found in the handlers. If provided, this parser will be
|
||||
used to parse blobs with all mime-types not found in
|
||||
the handlers.
|
||||
If not provided, a ValueError will be raised if the
|
||||
mime-type is not found in the handlers.
|
||||
"""
|
||||
self.handlers = handlers
|
||||
self.fallback_parser = fallback_parser
|
||||
|
||||
def lazy_parse(self, blob: Blob) -> Iterator[Document]:
|
||||
"""Load documents from a blob."""
|
||||
mimetype = blob.mimetype
|
||||
|
||||
if mimetype is None:
|
||||
raise ValueError(f"{blob} does not have a mimetype.")
|
||||
|
||||
if mimetype in self.handlers:
|
||||
handler = self.handlers[mimetype]
|
||||
yield from handler.lazy_parse(blob)
|
||||
else:
|
||||
if self.fallback_parser is not None:
|
||||
yield from self.fallback_parser.lazy_parse(blob)
|
||||
else:
|
||||
raise ValueError(f"Unsupported mime type: {mimetype}")
|
@ -0,0 +1,95 @@
|
||||
"""Module to test generic parsers."""
|
||||
|
||||
from typing import Iterator
|
||||
|
||||
import pytest
|
||||
|
||||
from langchain.document_loaders.base import BaseBlobParser
|
||||
from langchain.document_loaders.blob_loaders import Blob
|
||||
from langchain.document_loaders.parsers.generic import MimeTypeBasedParser
|
||||
from langchain.schema import Document
|
||||
|
||||
|
||||
class TestMimeBasedParser:
|
||||
"""Test mime based parser."""
|
||||
|
||||
def test_without_fallback_parser(self) -> None:
|
||||
class FirstCharParser(BaseBlobParser):
|
||||
def lazy_parse(self, blob: Blob) -> Iterator[Document]:
|
||||
"""Extract the first character of a blob."""
|
||||
yield Document(page_content=blob.as_string()[0])
|
||||
|
||||
class SecondCharParser(BaseBlobParser):
|
||||
def lazy_parse(self, blob: Blob) -> Iterator[Document]:
|
||||
"""Extract the second character of a blob."""
|
||||
yield Document(page_content=blob.as_string()[1])
|
||||
|
||||
parser = MimeTypeBasedParser(
|
||||
handlers={
|
||||
"text/plain": FirstCharParser(),
|
||||
"text/html": SecondCharParser(),
|
||||
},
|
||||
)
|
||||
|
||||
blob = Blob(data=b"Hello World", mimetype="text/plain")
|
||||
docs = parser.parse(blob)
|
||||
assert len(docs) == 1
|
||||
doc = docs[0]
|
||||
assert doc.page_content == "H"
|
||||
|
||||
# Check text/html handler.
|
||||
blob = Blob(data=b"Hello World", mimetype="text/html")
|
||||
docs = parser.parse(blob)
|
||||
assert len(docs) == 1
|
||||
doc = docs[0]
|
||||
assert doc.page_content == "e"
|
||||
|
||||
blob = Blob(data=b"Hello World", mimetype="text/csv")
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported mime type"):
|
||||
# Check that the fallback parser is used when the mimetype is not found.
|
||||
parser.parse(blob)
|
||||
|
||||
def test_with_fallback_parser(self) -> None:
|
||||
class FirstCharParser(BaseBlobParser):
|
||||
def lazy_parse(self, blob: Blob) -> Iterator[Document]:
|
||||
"""Extract the first character of a blob."""
|
||||
yield Document(page_content=blob.as_string()[0])
|
||||
|
||||
class SecondCharParser(BaseBlobParser):
|
||||
def lazy_parse(self, blob: Blob) -> Iterator[Document]:
|
||||
"""Extract the second character of a blob."""
|
||||
yield Document(page_content=blob.as_string()[1])
|
||||
|
||||
class ThirdCharParser(BaseBlobParser):
|
||||
def lazy_parse(self, blob: Blob) -> Iterator[Document]:
|
||||
"""Extract the third character of a blob."""
|
||||
yield Document(page_content=blob.as_string()[2])
|
||||
|
||||
parser = MimeTypeBasedParser(
|
||||
handlers={
|
||||
"text/plain": FirstCharParser(),
|
||||
"text/html": SecondCharParser(),
|
||||
},
|
||||
fallback_parser=ThirdCharParser(),
|
||||
)
|
||||
|
||||
blob = Blob(data=b"Hello World", mimetype="text/plain")
|
||||
docs = parser.parse(blob)
|
||||
assert len(docs) == 1
|
||||
doc = docs[0]
|
||||
assert doc.page_content == "H"
|
||||
|
||||
# Check text/html handler.
|
||||
blob = Blob(data=b"Hello World", mimetype="text/html")
|
||||
docs = parser.parse(blob)
|
||||
assert len(docs) == 1
|
||||
doc = docs[0]
|
||||
assert doc.page_content == "e"
|
||||
|
||||
# Check that the fallback parser is used when the mimetype is not found.
|
||||
blob = Blob(data=b"Hello World", mimetype="text/csv")
|
||||
docs = parser.parse(blob)
|
||||
assert len(docs) == 1
|
||||
doc = docs[0]
|
||||
assert doc.page_content == "l"
|
Loading…
Reference in New Issue