From 05d8969c79b2c5031d977aafbd2808fe41aa9fa8 Mon Sep 17 00:00:00 2001 From: cragwolfe Date: Sun, 12 Feb 2023 14:56:48 -0800 Subject: [PATCH] Unstructured example notebook: add a pdf, related deps (#1011) Updates the Unstructured example notebook with a PDF example. Includes additional dependencies for PDF processing (and images, etc). --- .../examples/unstructured_file.ipynb | 77 ++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/docs/modules/document_loaders/examples/unstructured_file.ipynb b/docs/modules/document_loaders/examples/unstructured_file.ipynb index 92f043c4..afa0b18d 100644 --- a/docs/modules/document_loaders/examples/unstructured_file.ipynb +++ b/docs/modules/document_loaders/examples/unstructured_file.ipynb @@ -17,7 +17,9 @@ "outputs": [], "source": [ "# # Install package\n", - "# !pip install unstructured" + "!pip install unstructured[local-inference]\n", + "!pip install \"detectron2@git+https://github.com/facebookresearch/detectron2.git@v0.6#egg=detectron2\"\n", + "!pip install layoutparser[layoutmodels,tesseract]" ] }, { @@ -29,7 +31,12 @@ "source": [ "# # Install other dependencies\n", "# # https://github.com/Unstructured-IO/unstructured/blob/main/docs/source/installing.rst\n", - "# !brew install libmagic" + "# !brew install libmagic\n", + "# !brew install poppler\n", + "# !brew install tesseract\n", + "# # If parsing xml / html documents:\n", + "# !brew install libxml2\n", + "# !brew install libxslt" ] }, { @@ -149,6 +156,72 @@ "docs[:5]" ] }, + { + "cell_type": "markdown", + "id": "7874d01d", + "metadata": {}, + "source": [ + "## PDF Example\n", + "\n", + "Processing PDF documents works exactly the same way. Unstructured detects the file type and extracts the same types of `elements`. " + ] + }, + + { + "cell_type": "code", + "execution_count": 1, + "id": "8ca8a648", + "metadata": {}, + "outputs": [], + "source": [ + "!wget https://raw.githubusercontent.com/Unstructured-IO/unstructured/main/example-docs/layout-parser-paper.pdf -P \"../../\"" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "686e5eb4", + "metadata": {}, + "outputs": [], + "source": [ + "loader = UnstructuredFileLoader(\"../../layout-parser-paper.pdf\", mode=\"elements\")" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "f8348ca0", + "metadata": {}, + "outputs": [], + "source": [ + "docs = loader.load()" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "6ec859d8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(page_content='LayoutParser : A Uniļ¬ed Toolkit for Deep Learning Based Document Image Analysis', lookup_str='', metadata={'source': '../../layout-parser-paper.pdf'}, lookup_index=0),\n", + " Document(page_content='Zejiang Shen 1 ( (ea)\\n ), Ruochen Zhang 2 , Melissa Dell 3 , Benjamin Charles Germain Lee 4 , Jacob Carlson 3 , and Weining Li 5', lookup_str='', metadata={'source': '../../layout-parser-paper.pdf'}, lookup_index=0),\n", + " Document(page_content='Allen Institute for AI shannons@allenai.org', lookup_str='', metadata={'source': '../../layout-parser-paper.pdf'}, lookup_index=0),\n", + " Document(page_content='Brown University ruochen zhang@brown.edu', lookup_str='', metadata={'source': '../../layout-parser-paper.pdf'}, lookup_index=0),\n", + " Document(page_content='Harvard University { melissadell,jacob carlson } @fas.harvard.edu', lookup_str='', metadata={'source': '../../layout-parser-paper.pdf'}, lookup_index=0)]" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "docs[:5]" + ] + }, { "cell_type": "code", "execution_count": null,