mirror of
https://github.com/hwchase17/langchain
synced 2024-11-10 01:10:59 +00:00
192 lines
8.1 KiB
Plaintext
192 lines
8.1 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "4babfba5",
|
|
"metadata": {},
|
|
"source": [
|
|
"# GitBook\n",
|
|
"How to pull page data from any GitBook."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "ff49b177",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from langchain.document_loaders import GitbookLoader"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "849a8d52",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"loader = GitbookLoader(\"https://docs.gitbook.com\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "65d5ddce",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Load from single GitBook page"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "c2826836",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"page_data = loader.load()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "fefa2adc",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[Document(page_content='Introduction to GitBook\\nGitBook is a modern documentation platform where teams can document everything from products to internal knowledge bases and APIs.\\nWe want to help \\nteams to work more efficiently\\n by creating a simple yet powerful platform for them to \\nshare their knowledge\\n.\\nOur mission is to make a \\nuser-friendly\\n and \\ncollaborative\\n product for everyone to create, edit and share knowledge through documentation.\\nPublish your documentation in 5 easy steps\\nImport\\n\\nMove your existing content to GitBook with ease.\\nGit Sync\\n\\nBenefit from our bi-directional synchronisation with GitHub and GitLab.\\nOrganise your content\\n\\nCreate pages and spaces and organize them into collections\\nCollaborate\\n\\nInvite other users and collaborate asynchronously with ease.\\nPublish your docs\\n\\nShare your documentation with selected users or with everyone.\\nNext\\n - Getting started\\nOverview\\nLast modified \\n3mo ago', lookup_str='', metadata={'source': 'https://docs.gitbook.com', 'title': 'Introduction to GitBook'}, lookup_index=0)]"
|
|
]
|
|
},
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"page_data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "c325048c",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Load from all paths in a given GitBook\n",
|
|
"For this to work, the GitbookLoader needs to be initialized with the root path (`https://docs.gitbook.com` in this example) and have `load_all_paths` set to `True`."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"id": "938ff4ee",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Fetching text from https://docs.gitbook.com/\n",
|
|
"Fetching text from https://docs.gitbook.com/getting-started/overview\n",
|
|
"Fetching text from https://docs.gitbook.com/getting-started/import\n",
|
|
"Fetching text from https://docs.gitbook.com/getting-started/git-sync\n",
|
|
"Fetching text from https://docs.gitbook.com/getting-started/content-structure\n",
|
|
"Fetching text from https://docs.gitbook.com/getting-started/collaboration\n",
|
|
"Fetching text from https://docs.gitbook.com/getting-started/publishing\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/quick-find\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/editor\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/customization\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/member-management\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/pdf-export\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/activity-history\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/insights\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/notifications\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/internationalization\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/keyboard-shortcuts\n",
|
|
"Fetching text from https://docs.gitbook.com/tour/seo\n",
|
|
"Fetching text from https://docs.gitbook.com/advanced-guides/custom-domain\n",
|
|
"Fetching text from https://docs.gitbook.com/advanced-guides/advanced-sharing-and-security\n",
|
|
"Fetching text from https://docs.gitbook.com/advanced-guides/integrations\n",
|
|
"Fetching text from https://docs.gitbook.com/billing-and-admin/account-settings\n",
|
|
"Fetching text from https://docs.gitbook.com/billing-and-admin/plans\n",
|
|
"Fetching text from https://docs.gitbook.com/troubleshooting/faqs\n",
|
|
"Fetching text from https://docs.gitbook.com/troubleshooting/hard-refresh\n",
|
|
"Fetching text from https://docs.gitbook.com/troubleshooting/report-bugs\n",
|
|
"Fetching text from https://docs.gitbook.com/troubleshooting/connectivity-issues\n",
|
|
"Fetching text from https://docs.gitbook.com/troubleshooting/support\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"loader = GitbookLoader(\"https://docs.gitbook.com\", load_all_paths=True)\n",
|
|
"all_pages_data = loader.load()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "db92fc39",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"fetched 28 documents.\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"Document(page_content=\"Import\\nFind out how to easily migrate your existing documentation and which formats are supported.\\nThe import function allows you to migrate and unify existing documentation in GitBook. You can choose to import single or multiple pages although limits apply. \\nPermissions\\nAll members with editor permission or above can use the import feature.\\nSupported formats\\nGitBook supports imports from websites or files that are:\\nMarkdown (.md or .markdown)\\nHTML (.html)\\nMicrosoft Word (.docx).\\nWe also support import from:\\nConfluence\\nNotion\\nGitHub Wiki\\nQuip\\nDropbox Paper\\nGoogle Docs\\nYou can also upload a ZIP\\n \\ncontaining HTML or Markdown files when \\nimporting multiple pages.\\nNote: this feature is in beta.\\nFeel free to suggest import sources we don't support yet and \\nlet us know\\n if you have any issues.\\nImport panel\\nWhen you create a new space, you'll have the option to import content straight away:\\nThe new page menu\\nImport a page or subpage by selecting \\nImport Page\\n from the New Page menu, or \\nImport Subpage\\n in the page action menu, found in the table of contents:\\nImport from the page action menu\\nWhen you choose your input source, instructions will explain how to proceed.\\nAlthough GitBook supports importing content from different kinds of sources, the end result might be different from your source due to differences in product features and document format.\\nLimits\\nGitBook currently has the following limits for imported content:\\nThe maximum number of pages that can be uploaded in a single import is \\n20.\\nThe maximum number of files (images etc.) that can be uploaded in a single import is \\n20.\\nGetting started - \\nPrevious\\nOverview\\nNext\\n - Getting started\\nGit Sync\\nLast modified \\n4mo ago\", lookup_str='', metadata={'source': 'https://docs.gitbook.com/getting-started/import', 'title': 'Import'}, lookup_index=0)"
|
|
]
|
|
},
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"print(f\"fetched {len(all_pages_data)} documents.\")\n",
|
|
"# show second document\n",
|
|
"all_pages_data[2]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "92cb3eda",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.1"
|
|
},
|
|
"vscode": {
|
|
"interpreter": {
|
|
"hash": "2d002ec47225e662695b764370d7966aa11eeb4302edc2f497bbf96d49c8f899"
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|