2023-04-14 04:39:20 +00:00
|
|
|
{
|
|
|
|
"cells": [
|
|
|
|
{
|
|
|
|
"cell_type": "markdown",
|
|
|
|
"metadata": {},
|
|
|
|
"source": [
|
|
|
|
"# Git\n",
|
|
|
|
"\n",
|
|
|
|
"This notebook shows how to load text files from Git repository."
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "markdown",
|
|
|
|
"metadata": {},
|
|
|
|
"source": [
|
|
|
|
"## Load existing repository from disk"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": 5,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"source": [
|
|
|
|
"from git import Repo\n",
|
|
|
|
"\n",
|
|
|
|
"repo = Repo.clone_from(\n",
|
|
|
|
" \"https://github.com/hwchase17/langchain\", to_path=\"./example_data/test_repo1\"\n",
|
|
|
|
")\n",
|
|
|
|
"branch = repo.head.reference"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": null,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"source": [
|
2023-04-14 14:55:12 +00:00
|
|
|
"from langchain.document_loaders import GitLoader"
|
2023-04-14 04:39:20 +00:00
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": 6,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
2023-04-14 04:56:17 +00:00
|
|
|
"outputs": [],
|
2023-04-14 04:39:20 +00:00
|
|
|
"source": [
|
|
|
|
"loader = GitLoader(repo_path=\"./example_data/test_repo1/\", branch=branch)"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": 8,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"source": [
|
|
|
|
"data = loader.load()"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": null,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
2023-04-14 17:45:54 +00:00
|
|
|
"outputs": [],
|
2023-04-14 04:39:20 +00:00
|
|
|
"source": [
|
|
|
|
"len(data)"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": 9,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
|
|
|
"outputs": [
|
|
|
|
{
|
|
|
|
"name": "stdout",
|
|
|
|
"output_type": "stream",
|
|
|
|
"text": [
|
|
|
|
"page_content='.venv\\n.github\\n.git\\n.mypy_cache\\n.pytest_cache\\nDockerfile' metadata={'file_path': '.dockerignore', 'file_name': '.dockerignore', 'file_type': ''}\n"
|
|
|
|
]
|
|
|
|
}
|
|
|
|
],
|
|
|
|
"source": [
|
|
|
|
"print(data[0])"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "markdown",
|
|
|
|
"metadata": {},
|
|
|
|
"source": [
|
|
|
|
"## Clone repository from url"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": 5,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"source": [
|
2023-04-14 14:55:12 +00:00
|
|
|
"from langchain.document_loaders import GitLoader"
|
2023-04-14 04:39:20 +00:00
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": 2,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"source": [
|
|
|
|
"loader = GitLoader(\n",
|
|
|
|
" clone_url=\"https://github.com/hwchase17/langchain\",\n",
|
|
|
|
" repo_path=\"./example_data/test_repo2/\",\n",
|
|
|
|
" branch=\"master\",\n",
|
|
|
|
")"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": 10,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"source": [
|
|
|
|
"data = loader.load()"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": 11,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
|
|
|
"outputs": [
|
|
|
|
{
|
|
|
|
"data": {
|
|
|
|
"text/plain": [
|
2023-04-14 17:45:54 +00:00
|
|
|
"1074"
|
2023-04-14 04:39:20 +00:00
|
|
|
]
|
|
|
|
},
|
2023-04-14 17:45:54 +00:00
|
|
|
"execution_count": 11,
|
2023-04-14 04:39:20 +00:00
|
|
|
"metadata": {},
|
|
|
|
"output_type": "execute_result"
|
|
|
|
}
|
|
|
|
],
|
|
|
|
"source": [
|
|
|
|
"len(data)"
|
|
|
|
]
|
|
|
|
},
|
2023-04-14 17:45:54 +00:00
|
|
|
{
|
|
|
|
"cell_type": "markdown",
|
|
|
|
"metadata": {},
|
|
|
|
"source": [
|
|
|
|
"## Filtering files to load"
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"cell_type": "code",
|
|
|
|
"execution_count": 12,
|
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"source": [
|
|
|
|
"from langchain.document_loaders import GitLoader\n",
|
|
|
|
"\n",
|
|
|
|
"# eg. loading only python files\n",
|
|
|
|
"loader = GitLoader(repo_path=\"./example_data/test_repo1/\", file_filter=lambda file_path: file_path.endswith(\".py\"))"
|
|
|
|
]
|
|
|
|
},
|
2023-04-14 04:39:20 +00:00
|
|
|
{
|
|
|
|
"cell_type": "code",
|
|
|
|
"execution_count": null,
|
|
|
|
"metadata": {},
|
|
|
|
"outputs": [],
|
|
|
|
"source": []
|
|
|
|
}
|
|
|
|
],
|
|
|
|
"metadata": {
|
|
|
|
"kernelspec": {
|
2023-04-14 14:55:12 +00:00
|
|
|
"display_name": "Python 3 (ipykernel)",
|
2023-04-14 04:39:20 +00:00
|
|
|
"language": "python",
|
|
|
|
"name": "python3"
|
|
|
|
},
|
|
|
|
"language_info": {
|
|
|
|
"codemirror_mode": {
|
|
|
|
"name": "ipython",
|
|
|
|
"version": 3
|
|
|
|
},
|
|
|
|
"file_extension": ".py",
|
|
|
|
"mimetype": "text/x-python",
|
|
|
|
"name": "python",
|
|
|
|
"nbconvert_exporter": "python",
|
|
|
|
"pygments_lexer": "ipython3",
|
2023-04-14 17:45:54 +00:00
|
|
|
"version": "3.11.3"
|
2023-04-14 14:55:12 +00:00
|
|
|
}
|
2023-04-14 04:39:20 +00:00
|
|
|
},
|
|
|
|
"nbformat": 4,
|
|
|
|
"nbformat_minor": 2
|
|
|
|
}
|