langchain/cookbook/airbyte_github.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install -qU langchain-airbyte langchain_chroma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import getpass\n",
    "\n",
    "GITHUB_TOKEN = getpass.getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_airbyte import AirbyteLoader\n",
    "from langchain_core.prompts import PromptTemplate\n",
    "\n",
    "loader = AirbyteLoader(\n",
    "    source=\"source-github\",\n",
    "    stream=\"pull_requests\",\n",
    "    config={\n",
    "        \"credentials\": {\"personal_access_token\": GITHUB_TOKEN},\n",
    "        \"repositories\": [\"langchain-ai/langchain\"],\n",
    "    },\n",
    "    template=PromptTemplate.from_template(\n",
    "        \"\"\"# {title}\n",
    "by {user[login]}\n",
    "\n",
    "{body}\"\"\"\n",
    "    ),\n",
    "    include_metadata=False,\n",
    ")\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Updated partners/ibm README\n",
      "by williamdevena\n",
      "\n",
      "## PR title\n",
      "partners: changed the README file for the IBM Watson AI integration in the libs/partners/ibm folder.\n",
      "\n",
      "## PR message\n",
      "Description: Changed the README file of partners/ibm following the docs on https://python.langchain.com/docs/integrations/llms/ibm_watsonx\n",
      "\n",
      "The README includes:\n",
      "\n",
      "- Brief description\n",
      "- Installation\n",
      "- Setting-up instructions (API key, project id, ...)\n",
      "- Basic usage:\n",
      "  - Loading the model\n",
      "  - Direct inference\n",
      "  - Chain invoking\n",
      "  - Streaming the model output\n",
      "  \n",
      "Issue: https://github.com/langchain-ai/langchain/issues/17545\n",
      "\n",
      "Dependencies: None\n",
      "\n",
      "Twitter handle: None\n"
     ]
    }
   ],
   "source": [
    "print(docs[-2].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10283"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tiktoken\n",
    "from langchain_chroma import Chroma\n",
    "from langchain_openai import OpenAIEmbeddings\n",
    "\n",
    "enc = tiktoken.get_encoding(\"cl100k_base\")\n",
    "\n",
    "vectorstore = Chroma.from_documents(\n",
    "    docs,\n",
    "    embedding=OpenAIEmbeddings(\n",
    "        disallowed_special=(enc.special_tokens_set - {\"<|endofprompt|>\"})\n",
    "    ),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever = vectorstore.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='# Updated partners/ibm README\\nby williamdevena\\n\\n## PR title\\r\\npartners: changed the README file for the IBM Watson AI integration in the libs/partners/ibm folder.\\r\\n\\r\\n## PR message\\r\\nDescription: Changed the README file of partners/ibm following the docs on https://python.langchain.com/docs/integrations/llms/ibm_watsonx\\r\\n\\r\\nThe README includes:\\r\\n\\r\\n- Brief description\\r\\n- Installation\\r\\n- Setting-up instructions (API key, project id, ...)\\r\\n- Basic usage:\\r\\n  - Loading the model\\r\\n  - Direct inference\\r\\n  - Chain invoking\\r\\n  - Streaming the model output\\r\\n  \\r\\nIssue: https://github.com/langchain-ai/langchain/issues/17545\\r\\n\\r\\nDependencies: None\\r\\n\\r\\nTwitter handle: None'),\n",
       " Document(page_content='# Updated partners/ibm README\\nby williamdevena\\n\\n## PR title\\r\\npartners: changed the README file for the IBM Watson AI integration in the `libs/partners/ibm` folder. \\r\\n\\r\\n\\r\\n\\r\\n## PR message\\r\\n- **Description:** Changed the README file of partners/ibm following the docs on https://python.langchain.com/docs/integrations/llms/ibm_watsonx\\r\\n\\r\\n    The README includes:\\r\\n    - Brief description\\r\\n    - Installation\\r\\n    - Setting-up instructions (API key, project id, ...)\\r\\n    - Basic usage:\\r\\n        - Loading the model\\r\\n        - Direct inference\\r\\n        - Chain invoking\\r\\n        - Streaming the model output\\r\\n\\r\\n\\r\\n- **Issue:** #17545\\r\\n- **Dependencies:** None\\r\\n- **Twitter handle:** None'),\n",
       " Document(page_content='# IBM: added partners package `langchain_ibm`, added llm\\nby MateuszOssGit\\n\\n  - **Description:** Added `langchain_ibm` as an langchain partners package of IBM [watsonx.ai](https://www.ibm.com/products/watsonx-ai) LLM provider (`WatsonxLLM`)\\r\\n  - **Dependencies:** [ibm-watsonx-ai](https://pypi.org/project/ibm-watsonx-ai/),\\r\\n  - **Tag maintainer:** : \\r\\n\\r\\nPlease make sure your PR is passing linting and testing before submitting. Run `make format`, `make lint` and `make test` to check this locally. ✅'),\n",
       " Document(page_content='# Add WatsonX support\\nby baptistebignaud\\n\\nIt is a connector to use a LLM from WatsonX.\\r\\nIt requires python SDK \"ibm-generative-ai\"\\r\\n\\r\\n(It might not be perfect since it is my first PR on a public repository 😄)')]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retriever.invoke(\"pull requests related to IBM\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}