langchain/docs/docs/integrations/chat_loaders/twitter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d86853d2",
   "metadata": {},
   "source": [
    "# Twitter (via Apify)\n",
    "\n",
    "This notebook shows how to load chat messages from Twitter to fine-tune on. We do this by utilizing Apify. \n",
    "\n",
    "First, use Apify to export tweets. An example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e5034b4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from langchain.adapters.openai import convert_message_to_dict\n",
    "from langchain_core.messages import AIMessage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8bf0fb93",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"example_data/dataset_twitter-scraper_2023-08-23_22-13-19-740.json\") as f:\n",
    "    data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "468124fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter out tweets that reference other tweets, because it's a bit weird\n",
    "tweets = [d[\"full_text\"] for d in data if \"t.co\" not in d[\"full_text\"]]\n",
    "# Create them as AI messages\n",
    "messages = [AIMessage(content=t) for t in tweets]\n",
    "# Add in a system message at the start\n",
    "# TODO: we could try to extract the subject from the tweets, and put that in the system message.\n",
    "system_message = {\"role\": \"system\", \"content\": \"write a tweet\"}\n",
    "data = [[system_message, convert_message_to_dict(m)] for m in messages]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}