mirror of https://github.com/hwchase17/langchain
I've added a bilibili loader, bilibili is a very active video site in China and I think we need this loader. Example: ```python from langchain.document_loaders.bilibili import BiliBiliLoader loader = BiliBiliLoader( ["https://www.bilibili.com/video/BV1xt411o7Xu/", "https://www.bilibili.com/video/av330407025/"] ) docs = loader.load() ``` Co-authored-by: 了空 <568250549@qq.com>pull/2730/head
parent
d42deff402
commit
709f26b69e
@ -0,0 +1,87 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "66a7777e",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Bilibili\n",
|
||||||
|
"\n",
|
||||||
|
"This loader utilizes the `bilibili-api` to fetch the text transcript from Bilibili, one of the most beloved long-form video sites in China.\n",
|
||||||
|
"\n",
|
||||||
|
"With this BiliBiliLoader, users can easily obtain the transcript of their desired video content on the platform."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 11,
|
||||||
|
"id": "9ec8a3b3",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from langchain.document_loaders.bilibili import BiliBiliLoader"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 12,
|
||||||
|
"id": "43128d8d",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"#!pip install bilibili-api"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 13,
|
||||||
|
"id": "35d6809a",
|
||||||
|
"metadata": {
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"loader = BiliBiliLoader(\n",
|
||||||
|
" [\"https://www.bilibili.com/video/BV1xt411o7Xu/\"]\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"loader.load()"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"collapsed": false,
|
||||||
|
"pycharm": {
|
||||||
|
"name": "#%%\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.9"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
@ -0,0 +1,77 @@
|
|||||||
|
import json
|
||||||
|
import re
|
||||||
|
import warnings
|
||||||
|
from typing import List, Tuple
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from langchain.docstore.document import Document
|
||||||
|
from langchain.document_loaders.base import BaseLoader
|
||||||
|
|
||||||
|
|
||||||
|
class BiliBiliLoader(BaseLoader):
|
||||||
|
"""Loader that loads bilibili transcripts."""
|
||||||
|
|
||||||
|
def __init__(self, video_urls: List[str]):
|
||||||
|
"""Initialize with bilibili url."""
|
||||||
|
self.video_urls = video_urls
|
||||||
|
|
||||||
|
def load(self) -> List[Document]:
|
||||||
|
"""Load from bilibili url."""
|
||||||
|
results = []
|
||||||
|
for url in self.video_urls:
|
||||||
|
transcript, video_info = self._get_bilibili_subs_and_info(url)
|
||||||
|
doc = Document(page_content=transcript, metadata=video_info)
|
||||||
|
results.append(doc)
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
def _get_bilibili_subs_and_info(self, url: str) -> Tuple[str, dict]:
|
||||||
|
try:
|
||||||
|
from bilibili_api import sync, video
|
||||||
|
except ImportError:
|
||||||
|
raise ValueError(
|
||||||
|
"requests package not found, please install it with "
|
||||||
|
"`pip install bilibili-api`"
|
||||||
|
)
|
||||||
|
|
||||||
|
bvid = re.search(r"BV\w+", url)
|
||||||
|
if bvid is not None:
|
||||||
|
v = video.Video(bvid=bvid.group())
|
||||||
|
else:
|
||||||
|
aid = re.search(r"av[0-9]+", url)
|
||||||
|
if aid is not None:
|
||||||
|
try:
|
||||||
|
v = video.Video(aid=int(aid.group()[2:]))
|
||||||
|
except AttributeError:
|
||||||
|
raise ValueError(f"{url} is not bilibili url.")
|
||||||
|
else:
|
||||||
|
raise ValueError(f"{url} is not bilibili url.")
|
||||||
|
|
||||||
|
video_info = sync(v.get_info())
|
||||||
|
video_info.update({"url": url})
|
||||||
|
|
||||||
|
# Get subtitle url
|
||||||
|
subtitle = video_info.pop("subtitle")
|
||||||
|
sub_list = subtitle["list"]
|
||||||
|
if sub_list:
|
||||||
|
sub_url = sub_list[0]["subtitle_url"]
|
||||||
|
result = requests.get(sub_url)
|
||||||
|
raw_sub_titles = json.loads(result.content)["body"]
|
||||||
|
raw_transcript = " ".join([c["content"] for c in raw_sub_titles])
|
||||||
|
|
||||||
|
raw_transcript_with_meta_info = f"""
|
||||||
|
Video Title: {video_info['title']},
|
||||||
|
description: {video_info['desc']}\n
|
||||||
|
Transcript: {raw_transcript}
|
||||||
|
"""
|
||||||
|
return raw_transcript_with_meta_info, video_info
|
||||||
|
else:
|
||||||
|
raw_transcript = ""
|
||||||
|
warnings.warn(
|
||||||
|
f"""
|
||||||
|
No subtitles found for video: {url}.
|
||||||
|
Return Empty transcript.
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
return raw_transcript, video_info
|
@ -0,0 +1,20 @@
|
|||||||
|
from langchain.document_loaders.bilibili import BiliBiliLoader
|
||||||
|
|
||||||
|
|
||||||
|
def test_bilibili_loader() -> None:
|
||||||
|
"""Test Bilibili Loader."""
|
||||||
|
loader = BiliBiliLoader(
|
||||||
|
[
|
||||||
|
"https://www.bilibili.com/video/BV1xt411o7Xu/",
|
||||||
|
"https://www.bilibili.com/video/av330407025/",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
docs = loader.load()
|
||||||
|
|
||||||
|
assert len(docs) == 2
|
||||||
|
|
||||||
|
assert len(docs[0].page_content) > 0
|
||||||
|
assert docs[1].metadata["owner"]["mid"] == 398095160
|
||||||
|
|
||||||
|
assert docs[1].page_content == ""
|
||||||
|
assert docs[1].metadata["owner"]["mid"] == 398095160
|
Loading…
Reference in New Issue