chore: add support for TypeScript code splitting (#11160)

- **Description:** Adds typescript language to `TextSplitter`

---------

Co-authored-by: Jacob Lee <jacoblee93@gmail.com>
pull/11202/head
Fynn Flügge 1 year ago committed by GitHub
parent 17fcbed92c
commit b738ccd91e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -18,6 +18,7 @@ from langchain.text_splitter import (
'go',
'java',
'js',
'ts',
'php',
'proto',
'python',
@ -107,6 +108,36 @@ js_docs
</CodeOutputBlock>
## TS
Here's an example using the TS text splitter:
```python
TS_CODE = """
function helloWorld(): void {
console.log("Hello, World!");
}
// Call the function
helloWorld();
"""
ts_splitter = RecursiveCharacterTextSplitter.from_language(
language=Language.TS, chunk_size=60, chunk_overlap=0
)
ts_docs = ts_splitter.create_documents([TS_CODE])
ts_docs
```
<CodeOutputBlock lang="python">
```
[Document(page_content='function helloWorld(): void {\n console.log("Hello, World!");\n}', metadata={}),
Document(page_content='// Call the function\nhelloWorld();', metadata={})]
```
</CodeOutputBlock>
## Markdown
Here's an example using the Markdown text splitter:

@ -615,6 +615,7 @@ class Language(str, Enum):
GO = "go"
JAVA = "java"
JS = "js"
TS = "ts"
PHP = "php"
PROTO = "proto"
PYTHON = "python"
@ -782,6 +783,32 @@ class RecursiveCharacterTextSplitter(TextSplitter):
" ",
"",
]
elif language == Language.TS:
return [
"\nenum ",
"\ninterface ",
"\nnamespace ",
"\ntype ",
# Split along class definitions
"\nclass ",
# Split along function definitions
"\nfunction ",
"\nconst ",
"\nlet ",
"\nvar ",
# Split along control flow statements
"\nif ",
"\nfor ",
"\nwhile ",
"\nswitch ",
"\ncase ",
"\ndefault ",
# Split by the normal type of lines
"\n\n",
"\n",
" ",
"",
]
elif language == Language.PHP:
return [
# Split along function definitions

@ -472,6 +472,33 @@ helloWorld();
]
def test_typescript_code_splitter() -> None:
splitter = RecursiveCharacterTextSplitter.from_language(
Language.TS, chunk_size=CHUNK_SIZE, chunk_overlap=0
)
code = """
function helloWorld(): void {
console.log("Hello, World!");
}
// Call the function
helloWorld();
"""
chunks = splitter.split_text(code)
assert chunks == [
"function",
"helloWorld():",
"void {",
'console.log("He',
"llo,",
'World!");',
"}",
"// Call the",
"function",
"helloWorld();",
]
def test_java_code_splitter() -> None:
splitter = RecursiveCharacterTextSplitter.from_language(
Language.JAVA, chunk_size=CHUNK_SIZE, chunk_overlap=0

Loading…
Cancel
Save