|
|
|
@ -16,7 +16,6 @@ from parser.js2doc import extract_functions_and_classes as extract_js
|
|
|
|
|
from parser.java2doc import extract_functions_and_classes as extract_java
|
|
|
|
|
from parser.token_func import group_split
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dotenv.load_dotenv()
|
|
|
|
|
|
|
|
|
|
app = typer.Typer(add_completion=False)
|
|
|
|
@ -41,12 +40,12 @@ def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
|
|
|
|
|
help="""List of required extensions (list with .)
|
|
|
|
|
Currently supported: .rst, .md, .pdf, .docx, .csv, .epub, .html, .mdx"""),
|
|
|
|
|
exclude: Optional[bool] = typer.Option(True, help="Whether to exclude hidden files (dotfiles)."),
|
|
|
|
|
sample: Optional[bool] = typer.Option(False, help="Whether to output sample of the first 5 split documents."),
|
|
|
|
|
sample: Optional[bool] = typer.Option(False,
|
|
|
|
|
help="Whether to output sample of the first 5 split documents."),
|
|
|
|
|
token_check: Optional[bool] = typer.Option(True, help="Whether to group small documents and split large."),
|
|
|
|
|
min_tokens: Optional[int] = typer.Option(150, help="Minimum number of tokens to not group."),
|
|
|
|
|
max_tokens: Optional[int] = typer.Option(2000, help="Maximum number of tokens to not split."),
|
|
|
|
|
):
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
Creates index from specified location or files.
|
|
|
|
|
By default /inputs folder is used, .rst and .md are parsed.
|
|
|
|
@ -59,7 +58,8 @@ def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
|
|
|
|
|
|
|
|
|
|
# Here we split the documents, as needed, into smaller chunks.
|
|
|
|
|
# We do this due to the context limits of the LLMs.
|
|
|
|
|
raw_docs = group_split(documents=raw_docs, min_tokens=min_tokens, max_tokens=max_tokens, token_check=token_check)
|
|
|
|
|
raw_docs = group_split(documents=raw_docs, min_tokens=min_tokens, max_tokens=max_tokens,
|
|
|
|
|
token_check=token_check)
|
|
|
|
|
# Old method
|
|
|
|
|
# text_splitter = RecursiveCharacterTextSplitter()
|
|
|
|
|
# docs = text_splitter.split_documents(raw_docs)
|
|
|
|
@ -71,7 +71,6 @@ def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
|
|
|
|
|
|
|
|
|
|
docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Here we check for command line arguments for bot calls.
|
|
|
|
|
# If no argument exists or the yes is not True, then the
|
|
|
|
|
# user permission is requested to call the API.
|
|
|
|
@ -103,7 +102,6 @@ def convert(dir: Optional[str] = typer.Option("inputs",
|
|
|
|
|
formats: Optional[str] = typer.Option("py",
|
|
|
|
|
help="""Required language.
|
|
|
|
|
py, js, java supported for now""")):
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
Creates documentation linked to original functions from specified location.
|
|
|
|
|
By default /inputs folder is used, .py is parsed.
|
|
|
|
@ -117,7 +115,7 @@ def convert(dir: Optional[str] = typer.Option("inputs",
|
|
|
|
|
else:
|
|
|
|
|
raise Exception("Sorry, language not supported yet")
|
|
|
|
|
transform_to_docs(functions_dict, classes_dict, formats, dir)
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
app()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
app()
|
|
|
|
|