Add support for loading files with extensions not present in ALLOW_FILES in LOADER_MAPPING.

poetry.lock

@@ -1880,20 +1880,6 @@ files = [
 [package.dependencies]
 lxml = ">=2.3.2"
 
-[[package]]
-name = "python-fire"
-version = "0.1.0"
-description = "FIRE HOT. TREE PRETTY"
-optional = false
-python-versions = "*"
-files = [
-    {file = "python-fire-0.1.0.tar.gz", hash = "sha256:ed3cb202288c6e742a077611fb884b154068758f9ec65eb827520d670bdbada9"},
-    {file = "python_fire-0.1.0-py2.py3-none-any.whl", hash = "sha256:21c1c1de2106844091f0b7485ed5aa94cc943a83b5218b913a19822395da4bc0"},
-]
-
-[package.dependencies]
-pygments = "*"
-
 [[package]]
 name = "python-magic"
 version = "0.4.27"
@@ -2367,6 +2353,22 @@ files = [
     {file = "sentencepiece-0.1.99.tar.gz", hash = "sha256:189c48f5cb2949288f97ccdb97f0473098d9c3dcf5a3d99d4eabe719ec27297f"},
 ]
 
+[[package]]
+name = "setuptools"
+version = "68.0.0"
+description = "Easily download, build, install, upgrade, and uninstall Python packages"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"},
+    {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"},
+]
+
+[package.extras]
+docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
+testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
+testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -3128,4 +3130,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8.1,<4.0"
-content-hash = "8342d2a5d74f30c240656daf54b8186930e99158f03bc81422ede799ea3410c9"
+content-hash = "74f15ed94b5ed84e8f63071297b4897845473cc15df004bf40fd43622a7b03be"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "talk-codebase"
-version = "0.1.34"
+version = "0.1.35"
 description = "talk-codebase is a powerful tool for querying and analyzing codebases."
 authors = ["Saryev Rustam <rustam1997@gmail.com>"]
 readme = "README.md"
@@ -23,6 +23,9 @@ sentence-transformers = "^2.2.2"
 unstructured = "^0.6.10"
 
 
+[tool.poetry.group.dev.dependencies]
+setuptools = "^68.0.0"
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

talk_codebase/consts.py

@@ -46,9 +46,12 @@ LOADER_MAPPING = {
     ".pdf": {
         "loader": PDFMinerLoader,
         "args": {}
-    },
-    ".txt": {
-        "loader": TextLoader,
-        "args": {}
-    },
+    }
 }
+
+for ext in ALLOW_FILES:
+    if ext not in LOADER_MAPPING:
+        LOADER_MAPPING[ext] = {
+            "loader": TextLoader,
+            "args": {}
+        }

talk_codebase/utils.py

@@ -5,7 +5,6 @@ import sys
 
 import tiktoken
 from git import Repo
-from halo import Halo
 from langchain import FAISS
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler