feat: code editor (#4)

pull/3/head
이태호 1 year ago committed by GitHub
parent c5a8f85d7a
commit 78f9e081ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -7,9 +7,9 @@ from env import settings
from tools.base import BaseToolSet
from tools.factory import ToolsFactory
from .llm import ChatOpenAI
from .chat_agent import ConversationalChatAgent
from .parser import EvalOutputParser
from agents.llm import ChatOpenAI
from agents.chat_agent import ConversationalChatAgent
from agents.parser import EvalOutputParser
class AgentBuilder:

@ -26,7 +26,7 @@ from langchain.schema import (
)
from langchain.utils import get_from_dict_or_env
logger = logging.getLogger(__file__)
from logger import logger
def _create_retry_decorator(llm: ChatOpenAI) -> Callable[[Any], Any]:
@ -218,7 +218,7 @@ class ChatOpenAI(BaseChatModel, BaseModel):
@retry_decorator
def _completion_with_retry(**kwargs: Any) -> Any:
response = self.client.create(**kwargs)
print(response)
logger.debug("Response:\n\t%s", response)
return response
return _completion_with_retry(**kwargs)
@ -228,11 +228,12 @@ class ChatOpenAI(BaseChatModel, BaseModel):
) -> ChatResult:
message_dicts, params = self._create_message_dicts(messages, stop)
logger.debug("Messages:\n")
for item in message_dicts:
for k, v in item.items():
print(f"{k}: {v}")
print("-------")
print("===========")
logger.debug(f"\t\t{k}: {v}")
logger.debug("\t-------")
logger.debug("===========")
if self.streaming:
inner_completion = ""

@ -7,7 +7,7 @@ from langchain.memory.chat_memory import BaseChatMemory
from tools.base import BaseToolSet
from tools.cpu import ExitConversation
from .builder import AgentBuilder
from agents.builder import AgentBuilder
class AgentManager:
@ -31,6 +31,7 @@ class AgentManager:
agent=self.agent,
tools=self.tools,
memory=memory,
verbose=True,
)
def remove_executor(self, key: str) -> None:

@ -1,4 +1,3 @@
import json
import re
from typing import Dict
@ -15,7 +14,7 @@ class EvalOutputParser(BaseOutputParser):
regex = r"Action: (.*?)[\n]*Action Input: (.*)"
match = re.search(regex, text, re.DOTALL)
if not match:
raise ValueError(f"Could not parse LLM output: `{text}`")
return {"action": "Exit Conversation", "action_input": text}
action = match.group(1).strip()
action_input = match.group(2)
return {"action": action, "action_input": action_input.strip(" ").strip('"')}

@ -7,20 +7,22 @@ load_dotenv()
class DotEnv(TypedDict):
LOG_LEVEL: str # optional
BOT_NAME: str
AWS_ACCESS_KEY_ID: str
AWS_SECRET_ACCESS_KEY: str
AWS_REGION: str
AWS_S3_BUCKET: str
WINEDB_HOST: str
WINEDB_PASSWORD: str
WINEDB_HOST: str # optional
WINEDB_PASSWORD: str # optional
OPENAI_API_KEY: str
BING_SEARCH_URL: str
BING_SUBSCRIPTION_KEY: str
SERPAPI_API_KEY: str
BING_SEARCH_URL: str # optional
BING_SUBSCRIPTION_KEY: str # optional
SERPAPI_API_KEY: str # optional
settings: DotEnv = {
"LOG_LEVEL": os.getenv("LOG_LEVEL", "INFO"),
"BOT_NAME": os.getenv("BOT_NAME", "Orca"),
"AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID"),
"AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY"),

@ -2,7 +2,7 @@ import pandas as pd
from prompts.file import DATAFRAME_PROMPT
from .base import BaseHandler
from handlers.base import BaseHandler
class CsvToDataframe(BaseHandler):

@ -7,7 +7,7 @@ from transformers import (
from prompts.file import IMAGE_PROMPT
from .base import BaseHandler
from handlers.base import BaseHandler
class ImageCaptioning(BaseHandler):

@ -0,0 +1,8 @@
import logging
from env import settings
logger = logging.getLogger("EVAL")
if settings["LOG_LEVEL"] == "DEBUG":
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)

BIN
main

Binary file not shown.

@ -26,6 +26,7 @@ from tools.gpu import (
from handlers.base import BaseHandler, FileHandler, FileType
from handlers.image import ImageCaptioning
from handlers.dataframe import CsvToDataframe
from logger import logger
app = FastAPI()
@ -34,14 +35,14 @@ toolsets: List[BaseToolSet] = [
Terminal(),
CodeEditor(),
RequestsGet(),
Text2Image("cuda"),
ImageEditing("cuda"),
InstructPix2Pix("cuda"),
VisualQuestionAnswering("cuda"),
# Text2Image("cuda"),
# ImageEditing("cuda"),
# InstructPix2Pix("cuda"),
# VisualQuestionAnswering("cuda"),
]
handlers: Dict[FileType, BaseHandler] = {
FileType.IMAGE: ImageCaptioning("cuda"),
# FileType.IMAGE: ImageCaptioning("cuda"),
FileType.DATAFRAME: CsvToDataframe(),
}
@ -74,16 +75,16 @@ async def command(request: Request) -> Response:
files = request.files
key = request.key
print("=============== Running =============")
print("Inputs:", query, files)
logger.info("=============== Running =============")
logger.info(f"Query: {query}, Files: {files}")
executor = agent_manager.get_or_create_executor(key)
print("======>Previous memory:\n %s" % executor.memory)
logger.info(f"======> Previous memory:\n\t{executor.memory}")
# TODO: exit conversation
promptedQuery = "\n".join([file_handler.handle(file) for file in files])
promptedQuery += query
print("======>Prompted Text:\n %s" % promptedQuery)
logger.info(f"======> Prompted Text:\n\t{promptedQuery}")
try:
res = executor({"input": promptedQuery})

@ -1,5 +1,3 @@
from env import settings
EVAL_PREFIX = """{bot_name} is designed to be able to assist with a wide range of text, visual related tasks, data analysis related tasks, auditory related tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics.
{bot_name} is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.
{bot_name} is able to process and understand large amounts of various types of files(image, audio, video, dataframe, etc.). As a language model, {bot_name} can not directly read various types of files(text, image, audio, video, dataframe, etc.), but it has a list of tools to finish different visual tasks.
@ -22,7 +20,7 @@ EVAL_FORMAT_INSTRUCTIONS = """RESPONSE FORMAT INSTRUCTIONS
When responding to me please, please output a response in one of two formats:
**Option 1:**
**Option #1:**
Use this if you want the human to use a tool.
Your response should be in the following schema:

@ -11,7 +11,8 @@ from langchain.agents.agent import AgentExecutor
import subprocess
from .base import tool, BaseToolSet
from tools.base import tool, BaseToolSet
from logger import logger
class Terminal(BaseToolSet):
@ -22,7 +23,7 @@ class Terminal(BaseToolSet):
"Input should be valid commands, "
"and the output will be any output from running that command.",
)
def inference(self, commands: str) -> str:
def execute(self, commands: str) -> str:
"""Run commands and return final output."""
try:
output = subprocess.run(
@ -34,7 +35,7 @@ class Terminal(BaseToolSet):
except Exception as e:
output = str(e)
print(
logger.debug(
f"\nProcessed Terminal, Input Commands: {commands} "
f"Output Answer: {output}"
)
@ -42,33 +43,108 @@ class Terminal(BaseToolSet):
class CodeEditor(BaseToolSet):
@tool(
name="CodeEditor.READ",
description="Read and understand code. "
"Input should be filename and line number group. ex. test.py,1-10 "
"and the output will be code. ",
)
def read(self, inputs: str) -> str:
filename, line = inputs.split(",")
line = line.split("-")
if len(line) == 1:
line = int(line[0])
else:
line = [int(i) for i in line]
try:
with open(filename, "r") as f:
code = f.readlines()
if isinstance(line, int):
code = code[line - 1]
else:
code = "".join(code[line[0] - 1 : line[1]])
output = code
except Exception as e:
output = str(e)
logger.debug(
f"\nProcessed CodeEditor.READ, Input Commands: {inputs} "
f"Output Answer: {output}"
)
return output
@tool(
name="CodeEditor.WRITE",
description="Writes and appends code."
"It can be used to write or append code in any language. "
"If the code is completed, use the Terminal tool to execute it, if not, append the code through the CodeEditor tool."
"Input should be filename, status, code. Status will be 'complete' or 'incomplete'. ex. 'test.py|complete\nprint('hello world')\n"
"and the output will be status and last line. status will be 'complete' or 'incomplete' or 'error'.",
description="Write code to create a new tool. "
"You must check the file's contents before writing. This tool only supports append code. "
"If the code is completed, use the Terminal tool to execute it, if not, append the code through the CodeEditor tool. "
"Input should be filename and code. "
"ex. test.py\nprint('hello world')\n "
"and the output will be last 3 line.",
)
def write(self, inputs: str) -> str:
"""Save codes to file and return success or failure."""
filename, status_and_code = inputs.split("|", 1)
status, code = status_and_code.split("\n", 1)
if status != "complete" and status != "incomplete":
return "error: status must be complete or incomplete"
filename, code = inputs.split("\n", 1)
try:
with open(filename, "a") as f:
f.write(code)
output = status + "\nLast line was:" + code.split("\n")[-1]
output = "Last 3 line was:\n" + "\n".join(code.split("\n")[-3:])
except Exception as e:
output = "error"
print(
output = str(e)
logger.debug(
f"\nProcessed CodeEditor, Input Codes: {code} " f"Output Answer: {output}"
)
return output
@tool(
name="CodeEditor.PATCH",
description="Correct the error throught the code patch if an error occurs. "
"Input should be list of filename, line number, new line (Be sure to consider indentations.) Seperated by -||-."
"ex. \"test.py-||-1-||-print('hello world')\ntest.py-||-2-||-print('hello world')\n\" "
"and the output will be success or error message. ",
)
def patch(self, patches: str) -> str:
for patch in patches.split("\n"):
filename, line_number, new_line = patch.split("-||-") # TODO: fix this
try:
with open(filename, "r") as f:
lines = f.readlines()
lines[int(line_number) - 1] = new_line + "\n"
with open(filename, "w") as f:
f.writelines(lines)
output = "success"
except Exception as e:
output = str(e)
logger.debug(
f"\nProcessed CodeEditor, Input Patch: {patches} "
f"Output Answer: {output}"
)
return output
@tool(
name="CodeEditor.DELETE",
description="Delete code in file for a new start. "
"Input should be filename."
"ex. test.py "
"Output will be success or error message.",
)
def delete(self, inputs: str) -> str:
filename = inputs
try:
with open(filename, "w") as f:
f.write("")
output = "success"
except Exception as e:
output = str(e)
logger.debug(
f"\nProcessed CodeEditor, Input filename: {inputs} "
f"Output Answer: {output}"
)
return output
class RequestsGet(BaseToolSet):
@tool(
@ -78,7 +154,7 @@ class RequestsGet(BaseToolSet):
"Input should be a url (i.e. https://www.google.com)."
"The output will be the text response of the GET request.",
)
def inference(self, url: str) -> str:
def get(self, url: str) -> str:
"""Run the tool."""
html = requests.get(url).text
soup = BeautifulSoup(html)
@ -94,7 +170,7 @@ class RequestsGet(BaseToolSet):
if len(content) > 300:
content = content[:300] + "..."
print(
logger.debug(
f"\nProcessed RequestsGet, Input Url: {url} " f"Output Contents: {content}"
)
@ -128,7 +204,7 @@ class WineDB(BaseToolSet):
"The output will be a list of recommended wines."
"The tool is based on a database of wine reviews, which is stored in a database.",
)
def inference(self, query: str) -> str:
def recommend(self, query: str) -> str:
"""Run the tool."""
results = self.index.query(query)
wine = "\n".join(
@ -141,7 +217,9 @@ class WineDB(BaseToolSet):
)
output = results.response + "\n\n" + wine
print(f"\nProcessed WineDB, Input Query: {query} " f"Output Wine: {wine}")
logger.debug(
f"\nProcessed WineDB, Input Query: {query} " f"Output Wine: {wine}"
)
return output
@ -161,6 +239,6 @@ class ExitConversation(BaseToolSet):
"""Run the tool."""
self.executors.pop(key)
print(f"\nProcessed ExitConversation.")
logger.debug(f"\nProcessed ExitConversation.")
return f"Exit conversation."

@ -4,7 +4,7 @@ from langchain.agents import load_tools
from langchain.agents.tools import BaseTool
from langchain.llms.base import BaseLLM
from .base import BaseToolSet
from tools.base import BaseToolSet
class ToolsFactory:

@ -21,7 +21,9 @@ from diffusers import (
from diffusers import EulerAncestralDiscreteScheduler
from utils import get_new_image_name
from .base import tool, BaseToolSet
from logger import logger
from tools.base import tool, BaseToolSet
class MaskFormer(BaseToolSet):
@ -106,10 +108,12 @@ class ImageEditing(BaseToolSet):
)
updated_image = updated_image.resize(original_size)
updated_image.save(updated_image_path)
print(
logger.debug(
f"\nProcessed ImageEditing, Input Image: {image_path}, Replace {to_be_replaced_txt} to {replace_with_txt}, "
f"Output Image: {updated_image_path}"
)
return updated_image_path
@ -136,7 +140,7 @@ class InstructPix2Pix(BaseToolSet):
)
def inference(self, inputs):
"""Change style of image."""
print("===>Starting InstructPix2Pix Inference")
logger.debug("===> Starting InstructPix2Pix Inference")
image_path, text = inputs.split(",")[0], ",".join(inputs.split(",")[1:])
original_image = Image.open(image_path)
image = self.pipe(
@ -144,10 +148,12 @@ class InstructPix2Pix(BaseToolSet):
).images[0]
updated_image_path = get_new_image_name(image_path, func_name="pix2pix")
image.save(updated_image_path)
print(
logger.debug(
f"\nProcessed InstructPix2Pix, Input Image: {image_path}, Instruct Text: {text}, "
f"Output Image: {updated_image_path}"
)
return updated_image_path
@ -177,9 +183,11 @@ class Text2Image(BaseToolSet):
prompt = text + ", " + self.a_prompt
image = self.pipe(prompt, negative_prompt=self.n_prompt).images[0]
image.save(image_filename)
print(
logger.debug(
f"\nProcessed Text2Image, Input Text: {text}, Output Image: {image_filename}"
)
return image_filename
@ -207,8 +215,10 @@ class VisualQuestionAnswering(BaseToolSet):
)
out = self.model.generate(**inputs)
answer = self.processor.decode(out[0], skip_special_tokens=True)
print(
logger.debug(
f"\nProcessed VisualQuestionAnswering, Input Image: {image_path}, Input Question: {question}, "
f"Output Answer: {answer}"
)
return answer

Loading…
Cancel
Save