From 072e16b12225a39a81aff4833765a8ada6b0460d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9D=B4=ED=83=9C=ED=98=B8?= Date: Sun, 26 Mar 2023 15:28:14 +0900 Subject: [PATCH] feat: update prompt for app develop (#8) * feat: update prompt for app develop * feat: complement prompt and logic both --- .gitignore | 4 +++- api/main.py | 37 ++++++++++++++++++++++--------------- core/agents/builder.py | 3 ++- core/editor/write.py | 9 +++++++++ core/prompts/input.py | 19 ++++++------------- core/tools/cpu.py | 31 ++++++++++++++++++++++--------- docker-compose.yml | 4 +++- utils.py | 1 + 8 files changed, 68 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 88ca765..5723a43 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ audio/ video/ dataframe/ -static/* \ No newline at end of file +static/* + +playground/ \ No newline at end of file diff --git a/api/main.py b/api/main.py index 9817763..80d0dcf 100644 --- a/api/main.py +++ b/api/main.py @@ -2,6 +2,7 @@ from typing import Dict, List, TypedDict import re import uvicorn +import torch from fastapi import FastAPI from fastapi.staticfiles import StaticFiles @@ -37,21 +38,27 @@ app = FastAPI() app.mount("/static", StaticFiles(directory=StaticUploader.STATIC_DIR), name="static") uploader = StaticUploader.from_settings(settings) -toolsets: List[BaseToolSet] = [ - Terminal(), - CodeEditor(), - RequestsGet(), - ExitConversation(), - Text2Image("cuda"), - ImageEditing("cuda"), - InstructPix2Pix("cuda"), - VisualQuestionAnswering("cuda"), -] - -handlers: Dict[FileType, BaseHandler] = { - FileType.IMAGE: ImageCaptioning("cuda"), - FileType.DATAFRAME: CsvToDataframe(), -} +toolsets: List[BaseToolSet] = ( + [ + Terminal(), + CodeEditor(), + RequestsGet(), + ExitConversation(), + ] + + [ + Text2Image("cuda"), + ImageEditing("cuda"), + InstructPix2Pix("cuda"), + VisualQuestionAnswering("cuda"), + ] + if torch.cuda.is_available() + else [] +) + +handlers: Dict[FileType, BaseHandler] = {} +handlers[FileType.DATAFRAME] = CsvToDataframe() +if torch.cuda.is_available(): + handlers[FileType.IMAGE] = ImageCaptioning("cuda") if settings["WINEDB_HOST"] and settings["WINEDB_PASSWORD"]: toolsets.append(WineDB()) diff --git a/core/agents/builder.py b/core/agents/builder.py index 9675102..b4ad17e 100644 --- a/core/agents/builder.py +++ b/core/agents/builder.py @@ -29,7 +29,7 @@ class AgentBuilder: if self.llm is None: raise ValueError("LLM must be initialized before tools") - toolnames = ["python_repl", "wikipedia"] + toolnames = ["wikipedia"] if settings["SERPAPI_API_KEY"]: toolnames.append("serpapi") @@ -68,4 +68,5 @@ class AgentBuilder: system_message=EVAL_PREFIX.format(bot_name=settings["BOT_NAME"]), human_message=EVAL_SUFFIX.format(bot_name=settings["BOT_NAME"]), output_parser=self.parser, + max_iterations=30, ) diff --git a/core/editor/write.py b/core/editor/write.py index 61e069b..0a10e89 100644 --- a/core/editor/write.py +++ b/core/editor/write.py @@ -4,6 +4,8 @@ write protocol: """ +import os +from pathlib import Path class WriteCommand: @@ -19,6 +21,13 @@ class WriteCommand: return self def execute(self) -> str: + # make sure the directory exists + if not str(Path(self.filepath).resolve()).startswith( + str(Path("playground/").resolve()) + ): + return "You can't write file outside of playground folder." + + os.makedirs(os.path.dirname(self.filepath), exist_ok=True) with open(self.filepath, self.mode) as f: f.write(self.content) return self.content diff --git a/core/prompts/input.py b/core/prompts/input.py index 69bff23..37813b9 100644 --- a/core/prompts/input.py +++ b/core/prompts/input.py @@ -1,24 +1,17 @@ -EVAL_PREFIX = """{bot_name} is designed to be able to assist with a wide range of text, visual related tasks, data analysis related tasks, auditory related tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. -{bot_name} is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand. -{bot_name} is able to process and understand large amounts of various types of files(image, audio, video, dataframe, etc.). As a language model, {bot_name} can not directly read various types of files(text, image, audio, video, dataframe, etc.), but it has a list of tools to finish different visual tasks. +EVAL_PREFIX = """{bot_name} can execute any user's request. -Each image will have a file name formed as "image/xxx.png" -Each audio will have a file name formed as "audio/xxx.mp3" -Each video will have a file name formed as "video/xxx.mp4" -Each dataframe will have a file name formed as "dataframe/xxx.csv" +{bot_name} has permission to handle one instance and can handle the environment in it at will. File creation is only possible in playground folder, and other folders and files can never be modified under any circumstances. +You can code, run, debug, and test yourself. You can correct the code appropriately by looking at the error message. Please do TDD. -{bot_name} can invoke different tools to indirectly understand files(image, audio, video, dataframe, etc.). When talking about files(image, audio, video, dataframe, etc.), {bot_name} is very strict to the file name and will never fabricate nonexistent files. -When using tools to generate new files, {bot_name} is also known that the file(image, audio, video, dataframe, etc.) may not be the same as the user's demand, and will use other visual question answering tools or description tools to observe the real file. -{bot_name} is able to use tools in a sequence, and is loyal to the tool observation outputs rather than faking the file content and file name. It will remember to provide the file name from the last tool observation, if a new file is generated. -Human may provide new figures to {bot_name} with a description. The description helps {bot_name} to understand this file, but {bot_name} should use tools to finish following tasks, rather than directly imagine from the description. +I can understand, process, and create various types of files. Images must be stored in the ./image/, audio in the ./audio/, video in the ./video/, and dataframes must be stored in the ./dataframe/. -Overall, {bot_name} is a powerful visual dialogue assistant tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. +{bot_name} can do whatever it takes to execute the user's request. Let's think step by step. """ EVAL_FORMAT_INSTRUCTIONS = """RESPONSE FORMAT INSTRUCTIONS ---------------------------- -When responding to me please, please output a response in one of two formats: +When responding to me please, please output a response in one of two formats. No explanation is allowed after action input.: **Option #1:** Use this if you want the human to use a tool. diff --git a/core/tools/cpu.py b/core/tools/cpu.py index ce7a4d5..37c9136 100644 --- a/core/tools/cpu.py +++ b/core/tools/cpu.py @@ -18,7 +18,8 @@ class Terminal(BaseToolSet): @tool( name="Terminal", description="Executes commands in a terminal." - "You can install packages with pip, apt, etc." + "If linux errno occurs, we have to solve the problem with the terminal. " + "It can't execute interactive operations or blocking operations. " "Input should be valid commands, " "and the output will be any output from running that command.", ) @@ -34,6 +35,9 @@ class Terminal(BaseToolSet): except Exception as e: output = str(e) + if len(output) > 1000: + output = output[:1000] + "..." + logger.debug( f"\nProcessed Terminal, Input Commands: {commands} " f"Output Answer: {output}" @@ -89,15 +93,20 @@ class CodeEditor(BaseToolSet): "ex. test.py\nprint('hello world')\n " "and the output will be last 3 line.", ) - def write(self, inputs: str) -> str: + def append(self, inputs: str) -> str: try: code = CodeWriter.append(inputs) - output = "Last 3 line was:\n" + "\n".join(code.split("\n")[-3:]) + output = ( + "Last 3 line was:\n" + + "\n".join(code.split("\n")[-3:]) + + "\nYou can use CodeEditor.APPEND tool to append the code if it is not completed." + ) except Exception as e: output = str(e) logger.debug( - f"\nProcessed CodeEditor, Input: {inputs} " f"Output Answer: {output}" + f"\nProcessed CodeEditor.APPEND, Input: {inputs} " + f"Output Answer: {output}" ) return output @@ -105,19 +114,23 @@ class CodeEditor(BaseToolSet): name="CodeEditor.WRITE", description="Write code to create a new tool. " "If the code is completed, use the Terminal tool to execute it, if not, append the code through the CodeEditor.APPEND tool. " - "Input should be filename and code. " + "Input should be filename and code. This file must be in playground folder. " "ex. test.py\nprint('hello world')\n " "and the output will be last 3 line.", ) def write(self, inputs: str) -> str: try: code = CodeWriter.write(inputs) - output = "Last 3 line was:\n" + "\n".join(code.split("\n")[-3:]) + output = ( + "Last 3 line was:\n" + + "\n".join(code.split("\n")[-3:]) + + "\nYou can use CodeEditor.APPEND tool to append the code if it is not completed." + ) except Exception as e: output = str(e) logger.debug( - f"\nProcessed CodeEditor, Input: {inputs} " f"Output Answer: {output}" + f"\nProcessed CodeEditor.WRITE, Input: {inputs} " f"Output Answer: {output}" ) return output @@ -140,7 +153,7 @@ class CodeEditor(BaseToolSet): output = str(e) logger.debug( - f"\nProcessed CodeEditor, Input Patch: {patches} " + f"\nProcessed CodeEditor.PATCH, Input Patch: {patches} " f"Output Answer: {output}" ) return output @@ -162,7 +175,7 @@ class CodeEditor(BaseToolSet): output = str(e) logger.debug( - f"\nProcessed CodeEditor, Input filename: {inputs} " + f"\nProcessed CodeEditor.DELETE, Input filename: {inputs} " f"Output Answer: {output}" ) return output diff --git a/docker-compose.yml b/docker-compose.yml index 9b475d8..654ce15 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,9 @@ services: - ../.cache/huggingface/:/root/.cache/huggingface/ - ./static/:/app/static/ ports: - - "8000:8000" + - "7500:7500" + - "7501:7501" + - "8000:8000" # eval port env_file: - .env deploy: diff --git a/utils.py b/utils.py index 3e0c7cd..b3bc08f 100644 --- a/utils.py +++ b/utils.py @@ -9,6 +9,7 @@ os.makedirs("image", exist_ok=True) os.makedirs("audio", exist_ok=True) os.makedirs("video", exist_ok=True) os.makedirs("dataframe", exist_ok=True) +os.makedirs("playground", exist_ok=True) def seed_everything(seed):