mirror of https://github.com/corca-ai/EVAL
refactor: handle file
adldotori
1 year ago
parent
073b22927a
commit
71e9489ff0
@ -0,0 +1,16 @@
|
||||
FROM nvidia/cuda:11.7.0-runtime-ubuntu20.04
|
||||
WORKDIR /app/
|
||||
|
||||
RUN \
|
||||
apt-get update && \
|
||||
apt-get install -y python3 python3-pip
|
||||
RUN apt-get install uvicorn -y
|
||||
|
||||
RUN pip install --upgrade pip
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
# ENTRYPOINT ["python3", "-m", "uvicorn", "main:app", "--reload", "--host=0.0.0.0", "--port=8000"]
|
||||
@ -0,0 +1,38 @@
|
||||
# Usage
|
||||
|
||||
### S3
|
||||
|
||||
1. Create a bucket.
|
||||
2. Turn off the "Block all public access" setting for the bucket. 
|
||||
3. Add the following text to Bucket Policy.
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "AllowPublicRead",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"AWS": "*"
|
||||
},
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::{your-bucket-name}/*"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Environment
|
||||
|
||||
You must need this environments.
|
||||
|
||||
```
|
||||
OPENAI_API_KEY
|
||||
```
|
||||
|
||||
You need this environments.
|
||||
|
||||
```
|
||||
serpapi: SERPAPI_API_KEY
|
||||
bing-search: BING_SEARCH_URL, BING_SUBSCRIPTION_KEY
|
||||
```
|
||||
@ -0,0 +1,79 @@
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
from llm import ChatOpenAI
|
||||
from langchain.agents import load_tools
|
||||
from langchain.agents.agent import AgentExecutor
|
||||
from langchain.agents.tools import Tool
|
||||
from langchain.agents.initialize import initialize_agent
|
||||
from langchain.chains.conversation.memory import ConversationBufferMemory
|
||||
|
||||
from utils import AWESOMEGPT_PREFIX, AWESOMEGPT_SUFFIX
|
||||
|
||||
from tools.cpu import (
|
||||
RequestsGet,
|
||||
WineDB,
|
||||
ExitConversation,
|
||||
)
|
||||
from tools.gpu import (
|
||||
ImageEditing,
|
||||
InstructPix2Pix,
|
||||
Text2Image,
|
||||
ImageCaptioning,
|
||||
VisualQuestionAnswering,
|
||||
)
|
||||
from handler import Handler, FileType
|
||||
|
||||
|
||||
def get_agent() -> Tuple[AgentExecutor, Handler]:
|
||||
print("Initializing AwesomeGPT")
|
||||
llm = ChatOpenAI(temperature=0)
|
||||
tools = [
|
||||
*load_tools(
|
||||
["python_repl", "terminal", "serpapi", "wikipedia", "bing-search"],
|
||||
llm=llm,
|
||||
),
|
||||
]
|
||||
memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
|
||||
|
||||
models = {
|
||||
"RequestsGet": RequestsGet(),
|
||||
"WineDB": WineDB(),
|
||||
"ExitConversation": ExitConversation(memory),
|
||||
"Text2Image": Text2Image("cuda"),
|
||||
"ImageEditing": ImageEditing("cuda"),
|
||||
"InstructPix2Pix": InstructPix2Pix("cuda"),
|
||||
"VisualQuestionAnswering": VisualQuestionAnswering("cuda"),
|
||||
}
|
||||
|
||||
for _, instance in models.items():
|
||||
for e in dir(instance):
|
||||
if e.startswith("inference"):
|
||||
func = getattr(instance, e)
|
||||
tools.append(
|
||||
Tool(name=func.name, description=func.description, func=func)
|
||||
)
|
||||
|
||||
handle_models: Dict[FileType, str] = {
|
||||
FileType.IMAGE: ImageCaptioning("cuda"),
|
||||
}
|
||||
|
||||
handler = Handler(
|
||||
handle_func={
|
||||
file_type: model.inference for file_type, model in handle_models.items()
|
||||
}
|
||||
)
|
||||
|
||||
return (
|
||||
initialize_agent(
|
||||
tools,
|
||||
llm,
|
||||
agent="chat-conversational-react-description",
|
||||
verbose=True,
|
||||
memory=memory,
|
||||
agent_kwargs={
|
||||
"system_message": AWESOMEGPT_PREFIX,
|
||||
"human_message": AWESOMEGPT_SUFFIX,
|
||||
},
|
||||
),
|
||||
handler,
|
||||
)
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 145 KiB |
@ -0,0 +1,16 @@
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
awesomegpt:
|
||||
build:
|
||||
dockerfile: Dockerfile
|
||||
context: .
|
||||
env_file:
|
||||
- .env
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
device_ids: ["3"] # You can choose which GPU to use
|
||||
capabilities: [gpu]
|
||||
@ -1,87 +0,0 @@
|
||||
import os
|
||||
import requests
|
||||
import uuid
|
||||
from typing import Callable
|
||||
from enum import Enum
|
||||
|
||||
from PIL import Image
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from utils import IMAGE_PROMPT, DATAFRAME_PROMPT
|
||||
from tools import IMAGE_MODEL
|
||||
|
||||
|
||||
class FileType(Enum):
|
||||
IMAGE = "image"
|
||||
AUDIO = "audio"
|
||||
VIDEO = "video"
|
||||
DATAFRAME = "dataframe"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
def handle(file_name: str) -> Callable:
|
||||
"""
|
||||
Parse file type from file name (ex. image, audio, video, dataframe, etc.)
|
||||
"""
|
||||
file_name = file_name.split("?")[0]
|
||||
|
||||
if file_name.endswith(".png") or file_name.endswith(".jpg"):
|
||||
return handle_image
|
||||
elif file_name.endswith(".mp3") or file_name.endswith(".wav"):
|
||||
return handle_audio
|
||||
elif file_name.endswith(".mp4") or file_name.endswith(".avi"):
|
||||
return handle_video
|
||||
elif file_name.endswith(".csv"):
|
||||
return handle_dataframe
|
||||
else:
|
||||
return handle_unknown
|
||||
|
||||
|
||||
def handle_image(i: int, file: str) -> str:
|
||||
img_data = requests.get(file).content
|
||||
filename = os.path.join("image", str(uuid.uuid4())[0:8] + ".png")
|
||||
with open(filename, "wb") as f:
|
||||
size = f.write(img_data)
|
||||
print(f"Inputs: {file} ({size//1000}MB) => {filename}")
|
||||
img = Image.open(filename)
|
||||
width, height = img.size
|
||||
ratio = min(512 / width, 512 / height)
|
||||
width_new, height_new = (round(width * ratio), round(height * ratio))
|
||||
img = img.resize((width_new, height_new))
|
||||
img = img.convert("RGB")
|
||||
img.save(filename, "PNG")
|
||||
print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
|
||||
try:
|
||||
description = IMAGE_MODEL.inference(filename)
|
||||
except Exception as e:
|
||||
return {"text": "image upload", "response": str(e), "additional": []}
|
||||
|
||||
return IMAGE_PROMPT.format(i=i, filename=filename, description=description)
|
||||
|
||||
|
||||
def handle_audio(i: int, file: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def handle_video(i: int, file: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def handle_dataframe(i: int, file: str) -> str:
|
||||
content = requests.get(file).content
|
||||
filename = os.path.join("dataframe/", str(uuid.uuid4())[0:8] + ".csv")
|
||||
with open(filename, "wb") as f:
|
||||
size = f.write(content)
|
||||
print(f"Inputs: {file} ({size//1000}MB) => {filename}")
|
||||
df = pd.read_csv(filename)
|
||||
try:
|
||||
description = str(df.describe())
|
||||
except Exception as e:
|
||||
return {"text": "image upload", "response": str(e), "additional": []}
|
||||
|
||||
return DATAFRAME_PROMPT.format(i=i, filename=filename, description=description)
|
||||
|
||||
|
||||
def handle_unknown(i: int, file: str) -> str:
|
||||
return ""
|
||||
@ -0,0 +1,89 @@
|
||||
import os
|
||||
import requests
|
||||
import uuid
|
||||
from typing import Callable, Dict
|
||||
from enum import Enum
|
||||
|
||||
from PIL import Image
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from utils import IMAGE_PROMPT, DATAFRAME_PROMPT
|
||||
|
||||
|
||||
class FileType(Enum):
|
||||
IMAGE = "image"
|
||||
AUDIO = "audio"
|
||||
VIDEO = "video"
|
||||
DATAFRAME = "dataframe"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class Handler:
|
||||
def __init__(self, handle_func: Dict[FileType, Callable]):
|
||||
self.handle_func = handle_func
|
||||
|
||||
def handle(self, i: int, file_name: str) -> str:
|
||||
"""
|
||||
Parse file type from file name (ex. image, audio, video, dataframe, etc.)
|
||||
"""
|
||||
file_type = file_name.split("?")[0]
|
||||
|
||||
if file_type.endswith(".png") or file_type.endswith(".jpg"):
|
||||
return self.handle_image(i, file_name)
|
||||
elif file_type.endswith(".mp3") or file_type.endswith(".wav"):
|
||||
return self.handle_audio(i, file_name)
|
||||
elif file_type.endswith(".mp4") or file_type.endswith(".avi"):
|
||||
return self.handle_video(i, file_name)
|
||||
elif file_type.endswith(".csv"):
|
||||
return self.handle_dataframe(i, file_name)
|
||||
else:
|
||||
return self.handle_unknown(i, file_name)
|
||||
|
||||
def handle_image(self, i: int, remote_filename: str) -> str:
|
||||
img_data = requests.get(remote_filename).content
|
||||
local_filename = os.path.join("image", str(uuid.uuid4())[0:8] + ".png")
|
||||
with open(local_filename, "wb") as f:
|
||||
size = f.write(img_data)
|
||||
print(f"Inputs: {remote_filename} ({size//1000}MB) => {local_filename}")
|
||||
img = Image.open(local_filename)
|
||||
width, height = img.size
|
||||
ratio = min(512 / width, 512 / height)
|
||||
width_new, height_new = (round(width * ratio), round(height * ratio))
|
||||
img = img.resize((width_new, height_new))
|
||||
img = img.convert("RGB")
|
||||
img.save(local_filename, "PNG")
|
||||
print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
|
||||
try:
|
||||
description = self.handle_func[FileType.IMAGE](local_filename)
|
||||
except Exception as e:
|
||||
return "Error: " + str(e)
|
||||
|
||||
return IMAGE_PROMPT.format(
|
||||
i=i, filename=local_filename, description=description
|
||||
)
|
||||
|
||||
def handle_audio(self, i: int, remote_filename: str) -> str:
|
||||
return ""
|
||||
|
||||
def handle_video(self, i: int, remote_filename: str) -> str:
|
||||
return ""
|
||||
|
||||
def handle_dataframe(self, i: int, remote_filename: str) -> str:
|
||||
content = requests.get(remote_filename).content
|
||||
local_filename = os.path.join("dataframe/", str(uuid.uuid4())[0:8] + ".csv")
|
||||
with open(local_filename, "wb") as f:
|
||||
size = f.write(content)
|
||||
print(f"Inputs: {remote_filename} ({size//1000}MB) => {local_filename}")
|
||||
df = pd.read_csv(local_filename)
|
||||
try:
|
||||
description = str(df.describe())
|
||||
except Exception as e:
|
||||
return "Error: " + str(e)
|
||||
|
||||
return DATAFRAME_PROMPT.format(
|
||||
i=i, filename=local_filename, description=description
|
||||
)
|
||||
|
||||
def handle_unknown(self, i: int, file: str) -> str:
|
||||
return ""
|
||||
@ -1,115 +1,73 @@
|
||||
from typing import List, TypedDict, Callable
|
||||
from typing import List, TypedDict
|
||||
import re
|
||||
|
||||
from langchain.agents import load_tools
|
||||
from langchain.agents.initialize import initialize_agent
|
||||
from langchain.agents.tools import Tool
|
||||
|
||||
|
||||
from fastapi import FastAPI
|
||||
from pydantic import BaseModel
|
||||
from dotenv import load_dotenv
|
||||
from s3 import upload
|
||||
|
||||
from llm import ChatOpenAI
|
||||
from file import handle
|
||||
from utils import (
|
||||
AWESOMEGPT_PREFIX,
|
||||
AWESOMEGPT_SUFFIX,
|
||||
ERROR_PROMPT,
|
||||
)
|
||||
from tools import AWESOME_MODEL, memory
|
||||
|
||||
load_dotenv()
|
||||
from utils import ERROR_PROMPT
|
||||
from agent import get_agent
|
||||
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
|
||||
print("Initializing AwesomeGPT")
|
||||
llm = ChatOpenAI(temperature=0)
|
||||
tools = [
|
||||
*load_tools(
|
||||
["python_repl", "serpapi", "wikipedia", "bing-search"],
|
||||
llm=llm,
|
||||
),
|
||||
]
|
||||
|
||||
for class_name, instance in AWESOME_MODEL.items():
|
||||
for e in dir(instance):
|
||||
if e.startswith("inference"):
|
||||
func = getattr(instance, e)
|
||||
tools.append(Tool(name=func.name, description=func.description, func=func))
|
||||
|
||||
agent = initialize_agent(
|
||||
tools,
|
||||
llm,
|
||||
agent="chat-conversational-react-description",
|
||||
verbose=True,
|
||||
memory=memory,
|
||||
agent_kwargs={
|
||||
"system_message": AWESOMEGPT_PREFIX,
|
||||
"human_message": AWESOMEGPT_SUFFIX,
|
||||
},
|
||||
)
|
||||
agent, handler = get_agent()
|
||||
|
||||
|
||||
class Request(BaseModel):
|
||||
text: str
|
||||
state: List[str]
|
||||
files: List[str]
|
||||
key: str
|
||||
query: str
|
||||
files: List[str]
|
||||
|
||||
|
||||
class Response(TypedDict):
|
||||
text: str
|
||||
response: str
|
||||
additional: List[str]
|
||||
files: List[str]
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def index():
|
||||
return {"message": "Hello World"}
|
||||
return {"message": "Hello World. I'm AwesomeGPT."}
|
||||
|
||||
|
||||
@app.post("/command")
|
||||
async def command(request: Request) -> Response:
|
||||
text = request.text
|
||||
state = request.state
|
||||
query = request.query
|
||||
files = request.files
|
||||
key = request.key
|
||||
|
||||
print("=============== Running =============")
|
||||
print("Inputs:", text, state, files)
|
||||
print("Inputs:", query, files)
|
||||
# TODO - add state to memory (use key)
|
||||
|
||||
print("======>Previous memory:\n %s" % agent.memory)
|
||||
|
||||
promptedText = ""
|
||||
promptedQuery = ""
|
||||
import time
|
||||
|
||||
for i, file in enumerate(files):
|
||||
promptedText += handle(file)(i + 1, file)
|
||||
promptedQuery += handler.handle(i + 1, file)
|
||||
|
||||
promptedText += text
|
||||
promptedQuery += query
|
||||
|
||||
print("======>Prompted Text:\n %s" % promptedText)
|
||||
print("======>Prompted Text:\n %s" % promptedQuery)
|
||||
|
||||
try:
|
||||
res = agent({"input": promptedText})
|
||||
res = agent({"input": promptedQuery})
|
||||
except Exception as e:
|
||||
try:
|
||||
res = agent(
|
||||
{
|
||||
"input": ERROR_PROMPT.format(promptedText=promptedText, e=str(e)),
|
||||
"input": ERROR_PROMPT.format(promptedQuery=promptedQuery, e=str(e)),
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
return {"text": promptedText, "response": str(e), "additional": []}
|
||||
return {"response": str(e), "files": []}
|
||||
|
||||
images = re.findall("(image/\S*png)", res["output"])
|
||||
dataframes = re.findall("(dataframe/\S*csv)", res["output"])
|
||||
|
||||
return {
|
||||
"text": promptedText,
|
||||
"response": res["output"],
|
||||
"additional": [upload(image) for image in images],
|
||||
"files": [upload(image) for image in images]
|
||||
+ [upload(dataframe) for dataframe in dataframes],
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue