mirror of https://github.com/corca-ai/EVAL
refactor: handle file
parent
073b22927a
commit
71e9489ff0
@ -0,0 +1,16 @@
|
|||||||
|
FROM nvidia/cuda:11.7.0-runtime-ubuntu20.04
|
||||||
|
WORKDIR /app/
|
||||||
|
|
||||||
|
RUN \
|
||||||
|
apt-get update && \
|
||||||
|
apt-get install -y python3 python3-pip
|
||||||
|
RUN apt-get install uvicorn -y
|
||||||
|
|
||||||
|
RUN pip install --upgrade pip
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install -r requirements.txt
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
ENTRYPOINT ["sleep", "infinity"]
|
||||||
|
# ENTRYPOINT ["python3", "-m", "uvicorn", "main:app", "--reload", "--host=0.0.0.0", "--port=8000"]
|
@ -0,0 +1,38 @@
|
|||||||
|
# Usage
|
||||||
|
|
||||||
|
### S3
|
||||||
|
|
||||||
|
1. Create a bucket.
|
||||||
|
2. Turn off the "Block all public access" setting for the bucket. ![image](assets/block_public_access.png)
|
||||||
|
3. Add the following text to Bucket Policy.
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Sid": "AllowPublicRead",
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Principal": {
|
||||||
|
"AWS": "*"
|
||||||
|
},
|
||||||
|
"Action": "s3:GetObject",
|
||||||
|
"Resource": "arn:aws:s3:::{your-bucket-name}/*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Environment
|
||||||
|
|
||||||
|
You must need this environments.
|
||||||
|
|
||||||
|
```
|
||||||
|
OPENAI_API_KEY
|
||||||
|
```
|
||||||
|
|
||||||
|
You need this environments.
|
||||||
|
|
||||||
|
```
|
||||||
|
serpapi: SERPAPI_API_KEY
|
||||||
|
bing-search: BING_SEARCH_URL, BING_SUBSCRIPTION_KEY
|
||||||
|
```
|
@ -0,0 +1,79 @@
|
|||||||
|
from typing import Dict, List, Tuple
|
||||||
|
|
||||||
|
from llm import ChatOpenAI
|
||||||
|
from langchain.agents import load_tools
|
||||||
|
from langchain.agents.agent import AgentExecutor
|
||||||
|
from langchain.agents.tools import Tool
|
||||||
|
from langchain.agents.initialize import initialize_agent
|
||||||
|
from langchain.chains.conversation.memory import ConversationBufferMemory
|
||||||
|
|
||||||
|
from utils import AWESOMEGPT_PREFIX, AWESOMEGPT_SUFFIX
|
||||||
|
|
||||||
|
from tools.cpu import (
|
||||||
|
RequestsGet,
|
||||||
|
WineDB,
|
||||||
|
ExitConversation,
|
||||||
|
)
|
||||||
|
from tools.gpu import (
|
||||||
|
ImageEditing,
|
||||||
|
InstructPix2Pix,
|
||||||
|
Text2Image,
|
||||||
|
ImageCaptioning,
|
||||||
|
VisualQuestionAnswering,
|
||||||
|
)
|
||||||
|
from handler import Handler, FileType
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent() -> Tuple[AgentExecutor, Handler]:
|
||||||
|
print("Initializing AwesomeGPT")
|
||||||
|
llm = ChatOpenAI(temperature=0)
|
||||||
|
tools = [
|
||||||
|
*load_tools(
|
||||||
|
["python_repl", "terminal", "serpapi", "wikipedia", "bing-search"],
|
||||||
|
llm=llm,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
|
||||||
|
|
||||||
|
models = {
|
||||||
|
"RequestsGet": RequestsGet(),
|
||||||
|
"WineDB": WineDB(),
|
||||||
|
"ExitConversation": ExitConversation(memory),
|
||||||
|
"Text2Image": Text2Image("cuda"),
|
||||||
|
"ImageEditing": ImageEditing("cuda"),
|
||||||
|
"InstructPix2Pix": InstructPix2Pix("cuda"),
|
||||||
|
"VisualQuestionAnswering": VisualQuestionAnswering("cuda"),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, instance in models.items():
|
||||||
|
for e in dir(instance):
|
||||||
|
if e.startswith("inference"):
|
||||||
|
func = getattr(instance, e)
|
||||||
|
tools.append(
|
||||||
|
Tool(name=func.name, description=func.description, func=func)
|
||||||
|
)
|
||||||
|
|
||||||
|
handle_models: Dict[FileType, str] = {
|
||||||
|
FileType.IMAGE: ImageCaptioning("cuda"),
|
||||||
|
}
|
||||||
|
|
||||||
|
handler = Handler(
|
||||||
|
handle_func={
|
||||||
|
file_type: model.inference for file_type, model in handle_models.items()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return (
|
||||||
|
initialize_agent(
|
||||||
|
tools,
|
||||||
|
llm,
|
||||||
|
agent="chat-conversational-react-description",
|
||||||
|
verbose=True,
|
||||||
|
memory=memory,
|
||||||
|
agent_kwargs={
|
||||||
|
"system_message": AWESOMEGPT_PREFIX,
|
||||||
|
"human_message": AWESOMEGPT_SUFFIX,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
handler,
|
||||||
|
)
|
Binary file not shown.
After Width: | Height: | Size: 145 KiB |
@ -0,0 +1,16 @@
|
|||||||
|
version: "3"
|
||||||
|
|
||||||
|
services:
|
||||||
|
awesomegpt:
|
||||||
|
build:
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
context: .
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
reservations:
|
||||||
|
devices:
|
||||||
|
- driver: nvidia
|
||||||
|
device_ids: ["3"] # You can choose which GPU to use
|
||||||
|
capabilities: [gpu]
|
@ -1,87 +0,0 @@
|
|||||||
import os
|
|
||||||
import requests
|
|
||||||
import uuid
|
|
||||||
from typing import Callable
|
|
||||||
from enum import Enum
|
|
||||||
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
import pandas as pd
|
|
||||||
|
|
||||||
from utils import IMAGE_PROMPT, DATAFRAME_PROMPT
|
|
||||||
from tools import IMAGE_MODEL
|
|
||||||
|
|
||||||
|
|
||||||
class FileType(Enum):
|
|
||||||
IMAGE = "image"
|
|
||||||
AUDIO = "audio"
|
|
||||||
VIDEO = "video"
|
|
||||||
DATAFRAME = "dataframe"
|
|
||||||
UNKNOWN = "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
def handle(file_name: str) -> Callable:
|
|
||||||
"""
|
|
||||||
Parse file type from file name (ex. image, audio, video, dataframe, etc.)
|
|
||||||
"""
|
|
||||||
file_name = file_name.split("?")[0]
|
|
||||||
|
|
||||||
if file_name.endswith(".png") or file_name.endswith(".jpg"):
|
|
||||||
return handle_image
|
|
||||||
elif file_name.endswith(".mp3") or file_name.endswith(".wav"):
|
|
||||||
return handle_audio
|
|
||||||
elif file_name.endswith(".mp4") or file_name.endswith(".avi"):
|
|
||||||
return handle_video
|
|
||||||
elif file_name.endswith(".csv"):
|
|
||||||
return handle_dataframe
|
|
||||||
else:
|
|
||||||
return handle_unknown
|
|
||||||
|
|
||||||
|
|
||||||
def handle_image(i: int, file: str) -> str:
|
|
||||||
img_data = requests.get(file).content
|
|
||||||
filename = os.path.join("image", str(uuid.uuid4())[0:8] + ".png")
|
|
||||||
with open(filename, "wb") as f:
|
|
||||||
size = f.write(img_data)
|
|
||||||
print(f"Inputs: {file} ({size//1000}MB) => {filename}")
|
|
||||||
img = Image.open(filename)
|
|
||||||
width, height = img.size
|
|
||||||
ratio = min(512 / width, 512 / height)
|
|
||||||
width_new, height_new = (round(width * ratio), round(height * ratio))
|
|
||||||
img = img.resize((width_new, height_new))
|
|
||||||
img = img.convert("RGB")
|
|
||||||
img.save(filename, "PNG")
|
|
||||||
print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
|
|
||||||
try:
|
|
||||||
description = IMAGE_MODEL.inference(filename)
|
|
||||||
except Exception as e:
|
|
||||||
return {"text": "image upload", "response": str(e), "additional": []}
|
|
||||||
|
|
||||||
return IMAGE_PROMPT.format(i=i, filename=filename, description=description)
|
|
||||||
|
|
||||||
|
|
||||||
def handle_audio(i: int, file: str) -> str:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
|
|
||||||
def handle_video(i: int, file: str) -> str:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
|
|
||||||
def handle_dataframe(i: int, file: str) -> str:
|
|
||||||
content = requests.get(file).content
|
|
||||||
filename = os.path.join("dataframe/", str(uuid.uuid4())[0:8] + ".csv")
|
|
||||||
with open(filename, "wb") as f:
|
|
||||||
size = f.write(content)
|
|
||||||
print(f"Inputs: {file} ({size//1000}MB) => {filename}")
|
|
||||||
df = pd.read_csv(filename)
|
|
||||||
try:
|
|
||||||
description = str(df.describe())
|
|
||||||
except Exception as e:
|
|
||||||
return {"text": "image upload", "response": str(e), "additional": []}
|
|
||||||
|
|
||||||
return DATAFRAME_PROMPT.format(i=i, filename=filename, description=description)
|
|
||||||
|
|
||||||
|
|
||||||
def handle_unknown(i: int, file: str) -> str:
|
|
||||||
return ""
|
|
@ -0,0 +1,89 @@
|
|||||||
|
import os
|
||||||
|
import requests
|
||||||
|
import uuid
|
||||||
|
from typing import Callable, Dict
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from utils import IMAGE_PROMPT, DATAFRAME_PROMPT
|
||||||
|
|
||||||
|
|
||||||
|
class FileType(Enum):
|
||||||
|
IMAGE = "image"
|
||||||
|
AUDIO = "audio"
|
||||||
|
VIDEO = "video"
|
||||||
|
DATAFRAME = "dataframe"
|
||||||
|
UNKNOWN = "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
class Handler:
|
||||||
|
def __init__(self, handle_func: Dict[FileType, Callable]):
|
||||||
|
self.handle_func = handle_func
|
||||||
|
|
||||||
|
def handle(self, i: int, file_name: str) -> str:
|
||||||
|
"""
|
||||||
|
Parse file type from file name (ex. image, audio, video, dataframe, etc.)
|
||||||
|
"""
|
||||||
|
file_type = file_name.split("?")[0]
|
||||||
|
|
||||||
|
if file_type.endswith(".png") or file_type.endswith(".jpg"):
|
||||||
|
return self.handle_image(i, file_name)
|
||||||
|
elif file_type.endswith(".mp3") or file_type.endswith(".wav"):
|
||||||
|
return self.handle_audio(i, file_name)
|
||||||
|
elif file_type.endswith(".mp4") or file_type.endswith(".avi"):
|
||||||
|
return self.handle_video(i, file_name)
|
||||||
|
elif file_type.endswith(".csv"):
|
||||||
|
return self.handle_dataframe(i, file_name)
|
||||||
|
else:
|
||||||
|
return self.handle_unknown(i, file_name)
|
||||||
|
|
||||||
|
def handle_image(self, i: int, remote_filename: str) -> str:
|
||||||
|
img_data = requests.get(remote_filename).content
|
||||||
|
local_filename = os.path.join("image", str(uuid.uuid4())[0:8] + ".png")
|
||||||
|
with open(local_filename, "wb") as f:
|
||||||
|
size = f.write(img_data)
|
||||||
|
print(f"Inputs: {remote_filename} ({size//1000}MB) => {local_filename}")
|
||||||
|
img = Image.open(local_filename)
|
||||||
|
width, height = img.size
|
||||||
|
ratio = min(512 / width, 512 / height)
|
||||||
|
width_new, height_new = (round(width * ratio), round(height * ratio))
|
||||||
|
img = img.resize((width_new, height_new))
|
||||||
|
img = img.convert("RGB")
|
||||||
|
img.save(local_filename, "PNG")
|
||||||
|
print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
|
||||||
|
try:
|
||||||
|
description = self.handle_func[FileType.IMAGE](local_filename)
|
||||||
|
except Exception as e:
|
||||||
|
return "Error: " + str(e)
|
||||||
|
|
||||||
|
return IMAGE_PROMPT.format(
|
||||||
|
i=i, filename=local_filename, description=description
|
||||||
|
)
|
||||||
|
|
||||||
|
def handle_audio(self, i: int, remote_filename: str) -> str:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def handle_video(self, i: int, remote_filename: str) -> str:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def handle_dataframe(self, i: int, remote_filename: str) -> str:
|
||||||
|
content = requests.get(remote_filename).content
|
||||||
|
local_filename = os.path.join("dataframe/", str(uuid.uuid4())[0:8] + ".csv")
|
||||||
|
with open(local_filename, "wb") as f:
|
||||||
|
size = f.write(content)
|
||||||
|
print(f"Inputs: {remote_filename} ({size//1000}MB) => {local_filename}")
|
||||||
|
df = pd.read_csv(local_filename)
|
||||||
|
try:
|
||||||
|
description = str(df.describe())
|
||||||
|
except Exception as e:
|
||||||
|
return "Error: " + str(e)
|
||||||
|
|
||||||
|
return DATAFRAME_PROMPT.format(
|
||||||
|
i=i, filename=local_filename, description=description
|
||||||
|
)
|
||||||
|
|
||||||
|
def handle_unknown(self, i: int, file: str) -> str:
|
||||||
|
return ""
|
@ -1,115 +1,73 @@
|
|||||||
from typing import List, TypedDict, Callable
|
from typing import List, TypedDict
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from langchain.agents import load_tools
|
|
||||||
from langchain.agents.initialize import initialize_agent
|
|
||||||
from langchain.agents.tools import Tool
|
|
||||||
|
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from dotenv import load_dotenv
|
|
||||||
from s3 import upload
|
from s3 import upload
|
||||||
|
|
||||||
from llm import ChatOpenAI
|
from utils import ERROR_PROMPT
|
||||||
from file import handle
|
from agent import get_agent
|
||||||
from utils import (
|
|
||||||
AWESOMEGPT_PREFIX,
|
|
||||||
AWESOMEGPT_SUFFIX,
|
|
||||||
ERROR_PROMPT,
|
|
||||||
)
|
|
||||||
from tools import AWESOME_MODEL, memory
|
|
||||||
|
|
||||||
load_dotenv()
|
|
||||||
|
|
||||||
|
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
|
agent, handler = get_agent()
|
||||||
|
|
||||||
print("Initializing AwesomeGPT")
|
|
||||||
llm = ChatOpenAI(temperature=0)
|
|
||||||
tools = [
|
|
||||||
*load_tools(
|
|
||||||
["python_repl", "serpapi", "wikipedia", "bing-search"],
|
|
||||||
llm=llm,
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
for class_name, instance in AWESOME_MODEL.items():
|
|
||||||
for e in dir(instance):
|
|
||||||
if e.startswith("inference"):
|
|
||||||
func = getattr(instance, e)
|
|
||||||
tools.append(Tool(name=func.name, description=func.description, func=func))
|
|
||||||
|
|
||||||
agent = initialize_agent(
|
|
||||||
tools,
|
|
||||||
llm,
|
|
||||||
agent="chat-conversational-react-description",
|
|
||||||
verbose=True,
|
|
||||||
memory=memory,
|
|
||||||
agent_kwargs={
|
|
||||||
"system_message": AWESOMEGPT_PREFIX,
|
|
||||||
"human_message": AWESOMEGPT_SUFFIX,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Request(BaseModel):
|
class Request(BaseModel):
|
||||||
text: str
|
|
||||||
state: List[str]
|
|
||||||
files: List[str]
|
|
||||||
key: str
|
key: str
|
||||||
|
query: str
|
||||||
|
files: List[str]
|
||||||
|
|
||||||
|
|
||||||
class Response(TypedDict):
|
class Response(TypedDict):
|
||||||
text: str
|
|
||||||
response: str
|
response: str
|
||||||
additional: List[str]
|
files: List[str]
|
||||||
|
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
async def index():
|
async def index():
|
||||||
return {"message": "Hello World"}
|
return {"message": "Hello World. I'm AwesomeGPT."}
|
||||||
|
|
||||||
|
|
||||||
@app.post("/command")
|
@app.post("/command")
|
||||||
async def command(request: Request) -> Response:
|
async def command(request: Request) -> Response:
|
||||||
text = request.text
|
query = request.query
|
||||||
state = request.state
|
|
||||||
files = request.files
|
files = request.files
|
||||||
key = request.key
|
key = request.key
|
||||||
|
|
||||||
print("=============== Running =============")
|
print("=============== Running =============")
|
||||||
print("Inputs:", text, state, files)
|
print("Inputs:", query, files)
|
||||||
# TODO - add state to memory (use key)
|
# TODO - add state to memory (use key)
|
||||||
|
|
||||||
print("======>Previous memory:\n %s" % agent.memory)
|
print("======>Previous memory:\n %s" % agent.memory)
|
||||||
|
|
||||||
promptedText = ""
|
promptedQuery = ""
|
||||||
|
import time
|
||||||
|
|
||||||
for i, file in enumerate(files):
|
for i, file in enumerate(files):
|
||||||
promptedText += handle(file)(i + 1, file)
|
promptedQuery += handler.handle(i + 1, file)
|
||||||
|
|
||||||
promptedText += text
|
promptedQuery += query
|
||||||
|
|
||||||
print("======>Prompted Text:\n %s" % promptedText)
|
print("======>Prompted Text:\n %s" % promptedQuery)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
res = agent({"input": promptedText})
|
res = agent({"input": promptedQuery})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
try:
|
try:
|
||||||
res = agent(
|
res = agent(
|
||||||
{
|
{
|
||||||
"input": ERROR_PROMPT.format(promptedText=promptedText, e=str(e)),
|
"input": ERROR_PROMPT.format(promptedQuery=promptedQuery, e=str(e)),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {"text": promptedText, "response": str(e), "additional": []}
|
return {"response": str(e), "files": []}
|
||||||
|
|
||||||
images = re.findall("(image/\S*png)", res["output"])
|
images = re.findall("(image/\S*png)", res["output"])
|
||||||
|
dataframes = re.findall("(dataframe/\S*csv)", res["output"])
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"text": promptedText,
|
|
||||||
"response": res["output"],
|
"response": res["output"],
|
||||||
"additional": [upload(image) for image in images],
|
"files": [upload(image) for image in images]
|
||||||
|
+ [upload(dataframe) for dataframe in dataframes],
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue