tiny mixtral running locally

Dockerfile

@@ -23,17 +23,17 @@ RUN conda install python~=3.10.12 pip && \
 
 VOLUME /cache
 ENV PETALS_CACHE=/cache
-COPY pip.freeze petals/pip.freeze
-RUN pip install --no-cache-dir -r petals/pip.freeze
-COPY pip.freeze2 petals/pip.freeze2
-RUN pip install --no-cache-dir -r petals/pip.freeze2
+#COPY pip.freeze petals/pip.freeze
+#RUN pip install --no-cache-dir -r petals/pip.freeze
+#COPY pip.freeze2 petals/pip.freeze2
+#RUN pip install --no-cache-dir -r petals/pip.freeze2
 
 
 ADD tests petals/tests
 ADD LICENSE  README.md pyproject.toml setup.cfg petals/
 ADD src petals/src
 RUN pip install --no-cache-dir -e petals
-RUN pip install --no-cache-dir --upgrade transformers==4.34.0
+#RUN pip install --no-cache-dir --upgrade transformers==4.34.0
 
 WORKDIR /home/petals/
 CMD python -m petals.cli.run_server --port 31331  --num_blocks=1 Maykeye/TinyLLama-v0 

docker-compose.yml

@@ -39,7 +39,26 @@ services:
               capabilities: [gpu]
     restart: always
     env_file: ./.env
-    
+
+#
+  tinymixtral_local_gpu:
+    profiles: ["local","gpu"]
+    build: .
+    depends_on:
+       - backbone
+    command: python -m petals.cli.run_server --port 31331  --num_blocks=1 SanjiWatsuki/TinyMixtral-32x248M  --initial_peers $INITIAL_PEERS  --device=$DEVICE
+    ports:
+      - "31331:31331"
+    restart: always
+    env_file: ./.env
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+
   tinyllama_local_cpu:
     profiles: ["local","cpu"]
     build: .
@@ -104,8 +123,8 @@ services:
     restart: always
     depends_on:
       - backbone
-#    image: h4ckermike/inference.petals:main
-    image: petals-inference-test
+    image: h4ckermike/inference.petals:main
+#    image: petals-inference-test
     ports:
       - "8010:5000"
     env_file: ./.env