Archives/gpt4all
2
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all synced 2024-11-06 09:20:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: peft

Browse Source
This commit is contained in:
Zach Nussbaum 2023-04-12 03:50:54 +00:00
parent 3c11e68ec8
commit 60155de2a6
4 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
configs/train/finetune_gptj.yaml
View File

@ -2,12 +2,12 @@
model_name: "EleutherAI/gpt-j-6B" model_name: "EleutherAI/gpt-j-6B"
tokenizer_name: "EleutherAI/gpt-j-6B" tokenizer_name: "EleutherAI/gpt-j-6B"
gradient_checkpointing: true gradient_checkpointing: true
save_name: "nomic-ai/gpt4all-warmup-lr" save_name: # CHANGE
# dataset # dataset
streaming: false streaming: false
num_proc: 64 num_proc: 64
dataset_path: "nomic-ai/turbo-500k-multi" dataset_path: # CHANGE
max_length: 1024 max_length: 1024
batch_size: 32 batch_size: 32
@ -18,8 +18,8 @@ weight_decay: 0.0
eval_every: 500 eval_every: 500
eval_steps: 105 eval_steps: 105
save_every: 500 save_every: 500
log_grads_every: 500 log_grads_every: 100
output_dir: "ckpts/gpt4all-gptj-multinode" output_dir: # CHANGE
checkpoint: null checkpoint: null
lora: false lora: false
warmup_steps: 500 warmup_steps: 500
@ -27,7 +27,7 @@ num_epochs: 2
# logging # logging
wandb: true wandb: true
wandb_entity: vicuna wandb_entity: # CHANGE
wandb_project_name: vicuna wandb_project_name: # CHANGE
seed: 42 seed: 42

10
configs/train/finetune_gptj_lora.yaml
View File

@ -2,12 +2,12 @@
model_name: "EleutherAI/gpt-j-6b" model_name: "EleutherAI/gpt-j-6b"
tokenizer_name: "EleutherAI/gpt-j-6b" tokenizer_name: "EleutherAI/gpt-j-6b"
gradient_checkpointing: false gradient_checkpointing: false
save_name: "nomic-ai/gpt4all-mosaic" save_name: # CHANGE
# dataset # dataset
streaming: false streaming: false
num_proc: 64 num_proc: 64
dataset_path: "nomic-ai/turbo-500k-multi" dataset_path: # CHANGE
max_length: 1024 max_length: 1024
batch_size: 1 batch_size: 1
@ -19,7 +19,7 @@ eval_every: 500
eval_steps: 105 eval_steps: 105
save_every: 500 save_every: 500
log_grads_every: 500 log_grads_every: 500
output_dir: "ckpts/gpt4all-gptj-multinode" output_dir: # CHANGE
checkpoint: null checkpoint: null
lora: true lora: true
warmup_steps: 500 warmup_steps: 500
@ -27,7 +27,7 @@ num_epochs: 2
# logging # logging
wandb: true wandb: true
wandb_entity: zanussbaum wandb_entity: # CHANGE
wandb_project_name: mosaic wandb_project_name: # CHANGE
seed: 42 seed: 42

2
configs/train/finetune_lora.yaml
View File

@ -2,7 +2,7 @@
model_name: # update model_name: # update
tokenizer_name: # update tokenizer_name: # update
gradient_checkpointing: false gradient_checkpointing: false
save_name: "nomic-ai/gpt4all-lora-multi-turn" save_name: # CHANGE
# dataset # dataset
streaming: false streaming: false

2
peft

@ -1 +1 @@
Subproject commit deff03f2c251534fffd2511fc2d440e84cc54b1b Subproject commit 098962fa6515f2e4fe83a757f5995d3ffbb1c373