fix rate

immediate_reflexion.py

@@ -56,6 +56,8 @@ def run_immediate_reflexion(
 
         is_solved = exe.evaluate(
             item["entry_point"], cur_func_impl, item["test"], timeout=10)
+        if is_solved:
+            num_success += 1
 
         item["is_solved"] = is_solved
         item["reflections"] = reflections

run_reflexion_humaneval_30.sh

@@ -0,0 +1,10 @@
+python main.py \
+  --run_name "reflexion_humaneval30_py" \
+  --root_dir "root" \
+  --dataset_path ./benchmarks/humaneval-py_sample30.jsonl \
+  --strategy "reflexion" \
+  --language "py" \
+  --model "gpt-4" \
+  --pass_at_k "1" \
+  --max_iters "5" \
+  --verbose