fix: don't rewrite clean file if exists!

1 year ago · dfee6963a1
parent eac7734cbf
commit dfee6963a1
1 changed files with 3 additions and 1 deletions
--- a/clean.py
+++ b/clean.py
@ -6,8 +6,10 @@ import jsonlines
 import pandas as pd


-prompt_generation_dir = "prompts-reponses"
+prompt_generation_dir = "raw_data_sanity_cleaned_delobotomized"
 for file in glob.glob(os.path.join(prompt_generation_dir, "*.jsonl")):
+    if "clean" in file:
+        continue
    data = []
    print(file)
    with open(file) as f: