From b8292dd7d0042cf54634ef63dd2495aa950ac214 Mon Sep 17 00:00:00 2001 From: MalikMAlna Date: Thu, 6 Apr 2023 19:56:49 -0400 Subject: [PATCH] Slight cleanup of superfluous comment and space after comma --- data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data.py b/data.py index ff79924c..72dc4574 100644 --- a/data.py +++ b/data.py @@ -57,7 +57,6 @@ def load_data(config, tokenizer): dataset_path = config["dataset_path"] if os.path.exists(dataset_path): - # check if path is a directory if os.path.isdir(dataset_path): files = glob.glob(os.path.join(dataset_path, "*_clean.jsonl")) else: @@ -68,7 +67,7 @@ def load_data(config, tokenizer): dataset = load_dataset("json", data_files=files, split="train") else: - dataset = load_dataset(dataset_path,split='train') + dataset = load_dataset(dataset_path, split='train') dataset = dataset.train_test_split(test_size=.05, seed=config["seed"])