From 30f37530d506aad0fd41c87e1ec0aa8aaf897e71 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 12 Jul 2023 09:52:20 -0700 Subject: [PATCH] Add back .replace('\r', '') --- modules/training.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/training.py b/modules/training.py index 9388436..11c3661 100644 --- a/modules/training.py +++ b/modules/training.py @@ -369,12 +369,12 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch for file_path in file_paths: if file_path.is_file(): with file_path.open('r', encoding='utf-8') as file: - raw_text += file.read() + raw_text += file.read().replace('\r', '') logger.info(f"Loaded training file: {file_path.name}") else: with open(clean_path('training/datasets', f'{raw_text_file}.txt'), 'r', encoding='utf-8') as file: - raw_text = file.read() + raw_text = file.read().replace('\r', '') cut_string = hard_cut_string.replace('\\n', '\n') out_tokens = []