processing/text.py

"""Text processing functions"""
from typing import Dict, Generator

from config import Config
import os

CFG = Config()


def read_txt_files(directory):
    all_text = ''

    for filename in os.listdir(directory):
        if filename.endswith('.txt'):
            with open(os.path.join(directory, filename), 'r') as file:
                all_text += file.read() + '\n'

    return all_text