<commit_before>import os
from AlphaGo.training.reinforcement_policy_trainer import run_training, log_loss, run_n_games
import unittest
import numpy as np
import numpy.testing as npt
import AlphaGo.go as go
from keras.optimizers import SGD
from AlphaGo.models.policy import CNNPolicy
from AlphaGo.util import sgf_iter_states

SGF_FOLDER = "tests/test_data/sgf/"


def _is_sgf(fname):
    return fname.strip()[-4:] == ".sgf"


def _list_mock_games(path):
    """helper function to get all SGF files in a directory (does not recurse)
    """
    files = os.listdir(path)
    return (os.path.join(path, f) for f in files if _is_sgf(f))<commit_msg>added os.path.join<commit_after>import os
from AlphaGo.training.reinforcement_policy_trainer import run_training, log_loss, run_n_games
import unittest
import numpy as np
import numpy.testing as npt
import AlphaGo.go as go
from keras.optimizers import SGD
from AlphaGo.models.policy import CNNPolicy
from AlphaGo.util import sgf_iter_states

SGF_FOLDER = os.path.join('tests', 'test_data', 'sgf/')

def _is_sgf(fname):
    return fname.strip()[-4:] == ".sgf"


def _list_mock_games(path):
    """helper function to get all SGF files in a directory (does not recurse)
    """
    files = os.listdir(path)
    return (os.path.join(path, f) for f in files if _is_sgf(f))
{'fraction_non_alphanumeric': 0.09618320610687023, 'fraction_numerical': 0.0015267175572519084, 'mean_word_length': 4.724890829694323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1549820', 'n_tokens_mistral': 463, 'n_tokens_neox': 447, 'n_words': 136}