<commit_before>import inspect
import os
import shutil
import sys
import unittest
import yaml
try:
    from unittest.mock import patch, Mock
except ImportError:
    try:
        from mock import patch, Mock
    except ImportError:
        exit("mock not found. Run: `pip install mock`")

import gnupg


__file__ = os.path.relpath(inspect.getsourcefile(lambda _: None))

TEST_DIR = os.path.join(os.path.dirname(os.path.relpath(__file__)))
TEST_DATA_DIR = os.path.join(TEST_DIR, "data")

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.relpath(__file__))))
import pysswords
from pysswords.db import Database


def mock_create_keyring(path, *args, **kwargs):
    """Import key.asc instead of generating new key
    passphrase used to create the key was '<PASSWORD>'"""
    keyring_path = os.path.join(path, ".keys")
    gpg = gnupg.GPG(homedir=keyring_path)
    with open(os.path.join(TEST_DATA_DIR, "key.asc")) as keyfile:
        gpg.import_keys(keyfile.read())
    return gpg.list_keys()[0]


def mock_gen_key(self, key_input):
    return mock_create_keyring(self.homedir)


def some_credential(**kwargs):
    return pysswords.db.Credential(
        name=kwargs.get("name", "example.com"),
        login=kwargs.get("login", "john.doe"),
        password=kwargs.get("password", "--BEGIN GPG-- X --END GPG--"),
        comment=kwargs.get("comment", "Some comments"),
    )


# @patch("pysswords.crypt.create_keyring", new=mock_create_keyring)
class CryptTests(unittest.TestCase):

    def setUp(self):
        self.path = os.path.join(TEST_DATA_DIR, "database")
        self.passphrase = "<PASSWORD>"
        self.cleanup()

    def tearDown(self):
        self.cleanup()

    def cleanup(self):
        if os.path.exists(self.path):
            shutil.rmtree(self.path)

    @patch("pysswords.crypt.create_keyring", new=mock_create_keyring)
    def test_create_keyring_adds_gpg_keys_to_path(self):
        pysswords.crypt.create_keyring(self.path, self.passphrase)
        pubring = os.path.join(self.path, ".keys", "pubring.gpg")
        secring = os.path.join(self.path, ".keys", "secring.gpg")
        self.assertTrue(os.path.isfile(pubring))
        self.assertTrue(os.path.isfile(secring))

    @patch("pysswords.crypt.create_keyring", new=mock_create_keyring)
    def test_create_keyring_adds_key_to_keyring(self):
        database = Database(self.path)
        pysswords.crypt.create_keyring(self.path, self.passphrase)
        gpg = gnupg.GPG(homedir=database.keys_path)
        self.assertEqual(1, len(gpg.list_keys()))

    @patch("pysswords.crypt.gnupg.GPG.gen_key", new=mock_gen_key)
    def test_generate_keys_return_valid_key(self):
        key = pysswords.crypt.generate_keys(self.path, self.passphrase)
        self.assertIsNotNone(key)
        self.assertEqual(key["fingerprint"],
                         '0927E8F7C7794683AFABDED698894B2D11886DF4')

    def test_generate_key_input_returns_batch_string_with_passphrase(self):
        batch = pysswords.crypt.generate_key_input(self.path, self.passphrase)
        self.assertIn("\nPassphrase: {}".format(self.passphrase), batch)

    def test_create_keyring_generate_keys(self):
        self.cleanup()
        with patch("pysswords.crypt.generate_keys") as mocked_generate:
            pysswords.crypt.create_keyring(self.path, self.passphrase)
            self.assertTrue(mocked_generate.called)


@patch("pysswords.db.database.create_keyring", new=mock_create_keyring)
class DatabaseTests(unittest.TestCase):

    __name__ = "DatabaseTests"

    def setUp(self):
        self.path = os.path.join(TEST_DATA_DIR, "database")
        self.passphrase = "<PASSWORD>"
        self.cleanup()

    def tearDown(self):
        self.cleanup()

    def cleanup(self):
        if os.path.exists(self.path):
            shutil.rmtree(self.path)

    def test_create_makedirs_at_path(self):
        test_path = os.path.join(self.path, "creation")
        if os.path.exists(self.path):
            shutil.rmtree(self.path)
        pysswords.db.Database.create(test_path, self.passphrase)
        self.assertTrue(os.path.exists(test_path))

    def test_create_keyring(self):
        database = Database.create(self.path, self.passphrase)
        self.assertIsInstance(database, pysswords.db.Database)
        self.assertTrue(len(database.gpg.list_keys()) == 1)

    def test_keys_path_returns_database_path_joined_with_dot_keys(self):
        database = Database.create(self.path, self.passphrase)
        keys_path = database.keys_path
        self.assertEqual(keys_path, os.path.join(self.path, ".keys"))

    def test_add_credential_make_dir_in_dbpath_with_credential_name(self):
        database = Database.create(self.path, self.passphrase)
        database.add(some_credential())
        credential_dir = os.path.join(self.path, some_credential().name)
        self.assertTrue(os.path.exists(credential_dir))
        self.assertTrue(os.path.isdir(credential_dir))

    def test_add_credential_createas_pyssword_file_named_after_login(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        database.add(credential)
        credential_dir = os.path.join(self.path, credential.name)
        credential_filename = "{}.pyssword".format(credential.login)
        credential_file = os.path.join(credential_dir, credential_filename)
        self.assertTrue(os.path.isfile(credential_file))
        with open(credential_file) as f:
            self.assertEqual(yaml.load(f.read()), credential)

    def test_add_credential_creates_dir_when_credential_name_is_a_dir(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential(name="emails/misc/example.com")
        emails_dir = os.path.join(self.path, "emails")
        misc_dir = os.path.join(emails_dir, "misc")
        database.add(credential)
        self.assertTrue(os.path.isdir(emails_dir))
        self.assertTrue(os.path.isdir(misc_dir))

    def test_add_credential_returns_credential_path(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        credential_path = database.add(credential)
        expected_path = os.path.join(
            self.path,
            os.path.basename(credential.name),
            "{}.pyssword".format(credential.login)
        )
        self.assertEqual(credential_path, expected_path)

    def test_gpg_returns_valid_gnupg_gpg_object(self):
        database = Database.create(self.path, self.passphrase)
        gpg = database.gpg
        self.assertIsInstance(gpg, pysswords.db.database.gnupg.GPG)

    def test_credentials_returns_a_list_of_all_added_credentials(self):
        database = Database.create(self.path, self.passphrase)
        database.add(some_credential(name="example.com"))
        database.add(some_credential(name="archive.org"))
        credentials = database.credentials
        self.assertIsInstance(credentials, list)
        self.assertEqual(2, len(credentials))
        for credential in credentials:
            self.assertIsInstance(credential, pysswords.db.Credential)

    def test_add_repeated_credential_without_overwrite_on_raises_error(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        database.add(credential)
        with self.assertRaises(pysswords.db.CredentialExistsError):
            database.add(credential)

    def test_remove_deletes_pysswords_file(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        credential_path = pysswords.db.credential.expandpath(
            self.path,
            credential)
        database.add(credential)
        self.assertTrue(os.path.isfile(credential_path))
        database.remove(credential)
        self.assertFalse(os.path.isfile(credential_path))

    def test_remove_deletes_pyssword_dir_if_empty_after_deletion(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        credential_path = pysswords.db.credential.expandpath(
            self.path,
            credential)

        database.add(credential)
        self.assertTrue(os.path.exists(os.path.dirname(credential_path)))
        database.remove(credential)
        self.assertFalse(os.path.exists(os.path.dirname(credential_path)))

    def test_get_credential_by_name_returns_expected_credential(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential(name="example.com")
        database.add(credential)
        found = database.credential(name=credential.name)

        self.assertIsInstance(found, list)
        self.assertTrue(all(True for c in found
                            if isinstance(c, pysswords.db.Credential)))
        self.assertTrue(any(True for c in found
                            if c == credential))

    def test_get_returns_unique_credential_when_login_is_passed(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential(name="example.com")
        credential2 = some_credential(name="example.com", login="jonny.doe")
        database.add(credential)
        database.add(credential2)
        found = database.credential(name=credential.name,
                                    login=credential.login)
        self.assertEqual(found, [credential])

    def test_get_returns_no_element_when_name_not_found(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential(name="example.com")
        database.add(credential)
        found = database.credential(name="not added")
        self.assertListEqual(found, [])

    def test_search_database_returns_list_with_matched_credentials(self):
        database = Database.create(self.path, self.passphrase)
        database.add(some_credential(name="example.com"))
        database.add(some_credential(name="github.com"))
        database.add(some_credential(name="twitter.com"))

        self.assertEqual(len(database.search("it")), 2)
        self.assertEqual(len(database.search("github")), 1)
        self.assertEqual(len(database.search("not there")), 0)

    def test_encrypt_text_returns_valid_encryption_ascii_gpg(self):
        database = Database.create(self.path, self.passphrase)
        text = "secret"
        encrypted = database.encrypt(text)
        self.assertIn("-BEGIN PGP MESSAGE-", encrypted)
        self.assertIn("-END PGP MESSAGE-", encrypted)

    @unittest.skip
    def test_key_returns_expected_key_fingerprint(self):
        database = Database.create(self.path, self.passphrase)
        self.assertEqual(
            database.key(),
            "<KEY>")

    def test_key_returns_private_key_when_private_is_true(self):
        to_patch = "pysswords.db.database.gnupg.GPG.list_keys"
        with patch(to_patch) as mocked_list_keys:
            mocked_list_keys.return_value = [
                {"fingerprint": "<KEY>"}
            ]
            database = Database.create(self.path, self.passphrase)
            database.key(private=True)
            database.gpg.list_keys.assert_any_call_with(secret=True)
        self.assertEqual(
            database.key(private=True),
            "<KEY>")

    def test_decrypt_returns_plain_text_data(self):
        database = Database.create(self.path, self.passphrase)
        text = "secret"
        encrypted = database.encrypt(text)
        decrypted = database.decrypt(encrypted, passphrase=self.passphrase)
        self.assertEqual(decrypted, text)


class CredentialTests(unittest.TestCase):

    def setUp(self):
        self.path = os.path.join(TEST_DATA_DIR, "database")
        self.cleanup()

    def tearDown(self):
        self.cleanup()

    def cleanup(self):
        if os.path.exists(self.path):
            shutil.rmtree(self.path)

    def test_credential_expandpath_returns_expected_path_to_credential(self):
        credential = some_credential()
        credential_path = pysswords.db.credential.expandpath(
            self.path,
            credential
        )
        expected_path = os.path.join(
            self.path,
            os.path.basename(credential.name),
            "{}.pyssword".format(credential.login)
        )
        self.assertEqual(credential_path, expected_path)

    def test_credential_content_returns_yaml_content_parseable_to_dict(self):
        content = pysswords.db.credential.content(some_credential())
        self.assertEqual(yaml.load(content), some_credential())


class UtilsTests(unittest.TestCase):

    def test_which_handle_windows_exe_extension_for_executables(self):
        with patch("pysswords.utils.os") as mocker:
            mocker.name = "nt"
            mocker.environ = {"PATH": "/"}
            mocker.pathsep = ":"
            mocked_join = Mock()
            mocker.path.join = mocked_join
            pysswords.utils.which("python")
            mocked_join.assert_any_call("/", "python.exe")


if __name__ == "__main__":
    if sys.version_info >= (3,):
        unittest.main(warnings=False)
    else:
        unittest.main()
<commit_msg>Update test key fingerprint and build test key function<commit_after>import inspect
import os
import shutil
import sys
import unittest
import yaml
try:
    from unittest.mock import patch, Mock
except ImportError:
    try:
        from mock import patch, Mock
    except ImportError:
        exit("mock not found. Run: `pip install mock`")

import gnupg


__file__ = os.path.relpath(inspect.getsourcefile(lambda _: None))

TEST_DIR = os.path.join(os.path.dirname(os.path.relpath(__file__)))
TEST_DATA_DIR = os.path.join(TEST_DIR, "data")

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.relpath(__file__))))
import pysswords
from pysswords.db import Database


def build_keys():
    gpg = gnupg.GPG(homedir="/tmp/pysswords")
    key_input = gpg.gen_key_input(
        name_real="Pysswords",
        name_email="<EMAIL>",
        name_comment="Auto-generated by Pysswords",
        key_length=512,
        expire_date=0,
        passphrase="<PASSWORD>"
    )
    key = gpg.gen_key(key_input)
    ascii_armored_public_keys = gpg.export_keys(key)
    ascii_armored_private_keys = gpg.export_keys(key, True)
    with open(os.path.join(TEST_DATA_DIR, "newkey.asc"), 'w') as f:
        f.write(ascii_armored_public_keys)
        f.write(ascii_armored_private_keys)


def mock_create_keyring(path, *args, **kwargs):
    """Import key.asc instead of generating new key
    passphrase used to create the key was '<PASSWORD>'"""
    keyring_path = os.path.join(path, ".keys")
    gpg = gnupg.GPG(homedir=keyring_path)
    with open(os.path.join(TEST_DATA_DIR, "key.asc")) as keyfile:
        gpg.import_keys(keyfile.read())
    return gpg.list_keys()[0]


def mock_gen_key(self, key_input):
    return mock_create_keyring(self.homedir)


def some_credential(**kwargs):
    return pysswords.db.Credential(
        name=kwargs.get("name", "example.com"),
        login=kwargs.get("login", "john.doe"),
        password=kwargs.get("password", "--<PASSWORD>-- X --END <PASSWORD>--"),
        comment=kwargs.get("comment", "Some comments"),
    )


# @patch("pysswords.crypt.create_keyring", new=mock_create_keyring)
class CryptTests(unittest.TestCase):

    def setUp(self):
        self.path = os.path.join(TEST_DATA_DIR, "database")
        self.passphrase = "<PASSWORD>"
        self.cleanup()

    def tearDown(self):
        self.cleanup()

    def cleanup(self):
        if os.path.exists(self.path):
            shutil.rmtree(self.path)

    @patch("pysswords.crypt.create_keyring", new=mock_create_keyring)
    def test_create_keyring_adds_gpg_keys_to_path(self):
        pysswords.crypt.create_keyring(self.path, self.passphrase)
        pubring = os.path.join(self.path, ".keys", "pubring.gpg")
        secring = os.path.join(self.path, ".keys", "secring.gpg")
        self.assertTrue(os.path.isfile(pubring))
        self.assertTrue(os.path.isfile(secring))

    @patch("pysswords.crypt.create_keyring", new=mock_create_keyring)
    def test_create_keyring_adds_key_to_keyring(self):
        database = Database(self.path)
        pysswords.crypt.create_keyring(self.path, self.passphrase)
        gpg = gnupg.GPG(homedir=database.keys_path)
        self.assertEqual(1, len(gpg.list_keys()))

    @patch("pysswords.crypt.gnupg.GPG.gen_key", new=mock_gen_key)
    def test_generate_keys_return_valid_key(self):
        key = pysswords.crypt.generate_keys(self.path, self.passphrase)
        self.assertIsNotNone(key)
        self.assertEqual(key["fingerprint"],
                         '<KEY>')

    def test_generate_key_input_returns_batch_string_with_passphrase(self):
        batch = pysswords.crypt.generate_key_input(self.path, self.passphrase)
        self.assertIn("\nPassphrase: {}".format(self.passphrase), batch)

    def test_create_keyring_generate_keys(self):
        self.cleanup()
        with patch("pysswords.crypt.generate_keys") as mocked_generate:
            pysswords.crypt.create_keyring(self.path, self.passphrase)
            self.assertTrue(mocked_generate.called)


@patch("pysswords.db.database.create_keyring", new=mock_create_keyring)
class DatabaseTests(unittest.TestCase):

    __name__ = "DatabaseTests"

    def setUp(self):
        self.path = os.path.join(TEST_DATA_DIR, "database")
        self.passphrase = "<PASSWORD>"
        self.cleanup()

    def tearDown(self):
        self.cleanup()

    def cleanup(self):
        if os.path.exists(self.path):
            shutil.rmtree(self.path)

    def test_create_makedirs_at_path(self):
        test_path = os.path.join(self.path, "creation")
        if os.path.exists(self.path):
            shutil.rmtree(self.path)
        pysswords.db.Database.create(test_path, self.passphrase)
        self.assertTrue(os.path.exists(test_path))

    def test_create_keyring(self):
        database = Database.create(self.path, self.passphrase)
        self.assertIsInstance(database, pysswords.db.Database)
        self.assertTrue(len(database.gpg.list_keys()) == 1)

    def test_keys_path_returns_database_path_joined_with_dot_keys(self):
        database = Database.create(self.path, self.passphrase)
        keys_path = database.keys_path
        self.assertEqual(keys_path, os.path.join(self.path, ".keys"))

    def test_add_credential_make_dir_in_dbpath_with_credential_name(self):
        database = Database.create(self.path, self.passphrase)
        database.add(some_credential())
        credential_dir = os.path.join(self.path, some_credential().name)
        self.assertTrue(os.path.exists(credential_dir))
        self.assertTrue(os.path.isdir(credential_dir))

    def test_add_credential_createas_pyssword_file_named_after_login(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        database.add(credential)
        credential_dir = os.path.join(self.path, credential.name)
        credential_filename = "{}.pyssword".format(credential.login)
        credential_file = os.path.join(credential_dir, credential_filename)
        self.assertTrue(os.path.isfile(credential_file))
        with open(credential_file) as f:
            self.assertEqual(yaml.load(f.read()), credential)

    def test_add_credential_creates_dir_when_credential_name_is_a_dir(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential(name="emails/misc/example.com")
        emails_dir = os.path.join(self.path, "emails")
        misc_dir = os.path.join(emails_dir, "misc")
        database.add(credential)
        self.assertTrue(os.path.isdir(emails_dir))
        self.assertTrue(os.path.isdir(misc_dir))

    def test_add_credential_returns_credential_path(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        credential_path = database.add(credential)
        expected_path = os.path.join(
            self.path,
            os.path.basename(credential.name),
            "{}.pyssword".format(credential.login)
        )
        self.assertEqual(credential_path, expected_path)

    def test_gpg_returns_valid_gnupg_gpg_object(self):
        database = Database.create(self.path, self.passphrase)
        gpg = database.gpg
        self.assertIsInstance(gpg, pysswords.db.database.gnupg.GPG)

    def test_credentials_returns_a_list_of_all_added_credentials(self):
        database = Database.create(self.path, self.passphrase)
        database.add(some_credential(name="example.com"))
        database.add(some_credential(name="archive.org"))
        credentials = database.credentials
        self.assertIsInstance(credentials, list)
        self.assertEqual(2, len(credentials))
        for credential in credentials:
            self.assertIsInstance(credential, pysswords.db.Credential)

    def test_add_repeated_credential_without_overwrite_on_raises_error(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        database.add(credential)
        with self.assertRaises(pysswords.db.CredentialExistsError):
            database.add(credential)

    def test_remove_deletes_pysswords_file(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        credential_path = pysswords.db.credential.expandpath(
            self.path,
            credential)
        database.add(credential)
        self.assertTrue(os.path.isfile(credential_path))
        database.remove(credential)
        self.assertFalse(os.path.isfile(credential_path))

    def test_remove_deletes_pyssword_dir_if_empty_after_deletion(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential()
        credential_path = pysswords.db.credential.expandpath(
            self.path,
            credential)

        database.add(credential)
        self.assertTrue(os.path.exists(os.path.dirname(credential_path)))
        database.remove(credential)
        self.assertFalse(os.path.exists(os.path.dirname(credential_path)))

    def test_get_credential_by_name_returns_expected_credential(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential(name="example.com")
        database.add(credential)
        found = database.credential(name=credential.name)

        self.assertIsInstance(found, list)
        self.assertTrue(all(True for c in found
                            if isinstance(c, pysswords.db.Credential)))
        self.assertTrue(any(True for c in found
                            if c == credential))

    def test_get_returns_unique_credential_when_login_is_passed(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential(name="example.com")
        credential2 = some_credential(name="example.com", login="jonny.doe")
        database.add(credential)
        database.add(credential2)
        found = database.credential(name=credential.name,
                                    login=credential.login)
        self.assertEqual(found, [credential])

    def test_get_returns_no_element_when_name_not_found(self):
        database = Database.create(self.path, self.passphrase)
        credential = some_credential(name="example.com")
        database.add(credential)
        found = database.credential(name="not added")
        self.assertListEqual(found, [])

    def test_search_database_returns_list_with_matched_credentials(self):
        database = Database.create(self.path, self.passphrase)
        database.add(some_credential(name="example.com"))
        database.add(some_credential(name="github.com"))
        database.add(some_credential(name="twitter.com"))

        self.assertEqual(len(database.search("it")), 2)
        self.assertEqual(len(database.search("github")), 1)
        self.assertEqual(len(database.search("not there")), 0)

    def test_encrypt_text_returns_valid_encryption_ascii_gpg(self):
        database = Database.create(self.path, self.passphrase)
        text = "secret"
        encrypted = database.encrypt(text)
        self.assertIn("-BEGIN PGP MESSAGE-", encrypted)
        self.assertIn("-END PGP MESSAGE-", encrypted)

    @unittest.skip
    def test_key_returns_expected_key_fingerprint(self):
        database = Database.create(self.path, self.passphrase)
        self.assertEqual(
            database.key(),
            "<KEY>")

    def test_key_returns_private_key_when_private_is_true(self):
        to_patch = "pysswords.db.database.gnupg.GPG.list_keys"
        with patch(to_patch) as mocked_list_keys:
            mocked_list_keys.return_value = [
                {"fingerprint": "<KEY>"}
            ]
            database = Database.create(self.path, self.passphrase)
            database.key(private=True)
            database.gpg.list_keys.assert_any_call_with(secret=True)
        self.assertEqual(
            database.key(private=True),
            "<KEY>")

    def test_decrypt_returns_plain_text_data(self):
        database = Database.create(self.path, self.passphrase)
        text = "secret"
        encrypted = database.encrypt(text)
        decrypted = database.decrypt(encrypted, passphrase=self.passphrase)
        self.assertEqual(decrypted, text)


class CredentialTests(unittest.TestCase):

    def setUp(self):
        self.path = os.path.join(TEST_DATA_DIR, "database")
        self.cleanup()

    def tearDown(self):
        self.cleanup()

    def cleanup(self):
        if os.path.exists(self.path):
            shutil.rmtree(self.path)

    def test_credential_expandpath_returns_expected_path_to_credential(self):
        credential = some_credential()
        credential_path = pysswords.db.credential.expandpath(
            self.path,
            credential
        )
        expected_path = os.path.join(
            self.path,
            os.path.basename(credential.name),
            "{}.pyssword".format(credential.login)
        )
        self.assertEqual(credential_path, expected_path)

    def test_credential_content_returns_yaml_content_parseable_to_dict(self):
        content = pysswords.db.credential.content(some_credential())
        self.assertEqual(yaml.load(content), some_credential())


class UtilsTests(unittest.TestCase):

    def test_which_handle_windows_exe_extension_for_executables(self):
        with patch("pysswords.utils.os") as mocker:
            mocker.name = "nt"
            mocker.environ = {"PATH": "/"}
            mocker.pathsep = ":"
            mocked_join = Mock()
            mocker.path.join = mocked_join
            pysswords.utils.which("python")
            mocked_join.assert_any_call("/", "python.exe")


if __name__ == "__main__":
    if sys.version_info >= (3,):
        unittest.main(warnings=False)
    else:
        unittest.main()

{'fraction_non_alphanumeric': 0.11122657264798665, 'fraction_numerical': 0.0019298571163481166, 'mean_word_length': 3.1391705069124423, 'pattern_counts': {'":': 8, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3459517', 'n_tokens_mistral': 8534, 'n_tokens_neox': 8445, 'n_words': 1170}