<commit_before># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from hamcrest import *
import os
import unittest

from ...test.hamcrest import ComposedMatcher, evaluates_to
from ..hathi import get_oclc_counts_from_json, get_hathi_data_from_json

FILE_BASE = os.path.join(os.path.dirname(__file__), "files")

def readfile (filename):
    with open(os.path.join(FILE_BASE, filename), "r") as f:
        result = f.read()

    return result

EG_HATHI_ASTRO = (readfile("hathitrust-706055947.json"),
                  "mdp.39015081447313")
EG_HATHI_BUSINESS = (readfile("hathitrust-756167029.json"),
                     "mdp.39015090867675")
EG_HATHI_MIDAILY = (readfile("hathitrust-009651208.json"),
                    "mdp.39015071755826")
EG_MULTI = (readfile("hathitrust-multi-eg.json"),
            "mdp.39015071754159")

class yields_oclc_counts (ComposedMatcher):

    def __init__ (self, mdp, other):
        self.expected = (mdp, other)

    def assertion (self, item):
        actual = get_oclc_counts_from_json(*item)
        yield actual, is_(equal_to(self.expected))

class empty_hathi_data (ComposedMatcher):

    def assertion (self, item):
        yield item, evaluates_to(False)
        yield item.titles, empty()
        yield item.htids, empty()

class yields_empty_hathi_data (ComposedMatcher):

    def assertion (self, item):
        yield get_hathi_data_from_json(item), is_(empty_hathi_data())

class OclcCountHelpers (unittest.TestCase):
    htid = None

    def assert_oclc_counts (self, a, b):
        assert_that(get_oclc_counts_from_json(self.json,
                                              self.htid),
                    is_(equal_to((a, b))))

    def get_json_from_file (self, hathitrust_filename):
        files_dir = os.path.join(os.path.dirname(__file__), "files")
        full_filename = "hathitrust-{}.json".format(hathitrust_filename)
        file_path = os.path.join(files_dir, full_filename)

        with open(file_path, "r") as f:
            self.json = f.read()

class GivenNothing (OclcCountHelpers):

    def test_null_yields_0_0 (self):
        self.json = None
        self.assert_oclc_counts(0, 0)

    def test_empty_str_yields_0_0 (self):
        self.json = ""
        self.assert_oclc_counts(0, 0)

    def test_invalid_json_yields_0_0 (self):
        self.json = "{{{{"
        self.assert_oclc_counts(0, 0)

class GivenAstroJson (OclcCountHelpers):

    def setUp (self):
        self.get_json_from_file("706055947")
        self.htid = "mdp.39015081447313"

    def test_count_is_1_0 (self):
        self.assert_oclc_counts(1, 0)

class GivenBusinessJson (OclcCountHelpers):

    def setUp (self):
        self.get_json_from_file("756167029")
        self.htid = "mdp.39015090867675"

    def test_count_is_1_0 (self):
        self.assert_oclc_counts(0, 0)

class GivenMidailyJson (OclcCountHelpers):

    def setUp (self):
        self.get_json_from_file("009651208")
        self.htid = "mdp.39015071755826"

    def test_count_is_1_0 (self):
        self.assert_oclc_counts(0, 1)

class GivenMultiJson (OclcCountHelpers):

    def setUp (self):
        self.get_json_from_file("multi-eg")
        self.htid = "mdp.39015071754159"

    def test_count_is_1_0 (self):
        self.assert_oclc_counts(1, 3)

class HathiRecordDataTest (unittest.TestCase):

    def test_no_args_yields_no_data (self):
        data = get_hathi_data_from_json()
        assert_that(data, is_(empty_hathi_data()))

    def test_null_yields_no_data (self):
        assert_that(None, yields_empty_hathi_data())

    def test_empty_str_yields_no_data (self):
        assert_that("", yields_empty_hathi_data())

    def test_invalid_json_yields_no_data (self):
        assert_that("{{{{]]", yields_empty_hathi_data())

    def test_json_with_no_data_yields_no_data (self):
        assert_that('{"records":{},"items":[]}',
                    yields_empty_hathi_data())

    def test_empty_data_has_title_distance_of_1 (self):
        data = get_hathi_data_from_json()
        assert_that(data.min_title_distance("anything"),
                    is_(close_to(1, 0.001)))

class HathiRecordDataGivenAstroJson (unittest.TestCase):

    def setUp (self):
        self.data = get_hathi_data_from_json(EG_HATHI_ASTRO[0])

    def test_yields_one_title_and_one_htid (self):
        assert_that(self.data.titles, has_length(1))
        assert_that(self.data.htids, has_length(1))
<commit_msg>Remove last mention of EG_HATHI_ASTRO<commit_after># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from hamcrest import *
import os
import unittest

from ...test.hamcrest import ComposedMatcher, evaluates_to
from ..hathi import get_oclc_counts_from_json, get_hathi_data_from_json

FILE_BASE = os.path.join(os.path.dirname(__file__), "files")

def readfile (filename):
    with open(os.path.join(FILE_BASE, filename), "r") as f:
        result = f.read()

    return result

EG_HATHI_ASTRO = (readfile("hathitrust-706055947.json"),
                  "mdp.39015081447313")
EG_HATHI_BUSINESS = (readfile("hathitrust-756167029.json"),
                     "mdp.39015090867675")
EG_HATHI_MIDAILY = (readfile("hathitrust-009651208.json"),
                    "mdp.39015071755826")
EG_MULTI = (readfile("hathitrust-multi-eg.json"),
            "mdp.39015071754159")

class yields_oclc_counts (ComposedMatcher):

    def __init__ (self, mdp, other):
        self.expected = (mdp, other)

    def assertion (self, item):
        actual = get_oclc_counts_from_json(*item)
        yield actual, is_(equal_to(self.expected))

class empty_hathi_data (ComposedMatcher):

    def assertion (self, item):
        yield item, evaluates_to(False)
        yield item.titles, empty()
        yield item.htids, empty()

class yields_empty_hathi_data (ComposedMatcher):

    def assertion (self, item):
        yield get_hathi_data_from_json(item), is_(empty_hathi_data())

class OclcCountHelpers (unittest.TestCase):
    htid = None

    def assert_oclc_counts (self, a, b):
        assert_that(get_oclc_counts_from_json(self.json,
                                              self.htid),
                    is_(equal_to((a, b))))

    def get_json_from_file (self, hathitrust_filename):
        files_dir = os.path.join(os.path.dirname(__file__), "files")
        full_filename = "hathitrust-{}.json".format(hathitrust_filename)
        file_path = os.path.join(files_dir, full_filename)

        with open(file_path, "r") as f:
            self.json = f.read()

class GivenNothing (OclcCountHelpers):

    def test_null_yields_0_0 (self):
        self.json = None
        self.assert_oclc_counts(0, 0)

    def test_empty_str_yields_0_0 (self):
        self.json = ""
        self.assert_oclc_counts(0, 0)

    def test_invalid_json_yields_0_0 (self):
        self.json = "{{{{"
        self.assert_oclc_counts(0, 0)

class GivenAstroJson:

    def setUp (self):
        self.get_json_from_file("706055947")
        self.htid = "mdp.39015081447313"

class TestAstroJson (GivenAstroJson, OclcCountHelpers):

    def test_count_is_1_0 (self):
        self.assert_oclc_counts(1, 0)

class GivenBusinessJson (OclcCountHelpers):

    def setUp (self):
        self.get_json_from_file("756167029")
        self.htid = "mdp.39015090867675"

    def test_count_is_1_0 (self):
        self.assert_oclc_counts(0, 0)

class GivenMidailyJson (OclcCountHelpers):

    def setUp (self):
        self.get_json_from_file("009651208")
        self.htid = "mdp.39015071755826"

    def test_count_is_1_0 (self):
        self.assert_oclc_counts(0, 1)

class GivenMultiJson (OclcCountHelpers):

    def setUp (self):
        self.get_json_from_file("multi-eg")
        self.htid = "mdp.39015071754159"

    def test_count_is_1_0 (self):
        self.assert_oclc_counts(1, 3)

class HathiRecordDataTest (unittest.TestCase):

    def test_no_args_yields_no_data (self):
        data = get_hathi_data_from_json()
        assert_that(data, is_(empty_hathi_data()))

    def test_null_yields_no_data (self):
        assert_that(None, yields_empty_hathi_data())

    def test_empty_str_yields_no_data (self):
        assert_that("", yields_empty_hathi_data())

    def test_invalid_json_yields_no_data (self):
        assert_that("{{{{]]", yields_empty_hathi_data())

    def test_json_with_no_data_yields_no_data (self):
        assert_that('{"records":{},"items":[]}',
                    yields_empty_hathi_data())

    def test_empty_data_has_title_distance_of_1 (self):
        data = get_hathi_data_from_json()
        assert_that(data.min_title_distance("anything"),
                    is_(close_to(1, 0.001)))

class TestAstroJsonRecordData (GivenAstroJson, OclcCountHelpers):

    def setUp (self):
        super().setUp()
        self.data = get_hathi_data_from_json(self.json)

    def test_yields_one_title_and_one_htid (self):
        assert_that(self.data.titles, has_length(1))
        assert_that(self.data.htids, has_length(1))

{'fraction_non_alphanumeric': 0.10676544285869921, 'fraction_numerical': 0.04499400806188038, 'mean_word_length': 3.246068455134135, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '217393', 'n_tokens_mistral': 3574, 'n_tokens_neox': 3219, 'n_words': 575}