<commit_before>#!/usr/bin/env python
from __future__ import division

from csv import DictReader
from collections import defaultdict
from glob import glob
from os.path import basename, join, splitext

class MetadataTable(object):
    VocabDelimiter = ':'

    @classmethod
    def fromFile(cls, fp, delimiter='\t'):
        with open(fp, 'U') as f:
            return cls(f, delimiter=delimiter)

    def __init__(self, metadata_f, delimiter='\t'):
        reader = DictReader(metadata_f, delimiter=delimiter)
        self.FieldNames = reader.fieldnames
        self._table = [row for row in reader]

    def candidateControlledFields(self, known_vocabs=None):
        cols = defaultdict(set)

        for row in self._table:
            for field in row:
                vocab_id, value = self._extract_vocab_id(row[field])

                if vocab_id is not None:
                    if known_vocabs is None or vocab_id in known_vocabs:
                        cols[field].add(vocab_id)

        return cols

    def validateControlledFields(self, known_vocabs):
        cont_fields = self.candidateControlledFields(known_vocabs=known_vocabs)

        field_to_vocab_id = {}
        for field, vocab_ids in cont_fields.iteritems():
            if len(vocab_ids) > 1:
                raise MultipleVocabulariesError("A controlled field should "
                        "only reference a single vocabulary. The field '%s' "
                        "references %d vocabularies." % (field,
                                                         len(vocab_ids)))
            else:
                (vocab_id,) = vocab_ids
                field_to_vocab_id[field] = vocab_id

        # Can remove this second pass though the file, but not important for
        # this first iteration.
        field_results = defaultdict(set)
        for row in self._table:
            for field in field_to_vocab_id:
                cell_value = row[field]
                vocab_id, value = self._extract_vocab_id(cell_value)

                if (vocab_id is None or vocab_id not in known_vocabs or
                    value.lower() not in known_vocabs[vocab_id]):
                    field_results[field].add(cell_value)

        return field_results

    def findCapitalizationDiscrepancies(self):
        """

        Currently checks all fields.
        """
        field_results = {}

        for field, values in self.fieldValues().iteritems():
            equal_values = defaultdict(set)

            for value in values:
                equal_values[value.lower()].add(value)

            discreps = [e for e in equal_values.values() if len(e) > 1]

            if discreps:
                field_results[field] = discreps

        return field_results

    def fieldValues(self):
        field_values = defaultdict(set)

        for row in self._table:
            for field in row:
                field_values[field].add(row[field])
<commit_msg>Add whitespace discrepancy methods<commit_after>#!/usr/bin/env python
from __future__ import division

from csv import DictReader
from collections import defaultdict
from glob import glob
from os.path import basename, join, splitext

class MetadataTable(object):
    VocabDelimiter = ':'

    @classmethod
    def fromFile(cls, fp, delimiter='\t'):
        with open(fp, 'U') as f:
            return cls(f, delimiter=delimiter)

    def __init__(self, metadata_f, delimiter='\t'):
        reader = DictReader(metadata_f, delimiter=delimiter)
        self.FieldNames = reader.fieldnames
        self._table = [row for row in reader]

    def candidateControlledFields(self, known_vocabs=None):
        cols = defaultdict(set)

        for row in self._table:
            for field in row:
                vocab_id, value = self._extract_vocab_id(row[field])

                if vocab_id is not None:
                    if known_vocabs is None or vocab_id in known_vocabs:
                        cols[field].add(vocab_id)

        return cols

    def validateControlledFields(self, known_vocabs):
        cont_fields = self.candidateControlledFields(known_vocabs=known_vocabs)

        field_to_vocab_id = {}
        for field, vocab_ids in cont_fields.iteritems():
            if len(vocab_ids) > 1:
                raise MultipleVocabulariesError("A controlled field should "
                        "only reference a single vocabulary. The field '%s' "
                        "references %d vocabularies." % (field,
                                                         len(vocab_ids)))
            else:
                (vocab_id,) = vocab_ids
                field_to_vocab_id[field] = vocab_id

        # Can remove this second pass though the file, but not important for
        # this first iteration.
        field_results = defaultdict(set)
        for row in self._table:
            for field in field_to_vocab_id:
                cell_value = row[field]
                vocab_id, value = self._extract_vocab_id(cell_value)

                if (vocab_id is None or vocab_id not in known_vocabs or
                    value.lower() not in known_vocabs[vocab_id]):
                    field_results[field].add(cell_value)

        return field_results

    def findCapitalizationDiscrepancies(self):
        """

        Currently checks all fields.
        """
        field_results = {}

        for field, values in self.fieldValues().iteritems():
            equal_values = defaultdict(set)

            for value in values:
                equal_values[value.lower()].add(value)

            discreps = [e for e in equal_values.values() if len(e) > 1]

            if discreps:
                field_results[field] = discreps

        return field_results

    def findHangingWhitespaceDiscrepancies(self):
        field_results = {}

        for field, values in self.fieldValues().iteritems():
            equal_values = defaultdict(set)

            for value in values:
                equal_values[value.strip()].add(value)

            discreps = [e for e in equal_values.values() if len(e) > 1]

            if discreps:
                field_results[field] = discreps

        return field_results

    def findWhitespaceDiscrepancies(self):
        field_results = {}

        for field, values in self.fieldValues().iteritems():
            equal_values = defaultdict(set)

            for value in values:
                equal_values[''.join(value.split())].add(value)

            discreps = [e for e in equal_values.values() if len(e) > 1]

            if discreps:
                field_results[field] = discreps

        return field_results

    def fieldValues(self):
        field_values = defaultdict(set)

        for row in self._table:
            for field in row:
                field_values[field].add(row[field])

{'fraction_non_alphanumeric': 0.07866490222026172, 'fraction_numerical': 0.000882223202470225, 'mean_word_length': 1.9382289416846652, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1628131', 'n_tokens_mistral': 1982, 'n_tokens_neox': 1879, 'n_words': 499}