<commit_before>import json
import os
import re

from jirafs import constants


class JiraFieldManager(dict):
    FIELD_MATCHER = re.compile(
        '^%s$' % (
            constants.TICKET_FILE_FIELD_TEMPLATE.replace(
                '{field_name}', '([\w_]+)'
            )
        )
    )

    def __init__(self):
        self._data = self.load()
        super(JiraFieldManager, self).__init__(self._data)

    def __sub__(self, other):
        differing = {}
        for k, v in other.items():
            if self.get(k) != v:
                differing[k] = (v, self.get(k), )

        return differing

    @classmethod
    def create(cls, folder, revision=None, path=None):
        if revision and path:
            raise TypeError(
                'You may specify a git revision or a local path; not both.'
            )

        if revision:
            return GitRevisionJiraFieldManager(folder, revision)
        else:
            return LocalFileJiraFieldManager(folder, path)

    def get_requested_per_ticket_fields(self):
        return constants.FILE_FIELDS

    def get_used_per_ticket_fields(self):
        raise NotImplementedError()

    def set_data_value(self, data, field_name, raw_value):
        raw_value = raw_value.strip()
        try:
            value = json.loads(raw_value)
        except (TypeError, ValueError):
            value = raw_value
        data[field_name] = value

    def get_fields_from_string(self, string):
        """ Gets field data from an incoming string.

        Parses through the string using the following RST-derived
        pattern::

            0 | * Field
            1 |     VALUE
            2 |     MORE VALUE

        """
        data = {}
        field_name = ''
        value = ''
        if not string:
            return data
        lines = string.split('\n')
        for idx, line in enumerate(lines):
            if line.startswith('*'):
                if value:
                    self.set_data_value(data, field_name, value)
                    value = ''
                raw_field_name = re.match('^\* ([^:]+):$', line).group(1)
                if '(' in raw_field_name:
                    # This field name's real name doesn't match the field ID
                    field_name = re.match(
                        '.*\(([^)]+)\)',
                        raw_field_name
                    ).group(1)
                else:
                    field_name = raw_field_name.replace(' ', '_')
            elif field_name:
                value = value + '\n' + line.strip()
        if value:
            self.set_data_value(data, field_name, value)

        return data

    def load(self):
        fields = self.get_fields_from_string(
            self.get_file_contents(constants.TICKET_DETAILS)
        )

        used_fields = set(self.get_used_per_ticket_fields())
        requested_fields = set(self.get_requested_per_ticket_fields())

        file_fields = {}
        for field_name in used_fields | requested_fields:
            try:
                field_path = constants.TICKET_FILE_FIELD_TEMPLATE.format(
                    field_name=field_name
                )
                file_fields[field_name] = self.get_file_contents(field_path)
            except (IOError, OSError):
                pass

        fields.update(file_fields)
        return fields


class LocalFileJiraFieldManager(JiraFieldManager):
    def __init__(self, folder, path):
        self.folder = folder
        self.path = path
        super(LocalFileJiraFieldManager, self).__init__()

    def get_file_contents(self, path):
        full_path = os.path.join(self.path, path)

        with open(self.folder.get_local_path(full_path), 'r') as _in:
            return _in.read().strip()

    def get_used_per_ticket_fields(self):<commit_msg>Also open the jira fields files using utf8.<commit_after>import io
import json
import os
import re

from jirafs import constants


class JiraFieldManager(dict):
    FIELD_MATCHER = re.compile(
        '^%s$' % (
            constants.TICKET_FILE_FIELD_TEMPLATE.replace(
                '{field_name}', '([\w_]+)'
            )
        )
    )

    def __init__(self):
        self._data = self.load()
        super(JiraFieldManager, self).__init__(self._data)

    def __sub__(self, other):
        differing = {}
        for k, v in other.items():
            if self.get(k) != v:
                differing[k] = (v, self.get(k), )

        return differing

    @classmethod
    def create(cls, folder, revision=None, path=None):
        if revision and path:
            raise TypeError(
                'You may specify a git revision or a local path; not both.'
            )

        if revision:
            return GitRevisionJiraFieldManager(folder, revision)
        else:
            return LocalFileJiraFieldManager(folder, path)

    def get_requested_per_ticket_fields(self):
        return constants.FILE_FIELDS

    def get_used_per_ticket_fields(self):
        raise NotImplementedError()

    def set_data_value(self, data, field_name, raw_value):
        raw_value = raw_value.strip()
        try:
            value = json.loads(raw_value)
        except (TypeError, ValueError):
            value = raw_value
        data[field_name] = value

    def get_fields_from_string(self, string):
        """ Gets field data from an incoming string.

        Parses through the string using the following RST-derived
        pattern::

            0 | * Field
            1 |     VALUE
            2 |     MORE VALUE

        """
        data = {}
        field_name = ''
        value = ''
        if not string:
            return data
        lines = string.split('\n')
        for idx, line in enumerate(lines):
            if line.startswith('*'):
                if value:
                    self.set_data_value(data, field_name, value)
                    value = ''
                raw_field_name = re.match('^\* ([^:]+):$', line).group(1)
                if '(' in raw_field_name:
                    # This field name's real name doesn't match the field ID
                    field_name = re.match(
                        '.*\(([^)]+)\)',
                        raw_field_name
                    ).group(1)
                else:
                    field_name = raw_field_name.replace(' ', '_')
            elif field_name:
                value = value + '\n' + line.strip()
        if value:
            self.set_data_value(data, field_name, value)

        return data

    def load(self):
        fields = self.get_fields_from_string(
            self.get_file_contents(constants.TICKET_DETAILS)
        )

        used_fields = set(self.get_used_per_ticket_fields())
        requested_fields = set(self.get_requested_per_ticket_fields())

        file_fields = {}
        for field_name in used_fields | requested_fields:
            try:
                field_path = constants.TICKET_FILE_FIELD_TEMPLATE.format(
                    field_name=field_name
                )
                file_fields[field_name] = self.get_file_contents(field_path)
            except (IOError, OSError):
                pass

        fields.update(file_fields)
        return fields


class LocalFileJiraFieldManager(JiraFieldManager):
    def __init__(self, folder, path):
        self.folder = folder
        self.path = path
        super(LocalFileJiraFieldManager, self).__init__()

    def get_file_contents(self, path):
        full_path = os.path.join(self.path, path)

        with io.open(
            self.folder.get_local_path(full_path),
            'r',
            encoding='utf-8'
        ) as _in:
            return _in.read().strip()

    def get_used_per_ticket_fields(self):
{'fraction_non_alphanumeric': 0.09505900661392816, 'fraction_numerical': 0.001556218389313967, 'mean_word_length': 1.727015558698727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1431346', 'n_tokens_mistral': 2271, 'n_tokens_neox': 2165, 'n_words': 488}