<commit_before>"""File-backed data interface
"""
import csv
import glob
import os
import re
from functools import wraps

import fiona
import pyarrow as pa
from smif.data_layer.data_interface import (DataExistsError, DataInterface,
                                            DataMismatchError,
                                            DataNotFoundError)
from smif.data_layer.load import dump, load
from smif.metadata import Spec


# Note: these decorators must be defined before being used below
def check_exists(dtype):
    """Decorator to check an item of dtype exists
    """
    def wrapper(func):
        """Decorator specialised by dtype (class/item type)
        """
        @wraps(func)
        def wrapped(self, name, primary=None, secondary=None, *func_args, **func_kwargs):
            """Wrapper to implement error checking
            """
            _assert_no_mismatch(dtype, name, primary, secondary)
            if dtype in _file_dtypes():
                _assert_file_exists(self.file_dir, dtype, name)
            if dtype in _config_dtypes():
                config = self.read_project_config()
                _assert_config_item_exists(config, dtype, name)
            if dtype in _nested_config_dtypes():
                config = self.read_project_config()
                _assert_nested_config_item_exists(config, dtype, primary, secondary)
            return func(self, name, primary, secondary, *func_args, **func_kwargs)
        return wrapped
    return wrapper


def check_not_exists(dtype):
    """Decorator creator to check an item of dtype does not exist
    """
    def wrapper(func):
        """Decorator specialised by dtype (class/item type)
        """
        @wraps(func)
        def wrapped(self, primary, secondary=None, *func_args, **func_kwargs):
            """Wrapper to implement error checking
            """
            if dtype in _file_dtypes():
                _assert_file_not_exists(self.file_dir, dtype, primary['name'])
            if dtype in _config_dtypes():
                config = self.read_project_config()
                _assert_config_item_not_exists(config, dtype, primary['name'])
            if dtype in _nested_config_dtypes():
                config = self.read_project_config()
                _assert_nested_config_item_not_exists(config, dtype, primary, secondary)
            return func(self, primary, secondary, *func_args, **func_kwargs)
        return wrapped
    return wrapper


class DatafileInterface(DataInterface):
    """Read and write interface to YAML / CSV configuration files
    and intermediate CSV / native-binary data storage.

    Project.yml
<commit_msg>Fix arity of function calls from decorators<commit_after>"""File-backed data interface
"""
import csv
import glob
import os
import re
from functools import wraps

import fiona
import pyarrow as pa
from smif.data_layer.data_interface import (DataExistsError, DataInterface,
                                            DataMismatchError,
                                            DataNotFoundError)
from smif.data_layer.load import dump, load
from smif.metadata import Spec


# Note: these decorators must be defined before being used below
def check_exists(dtype):
    """Decorator to check an item of dtype exists
    """
    def wrapper(func):
        """Decorator specialised by dtype (class/item type)
        """
        @wraps(func)
        def wrapped(self, name, primary=None, secondary=None, *func_args, **func_kwargs):
            """Wrapper to implement error checking
            """
            _assert_no_mismatch(dtype, name, primary, secondary)
            if dtype in _file_dtypes():
                _assert_file_exists(self.file_dir, dtype, name)
            if dtype in _config_dtypes():
                config = self.read_project_config()
                _assert_config_item_exists(config, dtype, name)
            if dtype in _nested_config_dtypes():
                config = self.read_project_config()
                _assert_nested_config_item_exists(config, dtype, primary, secondary)

            if primary is None:
                return func(self, name, *func_args, **func_kwargs)
            elif secondary is None:
                return func(self, name, primary, *func_args, **func_kwargs)
            return func(self, name, primary, secondary, *func_args, **func_kwargs)

        return wrapped
    return wrapper


def check_not_exists(dtype):
    """Decorator creator to check an item of dtype does not exist
    """
    def wrapper(func):
        """Decorator specialised by dtype (class/item type)
        """
        @wraps(func)
        def wrapped(self, primary, secondary=None, *func_args, **func_kwargs):
            """Wrapper to implement error checking
            """
            if dtype in _file_dtypes():
                _assert_file_not_exists(self.file_dir, dtype, primary['name'])
            if dtype in _config_dtypes():
                config = self.read_project_config()
                _assert_config_item_not_exists(config, dtype, primary['name'])
            if dtype in _nested_config_dtypes():
                config = self.read_project_config()
                _assert_nested_config_item_not_exists(config, dtype, primary, secondary)

            if secondary is None:
                return func(self, primary, *func_args, **func_kwargs)
            return func(self, primary, secondary, *func_args, **func_kwargs)
        return wrapped
    return wrapper


class DatafileInterface(DataInterface):
    """Read and write interface to YAML / CSV configuration files
    and intermediate CSV / native-binary data storage.

    Project.yml

{'fraction_non_alphanumeric': 0.07994340290060134, 'fraction_numerical': 0.0, 'mean_word_length': 2.382177033492823, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1202610', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1444, 'n_words': 446}