<commit_before>"""Standard test data.

For more information, see

 - http://www.wiley.com/legacy/wileychi/pesarin/material.html

"""

from __future__ import (absolute_import, division,
                        print_function, unicode_literals)

import os as _os

import numpy as np

from .. import data_dir


__all__ = ['load',
           'kenya', ]


def load(f):
    r"""Load a data file located in the data directory.

    Parameters
    ----------
    f : string
        File name.

    Returns
    -------
    x : array like
        Data loaded from permute.data_dir.
    """
    return np.recfromcsv(_os.path.join(data_dir, f), delimiter=",")


def nsgk():
    r"""NSGK test data for irr.

    Notes
    -----

    Here is first 5 lines of `nsgk.csv`::

        time_stamp,domain,video,rater
        1,8,1,1
        1,12,1,1
        1,15,1,1
        1,20,1,1

    """
    nz = np.loadtxt(_os.path.join(data_dir, "nsgk.csv"),
                    delimiter=',', skiprows=1, dtype=np.int32)
    shape = tuple(nz.max(axis=0))
    x = np.zeros(shape, dtype=np.int32)
    nz -= 1
    for r in nz:
        x[tuple(r)] = 1

    # given order: time_stamp,domain,video,rater
    # desired order: domain,video,rater,time_stamp
    x = x.transpose(1, 2, 3, 0)
    # hardcoding the number of timestamps per video<commit_msg>BUG: add optional argument to get rid of deprecation warning message<commit_after>"""Standard test data.

For more information, see

 - http://www.wiley.com/legacy/wileychi/pesarin/material.html

"""

from __future__ import (absolute_import, division,
                        print_function, unicode_literals)

import os as _os

import numpy as np

from .. import data_dir


__all__ = ['load',
           'kenya', ]


def load(f):
    r"""Load a data file located in the data directory.

    Parameters
    ----------
    f : string
        File name.

    Returns
    -------
    x : array like
        Data loaded from permute.data_dir.
    """
    return np.recfromcsv(_os.path.join(data_dir, f), delimiter=",", encoding=None)


def nsgk():
    r"""NSGK test data for irr.

    Notes
    -----

    Here is first 5 lines of `nsgk.csv`::

        time_stamp,domain,video,rater
        1,8,1,1
        1,12,1,1
        1,15,1,1
        1,20,1,1

    """
    nz = np.loadtxt(_os.path.join(data_dir, "nsgk.csv"),
                    delimiter=',', skiprows=1, dtype=np.int32)
    shape = tuple(nz.max(axis=0))
    x = np.zeros(shape, dtype=np.int32)
    nz -= 1
    for r in nz:
        x[tuple(r)] = 1

    # given order: time_stamp,domain,video,rater
    # desired order: domain,video,rater,time_stamp
    x = x.transpose(1, 2, 3, 0)
    # hardcoding the number of timestamps per video
{'fraction_non_alphanumeric': 0.13434492968171724, 'fraction_numerical': 0.023686158401184307, 'mean_word_length': 2.1725352112676055, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5596607', 'n_tokens_mistral': 988, 'n_tokens_neox': 930, 'n_words': 238}