<commit_before>#!/Users/takashi/.pyenv/shims/python

from scipy.io import arff as sparff
import numpy as np


def read_data(filename):
    """
    Extract the item specifying the features and the labels
    """
    d = sparff.loadarff(filename)
    data = datacleaning(d[0])
    return data


def datacleaning(data):
    n_data = np.zeros((len(data), len(data[0]) - 1))
    misslist = []
    labels = np.zeros(len(data))
    for i, v_x in enumerate(data):
        for j, vv_x in enumerate(v_x):
            if vv_x in {bytes(b'?'), '?'}:
                # REMOVE this entry
                misslist.append(i)
                break
            # 以下列挙
            elif j in {2, 3, 4}:
                n_data[i][j] = float(vv_x)
                continue
            elif j in {5, 6}:
                n_data[i][j] = (1) if (vv_x == bytes(b'normal')) else (0)
                continue
            elif j in {7, 8}:
                n_data[i][j] = (1) if (vv_x == bytes(b'present')) else (0)
                continue
            elif j in {18, 19, 20, 22, 23}:
                n_data[i][j] = (1) if (vv_x == bytes(b'yes')) else (0)
                continue
            elif j in {21}:
                n_data[i][j] = (1) if (vv_x == bytes(b'good')) else (0)
                continue
            elif j in {24}:
                labels[i] = (1) if (vv_x == bytes(b'ckd')) else (0)
                continue
            else:
                n_data[i][j] = vv_x
                continue

    slc_idx = np.array([(True) if v not in misslist else (False)
                        for v in range(len(data))])
    return (n_data[slc_idx], labels[slc_idx])


def test_readdata():
    fn = "./chronic_kidney_disease/Chronic_Kidney_Disease_full.arff"<commit_msg>added removing nan(missing) value data<commit_after>#!/Users/takashi/.pyenv/shims/python


from scipy.io import arff as sparff
import numpy as np


def read_data(filename):
    """
    Extract the item specifying the features and the labels
    """
    d = sparff.loadarff(filename)
    data = datacleaning(d[0])
    return data


def datacleaning(data):
    n_data = np.zeros((len(data), len(data[0]) - 1))
    misslist = []
    labels = np.zeros(len(data))
    for i, v_x in enumerate(data):
        for j, vv_x in enumerate(v_x):
            if vv_x in {bytes(b'?'), '?'}:
                # REMOVE this entry
                misslist.append(i)
                break
            elif j in {2, 3, 4}:
                n_data[i][j] = float(vv_x)
                continue
            elif j in {5, 6}:
                n_data[i][j] = (1) if (vv_x == bytes(b'normal')) else (0)
                continue
            elif j in {7, 8}:
                n_data[i][j] = (1) if (vv_x == bytes(b'present')) else (0)
                continue
            elif j in {18, 19, 20, 22, 23}:
                n_data[i][j] = (1) if (vv_x == bytes(b'yes')) else (0)
                continue
            elif j in {21}:
                n_data[i][j] = (1) if (vv_x == bytes(b'good')) else (0)
                continue
            elif j in {24}:
                labels[i] = (1) if (vv_x == bytes(b'ckd')) else (0)
                continue
            else:
                if np.isnan(vv_x):
                    misslist.append(i)
                    break
                else:
                    n_data[i][j] = vv_x
                    continue

    slc_idx = np.array([(True) if v not in misslist else (False)
                        for v in range(len(data))])
    return (n_data[slc_idx], labels[slc_idx])


def test_readdata():
    fn = "./chronic_kidney_disease/Chronic_Kidney_Disease_full.arff"
{'fraction_non_alphanumeric': 0.13173156586578294, 'fraction_numerical': 0.018779342723004695, 'mean_word_length': 1.5525017618040873, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1112714', 'n_tokens_mistral': 1299, 'n_tokens_neox': 1202, 'n_words': 319}