<commit_before># -*- coding: utf-8 -*-
__author__ = 'Ralph'

import arff
import numpy as np
import pandas as pd


class ARFF(object):

    @staticmethod
    def read(file_name, missing=None):
        """
        Loads ARFF file into data dictionary. Missing values indicated
        by '?' are automatically converted to None. If you want some
        other value to be treated as missing, specify them in the
        missing parameter.
        :param file_name: File name
        :param missing: List of missing value representations
        :return: Data dictionary
        """
        data = arff.load(open(file_name))
        if not missing == None:
            for i in range(len(data['data'])):
                for j in range(len(data['data'][i])):
                    if type(missing) is str:
                        if data['data'][i][j] == missing:
                            data['data'][i][j] = None
                    elif type(missing) is list:
                        for m in missing:
                            if data['data'][i][j] == m:
                                data['data'][i][j] = None                                
                    else:
                        raise RuntimeError('Invalid type for \'missing\' parameter ' + str(type(missing)))
        return data

    @staticmethod
    def to_data_frame(data, index_col=None):
        """
        Converts ARFF data dictionary to Pandas data frame.
        :param data: Data dictionary
        :param index_col: Column name to use as index
        :return: Data frame
        """
        # Create data frame by by taking rows and attributes from 
        # ARFF data. Data types should be automatically inferred
        rows = data['data']
        columns = [attribute[0] for attribute in data['attributes']]
        data_frame = pd.DataFrame(rows, columns=columns)

        # If index column specified, set it
        if not index_col is None:
            if not index_col in data_frame.columns:
                raise RuntimeError('Index column ' + index_col + ' not found')
            data_frame.set_index(index_col, drop=True, inplace=True, verify_integrity=True)

        return data_frame

    @staticmethod
    def from_data_frame(relation, attributes, data_frame, description=''):
        """
        Converts Pandas data frame to ARFF dictionary. This is only possible
        if the data frame was previously converted from ARFF data because we
        need the specific attribute information.
        :param relation: Relation<commit_msg>Added function to read from CSV<commit_after># -*- coding: utf-8 -*-
__author__ = 'Ralph'

import arff
import numpy as np
import pandas as pd


class ARFF(object):

    @staticmethod
    def read(file_name, missing=None):
        """
        Loads ARFF file into data dictionary. Missing values indicated
        by '?' are automatically converted to None. If you want some
        other value to be treated as missing, specify them in the
        missing parameter.
        :param file_name: File name
        :param missing: List of missing value representations
        :return: Data dictionary
        """
        data = arff.load(open(file_name))
        if not missing == None:
            for i in range(len(data['data'])):
                for j in range(len(data['data'][i])):
                    if type(missing) is str:
                        if data['data'][i][j] == missing:
                            data['data'][i][j] = None
                    elif type(missing) is list:
                        for m in missing:
                            if data['data'][i][j] == m:
                                data['data'][i][j] = None                                
                    else:
                        raise RuntimeError('Invalid type for \'missing\' parameter ' + str(type(missing)))
        return data

    @staticmethod
    def read_from_csv(file_name):
        """
        Loads CSV file and converts it to an ARFF data dictionary. This 
        function assumes the following:
        (1) First line contains a header with column names
        (2) First column contains IDs (interpreted as string values)
        (3) Remaining columns contain numeric values
        :param file_name: CSV file path
        """
        attributes = []
        f = open(file_name, 'r')
        header = f.readline().strip().split(',')
        header = [item.strip() for item in header]
        attributes.append((header[0], 'STRING'))
        for item in header[1:]:
            attributes.append((item, 'NUMERIC'))
        data = []
        for line in f.readlines():
            line = line.strip()
            if line.startswith('#') or line == '':
                continue
            parts = line.split(',')
            parts = [part.strip() for part in parts]
            parts[0]  = str(parts[0])
            parts[1:] = [float(part) for part in parts[1:]]
            data.append(parts)
        f.close()

        return {
            'relation': 'unknown',
            'attributes': attributes,
            'data': data,
            'description': ''
        }
        

    @staticmethod
    def to_data_frame(data, index_col=None):
        """
        Converts ARFF data dictionary to Pandas data frame.
        :param data: Data dictionary
        :param index_col: Column name to use as index
        :return: Data frame
        """
        # Create data frame by by taking rows and attributes from 
        # ARFF data. Data types should be automatically inferred
        rows = data['data']
        columns = [attribute[0] for attribute in data['attributes']]
        data_frame = pd.DataFrame(rows, columns=columns)

        # If index column specified, set it
        if not index_col is None:
            if not index_col in data_frame.columns:
                raise RuntimeError('Index column ' + index_col + ' not found')
            data_frame.set_index(index_col, drop=True, inplace=True, verify_integrity=True)

        return data_frame

    @staticmethod
    def from_data_frame(relation, attributes, data_frame, description=''):
        """
        Converts Pandas data frame to ARFF dictionary. This is only possible
        if the data frame was previously converted from ARFF data because we
        need the specific attribute information.
        :param relation: Relation
{'fraction_non_alphanumeric': 0.09129545096804659, 'fraction_numerical': 0.002046277349283803, 'mean_word_length': 1.8416815742397137, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5252037', 'n_tokens_mistral': 1623, 'n_tokens_neox': 1582, 'n_words': 578}