<commit_before>from typing import Dict, List

import numpy as np
import logging
import pandas as pd
from sqlalchemy import create_engine
import uuid
import shutil
from bayesianpy.jni import bayesServer, bayesServerAnalysis, bayesServerDiscovery, jp
import os
from bayesianpy.decorators import listify
import dask.dataframe as dd
from typing import Iterable
import bayesianpy.dask as dk
from collections import defaultdict

class DataFrameReader:
    def __init__(self, df):
        self._df = df
        self._row = None
        self._columns = df.columns.tolist()
        self.reset()
        self.row_index = 0
        self._writer = DataFrameWriter(df)

    def __getattr__(self, key):
        return self.__getitem__(key)

    def read(self) -> bool:
        self._row = next(self._iterator, None)
        self.row_index += 1
        return self._row is not None

    def reset(self) -> None:
        self._iterator = self._df.itertuples()

    def get_index(self):
        return self._row[0]

    def to_dict(self):
        return self.row()

    def tolist(self, cols) -> List[object]:
        return [self.__getitem__(c) for c in cols]

    def row(self) -> Dict[str, object]:
        return {c : self.__getitem__(c) for c in self._columns}

    def writer(self) -> 'DataFrameWriter':
        if self._row is not None:
            return self._writer.with_index(self.get_index())
        else:
            return self._writer

    def set_value(self, column, value):
        self.writer().set_value(column, value)

    def __getitem__(self, key) -> object:
        # the df index of the row is at index 0
        try:
            ix = self._columns.index(key) + 1
            return self._row[ix]
        except BaseException as e:
            return None

    def __next__(self) -> 'DataFrameReader':
        if self.read():
            return self
        else:
            raise StopIteration

    def __iter__(self) -> 'DataFrameReader':
        return self

class DataFrameWriter:

    def __init__(self, df):
        self._columns = defaultdict(list)
        self._row_indices = set()
        self._df = df

    def with_index(self, index):
        if index not in self._row_indices:
            self._row_indices.add(index)

        self._current_row_index = index
        return self

    def set_value(self, column, value):
        self._columns[column].append(value)

    def as_dataframe(self):
        self._columns['ix'] = list(self._row_indices)
        df = pd.DataFrame(self._columns)
        return df.set_index('ix')

    def get_dataframe(self, df: pd.DataFrame = None):
        df1 = self.as_dataframe()
        if df is None:
            return self._df.join(df1)
        else:
            return df.join(df1)







class AutoType:
    def __init__(self, df, discrete=[], continuous=[], continuous_to_discrete_limit = 20):
        self._df = df
        self._continuous_to_discrete_limit = continuous_to_discrete_limit
        self._continuous = continuous
        self._discrete = discrete

    @listify
    def get_continuous_variables(self):
        cols = self._df.columns.tolist()
        for col in cols:
            if not DataFrame.is_float(self._df[col].dtype) and not DataFrame.is_int(self._df[col].dtype):
                continue
            if col in self._discrete:
                continue
            elif col in self._continuous:
                yield col
            elif len(self._df[col].unique()) > self._continuous_to_discrete_limit:
                yield col

    @listify
    def get_discrete_variables(self):
        continuous = set(self.get_continuous_variables())
        for col in self._df.columns.tolist():
            if col in self._continuous:
                continue

            if DataFrame.is_timestamp(self._df[col].dtype):
                continue

            elif col in self._discrete:
                yield col
            elif col not in continuous:
                yield col


class DataFrame:

    def __init__(self, df):
        self._df = df

    @staticmethod
    def is_timestamp(dtype):
        for ts in ['timestamp64', 'timedelta64', 'datetime64']:
            if ts in str(dtype):
                return True

        return False

    @staticmethod
    def is_numeric(dtype):
        return DataFrame.is_float(dtype) or DataFrame.is_int(dtype)

    @staticmethod
    def is_float(dtype):
        return str(dtype) in {"float32", "float64"}

    @staticmethod
    def is_int(dtype):
        return str(dtype) in {"int32", "int64"}

    @staticmethod
    def is_bool(dtype):
        return str(dtype) == "bool"

    @staticmethod
    def is_string(dtype):
        return str(dtype) == "object" or str(dtype) == "O"

    @staticmethod
    def could_be_int(col):
        if DataFrame.is_int(col.dtype):
            return True

        if DataFrame.is_float(col.dtype):
            for val in col.dropna().unique():
                if int(val) != val:
                    return False

            return True

        return False

    @staticmethod
    def coerce_to_numeric(df: pd.DataFrame, logger: logging.Logger, cutoff=0.10, ignore=[]) -> pd.DataFrame:
        for col in df.columns:
            if DataFrame.is_timestamp(df[col].dtype):
                continue

            if col in ignore:
                continue

            values = df[col].dropna().unique()
            ratio = 0
            pre_length = len(values)

            if pre_length > 0:
                new_values = pd.to_numeric(df[col].dropna().unique(), errors='coerce')
                post_length = len(new_values[~np.isnan(new_values)])
                ratio = (pre_length - post_length) / pre_length

            if ratio <= cutoff:
                logger.debug("Converting column {} to numeric (ratio: {})".format(col, ratio))
                df[col] = pd.to_numeric(df[col], errors='coerce')
            else:
                logger.debug("Not converting column {} (ratio: {})".format(col, ratio))

        return df

    @staticmethod<commit_msg>Don't touch columns that are already numeric<commit_after>from typing import Dict, List

import numpy as np
import logging
import pandas as pd
from sqlalchemy import create_engine
import uuid
import shutil
from bayesianpy.jni import bayesServer, bayesServerAnalysis, bayesServerDiscovery, jp
import os
from bayesianpy.decorators import listify
import dask.dataframe as dd
from typing import Iterable
import bayesianpy.dask as dk
from collections import defaultdict

class DataFrameReader:
    def __init__(self, df):
        self._df = df
        self._row = None
        self._columns = df.columns.tolist()
        self.reset()
        self.row_index = 0
        self._writer = DataFrameWriter(df)

    def __getattr__(self, key):
        return self.__getitem__(key)

    def read(self) -> bool:
        self._row = next(self._iterator, None)
        self.row_index += 1
        return self._row is not None

    def reset(self) -> None:
        self._iterator = self._df.itertuples()

    def get_index(self):
        return self._row[0]

    def to_dict(self):
        return self.row()

    def tolist(self, cols) -> List[object]:
        return [self.__getitem__(c) for c in cols]

    def row(self) -> Dict[str, object]:
        return {c : self.__getitem__(c) for c in self._columns}

    def writer(self) -> 'DataFrameWriter':
        if self._row is not None:
            return self._writer.with_index(self.get_index())
        else:
            return self._writer

    def set_value(self, column, value):
        self.writer().set_value(column, value)

    def __getitem__(self, key) -> object:
        # the df index of the row is at index 0
        try:
            ix = self._columns.index(key) + 1
            return self._row[ix]
        except BaseException as e:
            return None

    def __next__(self) -> 'DataFrameReader':
        if self.read():
            return self
        else:
            raise StopIteration

    def __iter__(self) -> 'DataFrameReader':
        return self

class DataFrameWriter:

    def __init__(self, df):
        self._columns = defaultdict(list)
        self._row_indices = set()
        self._df = df

    def with_index(self, index):
        if index not in self._row_indices:
            self._row_indices.add(index)

        self._current_row_index = index
        return self

    def set_value(self, column, value):
        self._columns[column].append(value)

    def as_dataframe(self):
        self._columns['ix'] = list(self._row_indices)
        df = pd.DataFrame(self._columns)
        return df.set_index('ix')

    def get_dataframe(self, df: pd.DataFrame = None):
        df1 = self.as_dataframe()
        if df is None:
            return self._df.join(df1)
        else:
            return df.join(df1)







class AutoType:
    def __init__(self, df, discrete=[], continuous=[], continuous_to_discrete_limit = 20):
        self._df = df
        self._continuous_to_discrete_limit = continuous_to_discrete_limit
        self._continuous = continuous
        self._discrete = discrete

    @listify
    def get_continuous_variables(self):
        cols = self._df.columns.tolist()
        for col in cols:
            if not DataFrame.is_float(self._df[col].dtype) and not DataFrame.is_int(self._df[col].dtype):
                continue
            if col in self._discrete:
                continue
            elif col in self._continuous:
                yield col
            elif len(self._df[col].unique()) > self._continuous_to_discrete_limit:
                yield col

    @listify
    def get_discrete_variables(self):
        continuous = set(self.get_continuous_variables())
        for col in self._df.columns.tolist():
            if col in self._continuous:
                continue

            if DataFrame.is_timestamp(self._df[col].dtype):
                continue

            elif col in self._discrete:
                yield col
            elif col not in continuous:
                yield col


class DataFrame:

    def __init__(self, df):
        self._df = df

    @staticmethod
    def is_timestamp(dtype):
        for ts in ['timestamp64', 'timedelta64', 'datetime64']:
            if ts in str(dtype):
                return True

        return False

    @staticmethod
    def is_numeric(dtype):
        return DataFrame.is_float(dtype) or DataFrame.is_int(dtype)

    @staticmethod
    def is_float(dtype):
        return str(dtype) in {"float32", "float64"}

    @staticmethod
    def is_int(dtype):
        return str(dtype) in {"int32", "int64"}

    @staticmethod
    def is_bool(dtype):
        return str(dtype) == "bool"

    @staticmethod
    def is_string(dtype):
        return str(dtype) == "object" or str(dtype) == "O"

    @staticmethod
    def could_be_int(col):
        if DataFrame.is_int(col.dtype):
            return True

        if DataFrame.is_float(col.dtype):
            for val in col.dropna().unique():
                if int(val) != val:
                    return False

            return True

        return False

    @staticmethod
    def coerce_to_numeric(df: pd.DataFrame, logger: logging.Logger, cutoff=0.10, ignore=[]) -> pd.DataFrame:
        for col in df.columns:
            if DataFrame.is_numeric(df[col].dtype):
                continue

            if DataFrame.is_timestamp(df[col].dtype):
                continue

            if col in ignore:
                continue

            values = df[col].dropna().unique()
            ratio = 0
            pre_length = len(values)

            if pre_length > 0:
                new_values = pd.to_numeric(df[col].dropna().unique(), errors='coerce')
                post_length = len(new_values[~np.isnan(new_values)])
                ratio = (pre_length - post_length) / pre_length

            if ratio <= cutoff:
                logger.debug("Converting column {} to numeric (ratio: {})".format(col, ratio))
                df[col] = pd.to_numeric(df[col], errors='coerce')
            else:
                logger.debug("Not converting column {} (ratio: {})".format(col, ratio))

        return df

    @staticmethod
{'fraction_non_alphanumeric': 0.09684591945977106, 'fraction_numerical': 0.004776414395124763, 'mean_word_length': 2.0337247064701476, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7105005', 'n_tokens_mistral': 3639, 'n_tokens_neox': 3475, 'n_words': 924}