<commit_before>"""Module to handle data munging/wrangling
"""

# Author          : <NAME> jjinking(at)gmail(dot)com
# Creation date   : 2014.02.13
# Last Modified   : 2016.04.15
#
# License         : MIT

import csv
import numpy as np
import pandas as pd
import sys
from datetime import datetime

from . import dataio


def remove_null_big_data(fname_in, fname_out, delim=','):
    """Remove all rows containing any null values
    """
    with dataio.fopen(fname_in, 'r') as fi, dataio.fopen(fname_out, 'w') as fo:
        reader = csv.reader(fi, delimiter=delim)
        writer = csv.writer(fo, delimiter=delim)
        writer.writerow(next(reader))
        for row in reader:
            if '' in row:
                continue
            writer.writerow(row)


def remove_col_big_data(
        fname_in, fname_out, indices, delimiter=',', verbose=None):
    """Remove a column given the given column index idx (0-based)
    if progress_int can be set to an integer to set an interval to output
    number of rows seen every given interval
    """
    indices = set(indices)
    with dataio.fopen(fname_in, 'r') as fi, dataio.fopen(fname_out, 'w') as fo:
        reader = csv.reader(fi, delimiter=delimiter)
        writer = csv.writer(fo, delimiter=delimiter)
        for t, row in enumerate(reader):
            # Output progress
            if verbose is not None and t % verbose == 0:
                sys.stdout.write(
                    '{}\tencountered: {}\n'.format(datetime.now(), t))
            writer.writerow([x for i, x in enumerate(row) if i not in indices])


def one_hot_encode_series_of_lists(feature_col):
    """
    Take in a feature_col containing list values, and return a one-hot-encoded
    DataFrame
    """
    seen_vals = dict()
    idx = 0
    prefix = feature_col.name
    for vals in feature_col:
        for val in vals:
            val_name = prefix + '_' + val
            if val_name not in seen_vals:
                seen_vals[val_name] = ([0] * idx) + [1]
            else:
                for i in range(idx - len(seen_vals[val_name])):
                    seen_vals[val_name].append(0)
                seen_vals[val_name].append(1)
        idx += 1
    for k,v in seen_vals.items():
        for _ in range(idx - len(v)):
            v.append(0)
    return pd.DataFrame(seen_vals)
<commit_msg>Update munge.py<commit_after>"""Module to handle data munging/wrangling
"""

# Author          : <NAME> jjinking(at)gmail(dot)com
# Creation date   : 2014.02.13
# Last Modified   : 2016.04.15
#
# License         : MIT

import csv
import numpy as np
import pandas as pd
import sys
from datetime import datetime

from . import dataio


def remove_null_big_data(fname_in, fname_out, delim=','):
    """Remove all rows containing any null values
    """
    with dataio.fopen(fname_in, 'r') as fi, dataio.fopen(fname_out, 'w') as fo:
        reader = csv.reader(fi, delimiter=delim)
        writer = csv.writer(fo, delimiter=delim)
        writer.writerow(next(reader))
        for row in reader:
            if '' in row:
                continue
            writer.writerow(row)


def remove_col_big_data(
        fname_in, fname_out, indices, delimiter=',', verbose=None):
    """Remove a column given the given column index idx (0-based)
    if progress_int can be set to an integer to set an interval to output
    number of rows seen every given interval
    """
    indices = set(indices)
    with dataio.fopen(fname_in, 'r') as fi, dataio.fopen(fname_out, 'w') as fo:
        reader = csv.reader(fi, delimiter=delimiter)
        writer = csv.writer(fo, delimiter=delimiter)
        for t, row in enumerate(reader):
            # Output progress
            if verbose is not None and t % verbose == 0:
                sys.stdout.write(
                    '{}\tencountered: {}\n'.format(datetime.now(), t))
            writer.writerow([x for i, x in enumerate(row) if i not in indices])

def one_hot_encode_series_of_lists(feature_col, list_values=None):
    """
    Take in a pd.Series `feature_col` containing `list_values`, and return a
    pd.DataFrame containing one-hot encoded columns.
    
    If `list_values` is None, then a column for every unique value in
    the series will be created.
    
    Otherwise, a column will be created for each of the values in `list_values`.
    Also, an additional default column will be created to hold values that
    did not belong to any of the `list_values`. Note, if a row in `feature_col`
    contains multiple values that are not in `list_values`, there will only be a
    single 1 in the default column.
    
    """
    if list_values is None:
        seen_vals = dict()
    else:
        # Initialize default column
        default_colname = feature_col.name + '_default'
        seen_vals = {default_colname: []}
        for val in list_values:
            seen_vals[feature_col.name + '_' + val] = []
    idx = 0
    prefix = feature_col.name
    for vals in feature_col:
        for val in vals:
            val_name = prefix + '_' + val
            if val_name not in seen_vals and list_values is None:
                seen_vals[val_name] = ([0] * idx) + [1]
            else:
                if val_name not in seen_vals:
                    val_name = default_colname
                for i in range(idx - len(seen_vals[val_name])):
                    seen_vals[val_name].append(0)
                if len(seen_vals[val_name]) - 1 < idx:
                    seen_vals[val_name].append(1)
        idx += 1
    # Fill out remaining zeros
    for k,v in seen_vals.items():
        for _ in range(idx - len(v)):
            v.append(0)
    return pd.DataFrame(seen_vals)
{'fraction_non_alphanumeric': 0.09761694616063549, 'fraction_numerical': 0.009179170344218888, 'mean_word_length': 2.2865429234338746, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4576274', 'n_tokens_mistral': 1803, 'n_tokens_neox': 1709, 'n_words': 554}