<commit_before>"""Tiny library for handling csv"""


import os
import csv
import itertools


# HELPER FUNCTIONS
def is_type(_type, x):
    try:
        return str(_type(x)) == x
    except ValueError:
        return False


def determine_type(x):
    """Determine the type of x"""
    types = (int, float, str)
    _type = filter(lambda a: is_type(a, x), types)[0]
    return _type(x)


def dmap(fn, record):
    """map for a directory"""
    values = (fn(v) for k, v in record.items())
    return dict(itertools.izip(record, values))


# FUNCTIONS
def read_csv(filename, delimiter=",", skip=0, guess_type=True):
    """Read a CSV file
    
    Usage
    -----
    >>> data = read_csv(filename, delimiter=delimiter, skip=skip, guess_type=guess_type) 

    """
    f = open(filename, 'r')

    # Skip the n first lines
    for i in range(skip):
        f.readline()

    for line in csv.DictReader(f, delimiter=delimiter):
        if guess_type:
            yield dmap(determine_type, line)
        else:
            yield line

    f.close()


def write_csv(filename, fieldnames, data=None, rows=None, mode="w"):
    """Write the data to the specified filename
    
    Usage
    -----
    >>> write_csv(filename, fieldnames, data, mode=mode)

    Parameters
    ----------
    filename : str
        The name of the file

    fieldnames : list of strings
        The names of the columns (or fields):
        (fieldname1, fieldname2, ...)

    data : list of dictionaries (optional)
        [
         {fieldname1: a1, fieldname2: a2},
         {fieldname1: b1, fieldname2: b2},
         ...
        ]

    rows : list of lists (optional)
        [
        (a1, a2),
        (b1, b2),
        ...
        ]

    mode : str (optional)
        "w": write the data to the file by overwriting it
        "a": write the data to the file by appending them

    Returns
    -------
    None. A CSV file is written.

    """
    if data == rows == None:
        msg = "You must specify either data or rows"
        raise ValueError(msg)

    elif data != None and rows != None:
        msg = "You must specify either data or rows. Not both"
        raise ValueError(msg)

    header = dict((x, x) for x in fieldnames)
    f = open(filename, mode)
    if data:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        if mode == "w":
            writer.writerow(header)
        writer.writerows(data)
    elif rows:
        writer = csv.writer(f)
        if mode == "w":
            writer.writerow(fieldnames)
        writer.writerows(rows)

    f.close()
    print "Saved %s." % filename


def format_to_csv(filename, skiprows=0, delimiter=""):
    """Convert a file to a .csv file"""
    if not delimiter:
        delimiter = "\t"

    input_file = open(filename, "r")

    if skiprows:
        [input_file.readline() for _ in range(skiprows)]
 
    new_filename = os.path.splitext(filename)[0] + ".csv"
    output_file = open(new_filename, "w")

    header = input_file.readline().split()
    reader = csv.DictReader(input_file, fieldnames=header, delimiter=delimiter)
    writer = csv.DictWriter(output_file, fieldnames=header, delimiter=",")
    
    # Write header
    writer.writerow(dict((x, x) for x in header))
    
    # Write rows
    for line in reader:
        if None in line: del line[None]
        writer.writerow(line)
    
    input_file.close()
    output_file.close()
    print "Saved %s." % new_filename
<commit_msg>Fix bug: Header is not read when lines are skipped<commit_after>"""Tiny library for handling csv"""


import os
import csv
import itertools


# HELPER FUNCTIONS
def is_type(_type, x):
    try:
        return str(_type(x)) == x
    except ValueError:
        return False


def determine_type(x):
    """Determine the type of x"""
    types = (int, float, str)
    _type = filter(lambda a: is_type(a, x), types)[0]
    return _type(x)


def dmap(fn, record):
    """map for a directory"""
    values = (fn(v) for k, v in record.items())
    return dict(itertools.izip(record, values))


# FUNCTIONS
def read_csv(filename, delimiter=",", skip=0, guess_type=True, has_header=True):
    """Read a CSV file
    
    Usage
    -----
    >>> data = read_csv(filename, delimiter=delimiter, skip=skip,
            guess_type=guess_type, has_header=True) 

    keywords
    :has_header:
        Determine whether the file has a header or not

    """
    f = open(filename, 'r')

    # Skip the n first lines
    if has_header:
        header = f.readline().split(delimiter)
    else:
        header = None

    for i in range(skip):
        f.readline()

    for line in csv.DictReader(f, delimiter=delimiter, fieldnames=header):
        if guess_type:
            yield dmap(determine_type, line)
        else:
            yield line

    f.close()


def write_csv(filename, header, data=None, rows=None, mode="w"):
    """Write the data to the specified filename
    
    Usage
    -----
    >>> write_csv(filename, header, data, mode=mode)

    Parameters
    ----------
    filename : str
        The name of the file

    header : list of strings
        The names of the columns (or fields):
        (fieldname1, fieldname2, ...)

    data : list of dictionaries (optional)
        [
         {fieldname1: a1, fieldname2: a2},
         {fieldname1: b1, fieldname2: b2},
         ...
        ]

    rows : list of lists (optional)
        [
        (a1, a2),
        (b1, b2),
        ...
        ]

    mode : str (optional)
        "w": write the data to the file by overwriting it
        "a": write the data to the file by appending them

    Returns
    -------
    None. A CSV file is written.

    """
    if data == rows == None:
        msg = "You must specify either data or rows"
        raise ValueError(msg)

    elif data != None and rows != None:
        msg = "You must specify either data or rows. Not both"
        raise ValueError(msg)

    data_header = dict((x, x) for x in header)
    f = open(filename, mode)
    if data:
        writer = csv.DictWriter(f, fieldnames=header)
        if mode == "w":
            writer.writerow(data_header)
        writer.writerows(data)
    elif rows:
        writer = csv.writer(f)
        if mode == "w":
            writer.writerow(header)
        writer.writerows(rows)

    f.close()
    print "Saved %s." % filename


def format_to_csv(filename, skiprows=0, delimiter=""):
    """Convert a file to a .csv file"""
    if not delimiter:
        delimiter = "\t"

    input_file = open(filename, "r")

    if skiprows:
        [input_file.readline() for _ in range(skiprows)]
 
    new_filename = os.path.splitext(filename)[0] + ".csv"
    output_file = open(new_filename, "w")

    header = input_file.readline().split()
    reader = csv.DictReader(input_file, fieldnames=header, delimiter=delimiter)
    writer = csv.DictWriter(output_file, fieldnames=header, delimiter=",")
    
    # Write header
    writer.writerow(dict((x, x) for x in header))
    
    # Write rows
    for line in reader:
        if None in line: del line[None]
        writer.writerow(line)
    
    input_file.close()
    output_file.close()
    print "Saved %s." % new_filename

{'fraction_non_alphanumeric': 0.1209282818397875, 'fraction_numerical': 0.005032853348245492, 'mean_word_length': 2.2297968397291195, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2512677', 'n_tokens_mistral': 2305, 'n_tokens_neox': 2225, 'n_words': 686}