<commit_before>from __future__ import unicode_literals
import os
import csv
from io import StringIO
from django.utils import six
from csvkit import CSVKitReader, CSVKitWriter
from calaccess_raw import get_download_directory
from calaccess_raw.management.commands import CalAccessCommand


class Command(CalAccessCommand):
    help = 'Clean a source CAL-ACCESS file and reformat it as a CSV'

    def add_arguments(self, parser):

        super(Command, self).add_arguments(parser)

        parser.add_argument('file_name')

    def handle(self, **options):
        # Set options
        self.verbosity = int(options.get("verbosity"))
        self.data_dir = get_download_directory()
        self.tsv_dir = os.path.join(self.data_dir, "tsv/")
        self.csv_dir = os.path.join(self.data_dir, "csv/")
        self.log_dir = os.path.join(self.data_dir, "log/")
        # Do it
        self.clean(options['file_name'])

    def clean(self, name):
        """
        Cleans the provided source TSV file and writes it out in CSV format
        """
        if self.verbosity > 2:
            self.log(" Cleaning %s" % name)

        # Up the CSV data limit
        csv.field_size_limit(1000000000)

        # Input and output paths
        tsv_path = os.path.join(self.tsv_dir, name)
        csv_path = os.path.join(
            self.csv_dir,
            name.lower().replace("tsv", "csv")
        )

        # Writer
        csv_file = open(csv_path, 'w')
        csv_writer = CSVKitWriter(csv_file, quoting=csv.QUOTE_ALL)

        # Reader
        tsv_file = open(tsv_path, 'rb')

        # Pull and clean the headers
        try:
            headers = tsv_file.readline()
        except StopIteration:
            return
        headers = headers.decode("ascii", "replace")
        headers_csv = CSVKitReader(StringIO(headers), delimiter=str('\t'))
        try:
            headers_list = next(headers_csv)
        except StopIteration:
            return
        headers_count = len(headers_list)
        csv_writer.writerow(headers_list)

        log_rows = []

        # Loop through the rest of the data
        line_number = 1
        for tsv_line in tsv_file:
<commit_msg>Fix #1179<commit_after>from __future__ import unicode_literals
import os
import csv
from io import StringIO
from django.utils import six
from csvkit import CSVKitReader, CSVKitWriter
from calaccess_raw import get_download_directory
from calaccess_raw.management.commands import CalAccessCommand


class Command(CalAccessCommand):
    help = 'Clean a source CAL-ACCESS file and reformat it as a CSV'

    def add_arguments(self, parser):

        super(Command, self).add_arguments(parser)

        parser.add_argument('file_name')

    def handle(self, **options):
        # Set options
        self.verbosity = int(options.get("verbosity"))
        self.data_dir = get_download_directory()
        self.tsv_dir = os.path.join(self.data_dir, "tsv/")
        self.csv_dir = os.path.join(self.data_dir, "csv/")
        self.log_dir = os.path.join(self.data_dir, "log/")
        # Do it
        self.clean(options['file_name'])

    def clean(self, name):
        """
        Cleans the provided source TSV file and writes it out in CSV format
        """
        if self.verbosity > 2:
            self.log(" Cleaning %s" % name)

        # Up the CSV data limit
        csv.field_size_limit(1000000000)

        # Input and output paths
        tsv_path = os.path.join(self.tsv_dir, name)
        csv_path = os.path.join(
            self.csv_dir,
            name.lower().replace("tsv", "csv")
        )

        # Reader
        tsv_file = open(tsv_path, 'rb')

        # Writer
        csv_file = open(csv_path, 'w')
        csv_writer = CSVKitWriter(csv_file, quoting=csv.QUOTE_ALL)

        # Pull and clean the headers
        try:
            headers = tsv_file.readline()
        except StopIteration:
            return
        headers = headers.decode("ascii", "replace")
        headers_csv = CSVKitReader(StringIO(headers), delimiter=str('\t'))
        try:
            headers_list = next(headers_csv)
        except StopIteration:
            return
        headers_count = len(headers_list)
        csv_writer.writerow(headers_list)

        log_rows = []

        # Loop through the rest of the data
        line_number = 1
        for tsv_line in tsv_file:

{'fraction_non_alphanumeric': 0.09448275862068965, 'fraction_numerical': 0.006436781609195402, 'mean_word_length': 2.326452599388379, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4261767', 'n_tokens_mistral': 1374, 'n_tokens_neox': 1287, 'n_words': 325}