<commit_before>import codecs
import os
import re


class CsvWriter(object):
    """
    The csv library doesn't handle unicode strings, so we've written
    our own here.

    This class takes a generator function to iterate through all the
    dicts of data we wish to write to csv. This class also takes a key
    comparator (for sorting the keys), and a function to rename the
    headers.
    """
    def __init__(self):
        self._dict_iterator = []
        self._keys = []
        self._key_comparator = cmp
        self._key_rename_function = lambda x: x

    def set_generator_function(self, generator_function):
        """
        Unfortunately there doesn't seem to be a way to rewind a
        generator, so instead of simply passing a generator as my
        dict_iterator, I'm passing the generator function so we can
        get a new generator after we run through the first.
        """
        self._generator_function = generator_function
        self._reset_dict_iterator()
        self._create_list_of_keys()

    # def set_dict_iterator(self, dict_iterator):
    #     self._dict_iterator = dict_iterator
    #     self._create_list_of_keys()

    def _reset_dict_iterator(self):
        self._dict_iterator = self._generator_function()

    def _create_list_of_keys(self):
        key_set = set()
        for d in self._dict_iterator:
            for k in d.iterkeys():
                key_set.add(k)
        self._keys = list(key_set)
        self._reset_dict_iterator()

    def set_key_comparator(self, key_comparator):
        self._key_comparator = key_comparator

    def _sort_keys(self):
        self._keys.sort(cmp=self._key_comparator)

    def set_key_rename_function(self, key_rename_function):
        self._key_rename_function = key_rename_function

    def _ensure_directory_exists(self, path):
        abspath = os.path.abspath(path)
        directory = os.path.dirname(abspath)
        if not os.path.exists(directory):
            os.makedirs(directory)

    def write_to_file(self, path):
        self._ensure_directory_exists(path)
        self._file_object = codecs.open(path, mode="w", encoding="utf-8")

        self._sort_keys()
        headers = [self._key_rename_function(k) for k in self._keys]
        self._write_row(headers)

        for d in self._dict_iterator:
            # todo: figure out how to use csv.writer with unicode
            self._write_row([d.get(k, u"n/a") for k in self._keys])
        self._reset_dict_iterator()

        self._file_object.close()

    def _write_row(self, row):
        quote_escaped_row = []
        for cell in row:
            cell_string = unicode(cell)
            cell_string = re.sub(ur"\s+", u" ", cell_string)
            if u',' in cell_string:
                quote_escaped_row.append(u'"%s"' % cell_string)
            else:
                quote_escaped_row.append(cell_string)
        row_string = u",".join(quote_escaped_row)
        self._file_object.writelines([row_string, u"\n"])


from xform_manager.models import XForm
from parsed_xforms.models import ParsedInstance
from utils.reinhardt import queryset_iterator


class XFormWriter(CsvWriter):

    def __init__(self):
        super(XFormWriter, self).__init__()
        self._xform = None

    def set_xform(self, xform):
        self._xform = xform

        generator_function = self.get_data_for_csv_writer
        self.set_generator_function(generator_function)

    def get_data_for_csv_writer(self):
        qs = ParsedInstance.objects.filter(instance__xform=self._xform)
        for parsed_instance in queryset_iterator(qs):
            yield parsed_instance.to_dict()<commit_msg>Got csv export working.<commit_after>import codecs
import os
import re


class CsvWriter(object):
    """
    The csv library doesn't handle unicode strings, so we've written
    our own here.

    This class takes a generator function to iterate through all the
    dicts of data we wish to write to csv. This class also takes a key
    comparator (for sorting the keys), and a function to rename the
    headers.
    """
    def __init__(self):
        self._dict_iterator = []
        self._keys = []
        self._key_comparator = cmp
        self._key_rename_function = lambda x: x

    def set_generator_function(self, generator_function):
        """
        Unfortunately there doesn't seem to be a way to rewind a
        generator, so instead of simply passing a generator as my
        dict_iterator, I'm passing the generator function so we can
        get a new generator after we run through the first.
        """
        self._generator_function = generator_function
        self._reset_dict_iterator()
        self._create_list_of_keys()

    # def set_dict_iterator(self, dict_iterator):
    #     self._dict_iterator = dict_iterator
    #     self._create_list_of_keys()

    def _reset_dict_iterator(self):
        self._dict_iterator = self._generator_function()

    def _create_list_of_keys(self):
        key_set = set()
        for d in self._dict_iterator:
            for k in d.iterkeys():
                key_set.add(k)
        self._keys = list(key_set)
        self._reset_dict_iterator()

    def set_key_comparator(self, key_comparator):
        self._key_comparator = key_comparator

    def _sort_keys(self):
        self._keys.sort(cmp=self._key_comparator)

    def set_key_rename_function(self, key_rename_function):
        self._key_rename_function = key_rename_function

    def _ensure_directory_exists(self, path):
        abspath = os.path.abspath(path)
        directory = os.path.dirname(abspath)
        if not os.path.exists(directory):
            os.makedirs(directory)

    def write_to_file(self, path):
        self._ensure_directory_exists(path)
        self._file_object = codecs.open(path, mode="w", encoding="utf-8")

        self._sort_keys()
        headers = [self._key_rename_function(k) for k in self._keys]
        self._write_row(headers)

        for d in self._dict_iterator:
            # todo: figure out how to use csv.writer with unicode
            self._write_row([d.get(k, u"n/a") for k in self._keys])
        self._reset_dict_iterator()

        self._file_object.close()

    def _write_row(self, row):
        quote_escaped_row = []
        for cell in row:
            cell_string = unicode(cell)
            cell_string = re.sub(ur"\s+", u" ", cell_string)
            if u',' in cell_string:
                quote_escaped_row.append(u'"%s"' % cell_string)
            else:
                quote_escaped_row.append(cell_string)
        row_string = u",".join(quote_escaped_row)
        self._file_object.writelines([row_string, u"\n"])


from xform_manager.models import XForm, Instance
from utils.reinhardt import queryset_iterator


class XFormWriter(CsvWriter):

    def __init__(self):
        super(XFormWriter, self).__init__()
        self._xform = None

    def set_xform(self, xform):
        self._xform = xform

        generator_function = self.get_data_for_csv_writer
        self.set_generator_function(generator_function)

    def get_data_for_csv_writer(self):
        qs = Instance.objects.filter(xform=self._xform)
        for instance in queryset_iterator(qs):
            d = instance.get_dict()
            print d
            yield d
{'fraction_non_alphanumeric': 0.08396213472355604, 'fraction_numerical': 0.00027438606118809165, 'mean_word_length': 2.594674556213018, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1665928', 'n_tokens_mistral': 2221, 'n_tokens_neox': 2158, 'n_words': 537}