<commit_before>"""
Created on 2 Aug 2016

@author: <NAME> (<EMAIL>)

https://stackoverflow.com/questions/3348460/csv-file-written-with-python-has-blank-lines-between-each-row
"""

import csv
import os
import sys

from scs_core.csv.csv_dict import CSVDict
from scs_core.data.path_dict import PathDict


# --------------------------------------------------------------------------------------------------------------------

class CSVWriter(object):
    """
    classdocs
    """

    QUOTING = csv.QUOTE_MINIMAL

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, filename=None, append=False, exclude_header=False, header_scan=False):
        """
        Constructor
        """
        self.__filename = filename
        self.__paths = []

        if self.__filename is None:
            self.__append = append

            self.__file = sys.stdout
            self.__writer = csv.writer(self.__file, quoting=self.QUOTING)
        else:
            self.__append = append and os.path.exists(self.__filename)

            if self.__append and not header_scan:
                self.__paths = self.__build_paths()

            self.__file = open(self.__filename, "a" if self.__append else "w", newline='')
            self.__writer = csv.writer(self.__file, quoting=self.QUOTING)

        self.__exclude_header = exclude_header
        self.__header_scan = header_scan

        self.__data = []


    # ----------------------------------------------------------------------------------------------------------------

    def write(self, jstr):
        if jstr is None:
            return False

        datum = CSVDict.construct_from_jstr(jstr)

        if datum is None:
            return False

        if self.__header_scan:
            self.__data.append(datum)
            self.__paths = self.__scan_paths(datum)

            return True

        if not self.__paths:
            self.__paths = datum.paths()

            # write header...
            if not self.__append and not self.__exclude_header:
                self.__writer.writerow(self.__paths)

        # write row...
        self.__writer.writerow(datum.row(self.__paths))
        self.__file.flush()

        return True


    def close(self):
        if self.__header_scan:
            # write header...
            self.__writer.writerow(self.__paths)

            # write rows...
            for datum in self.__data:
                self.__writer.writerow(datum.row(self.__paths))

        if self.filename is None:
            return

        self.__file.close()


    # ----------------------------------------------------------------------------------------------------------------

    def __build_paths(self):
        file = sys.stdin if self.__filename is None else open(self.__filename, "r")
        reader = csv.reader(file)

        paths = next(reader)

        file.close()

        return paths


    def __scan_paths(self, datum):
        datum_paths = datum.paths()
        max_len = max(len(self.__paths), len(datum_paths))
        paths = []

        for i in range(max_len):
            try:
                if datum_paths[i] not in paths and not self.__is_sub_path(datum_paths[i], self.__paths):
                    paths.append(datum_paths[i])
            except IndexError:
                pass

            try:
                if self.__paths[i] not in paths and not self.__is_sub_path(self.__paths[i], datum_paths):
                    paths.append(self.__paths[i])
            except IndexError:
                pass

        return paths


    @staticmethod
    def __is_sub_path(candidate, paths):
        for path in paths:
            if candidate == path:
                continue

            if PathDict.sub_path_includes_path(candidate, path):
                return True

        return False


    # ----------------------------------------------------------------------------------------------------------------

    @property
    def filename(self):
        return self.__filename


    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "CSVWriter:{filename:%s, append:%s, exclude_header:%s, header_scan:%s, paths:%s}" % \
               (self.filename, self.__append, self.__exclude_header, self.__header_scan, self.__paths)<commit_msg>Improved --header-scan function in CSVWriter<commit_after>"""
Created on 2 Aug 2016

@author: <NAME> (<EMAIL>)

https://stackoverflow.com/questions/3348460/csv-file-written-with-python-has-blank-lines-between-each-row
"""

import csv
import os
import sys

from scs_core.csv.csv_dict import CSVDict
from scs_core.data.path_dict import PathDict


# --------------------------------------------------------------------------------------------------------------------

class CSVWriter(object):
    """
    classdocs
    """

    QUOTING = csv.QUOTE_MINIMAL

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, filename=None, append=False, exclude_header=False, header_scan=False):
        """
        Constructor
        """
        self.__filename = filename
        self.__paths = []

        if self.__filename is None:
            self.__append = append

            self.__file = sys.stdout
            self.__writer = csv.writer(self.__file, quoting=self.QUOTING)
        else:
            self.__append = append and os.path.exists(self.__filename)

            if self.__append and not header_scan:
                self.__paths = self.__build_paths()

            self.__file = open(self.__filename, "a" if self.__append else "w", newline='')
            self.__writer = csv.writer(self.__file, quoting=self.QUOTING)

        self.__exclude_header = exclude_header
        self.__header_scan = header_scan

        self.__data = []


    # ----------------------------------------------------------------------------------------------------------------

    def write(self, jstr):
        if jstr is None:
            return False

        datum = CSVDict.construct_from_jstr(jstr)

        if datum is None:
            return False

        if self.__header_scan:
            self.__data.append(datum)
            self.__update_paths(datum)

            return True

        if not self.__paths:
            self.__paths = datum.paths()

            # write header...
            if not self.__append and not self.__exclude_header:
                self.__writer.writerow(self.__paths)

        # write row...
        self.__writer.writerow(datum.row(self.__paths))
        self.__file.flush()

        return True


    def close(self):
        if self.__header_scan:
            # write header...
            self.__writer.writerow(self.__paths)

            # write rows...
            for datum in self.__data:
                self.__writer.writerow(datum.row(self.__paths))

        if self.filename is None:
            return

        self.__file.close()


    # ----------------------------------------------------------------------------------------------------------------

    def __build_paths(self):
        file = sys.stdin if self.__filename is None else open(self.__filename, "r")
        reader = csv.reader(file)

        paths = next(reader)

        file.close()

        return paths


    def __update_paths(self, datum):
        datum_paths = datum.paths()

        append_paths = []
        for i in range(len(datum_paths)):
            if datum_paths[i] not in self.__paths and not self.__is_sub_path(datum_paths[i], self.__paths):
                self.__paths.insert(i, datum_paths[i])
                append_paths.append(datum_paths[i])

        if append_paths:
            for i in reversed(range(len(self.__paths))):
                if self.__is_sub_path(self.__paths[i], append_paths):
                    self.__paths.pop(i)


    @staticmethod
    def __is_sub_path(candidate, paths):
        for path in paths:
            if candidate == path:
                return False            # exit here because paths are assumed to be unique

            if PathDict.sub_path_includes_path(candidate, path):
                return True

        return False


    # ----------------------------------------------------------------------------------------------------------------

    @property
    def filename(self):
        return self.__filename


    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "CSVWriter:{filename:%s, append:%s, exclude_header:%s, header_scan:%s, paths:%s}" % \
               (self.filename, self.__append, self.__exclude_header, self.__header_scan, self.__paths)
{'fraction_non_alphanumeric': 0.2357769282662469, 'fraction_numerical': 0.0026984483921744995, 'mean_word_length': 1.2428139183055975, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2861975', 'n_tokens_mistral': 2282, 'n_tokens_neox': 2114, 'n_words': 485}