<commit_before>#
# Copyright (C) 2016 by frePPLe bv
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

from datetime import datetime
import io
from importlib import import_module
from operator import attrgetter
import os
import sys
import logging
from threading import Thread


if __name__ == "__main__":
    # Support for running in Python virtual environments
    if "VIRTUAL_ENV" in os.environ:
        activate_script = os.path.join(
            os.environ["VIRTUAL_ENV"],
            "Scripts" if os.name == "nt" else "bin",
            "activate_this.py",
        )
        exec(open(activate_script).read(), {"__file__": activate_script})

    # Initialize django
    import django

    django.setup()

from django.conf import settings
from django.db import DEFAULT_DB_ALIAS, connections
from django.utils.encoding import force_text

from freppledb.execute.models import Task

logger = logging.getLogger(__name__)


def clean_value(value):
    """
    A small auxilary function to handle newline characters or backslashes
    in exporting data to PostgreSQL over a COPY command.
    """
    if value is None:
        return "\\N"
    elif "\n" in value or "\\" in value:
        return value.replace("\n", "\\n").replace("\\", "\\\\")
    else:
        return value


class CopyFromGenerator(io.TextIOBase):
    """
    File-like object to handle exporting data to PostgreSQL over
    a copy command.

    Inspired on and copied from:
      https://hakibenita.com/fast-load-data-python-postgresql
    """

    def __init__(self, itr):
        self._iter = itr
        self._buff = ""

    def readable(self):
        return True

    def _read1(self, n=None):
        while not self._buff:
            try:
                self._buff = next(self._iter)<commit_msg>If my item name is like this : "xxxxxx\n" Then after the first replace, it will become "xxxxx\\n" After the second replace, it will become "xxxx\\\\n" => That's a completely different item. Swapping the order of "replace" fixes this situation<commit_after>#
# Copyright (C) 2016 by frePPLe bv
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

from datetime import datetime
import io
from importlib import import_module
from operator import attrgetter
import os
import sys
import logging
from threading import Thread


if __name__ == "__main__":
    # Support for running in Python virtual environments
    if "VIRTUAL_ENV" in os.environ:
        activate_script = os.path.join(
            os.environ["VIRTUAL_ENV"],
            "Scripts" if os.name == "nt" else "bin",
            "activate_this.py",
        )
        exec(open(activate_script).read(), {"__file__": activate_script})

    # Initialize django
    import django

    django.setup()

from django.conf import settings
from django.db import DEFAULT_DB_ALIAS, connections
from django.utils.encoding import force_text

from freppledb.execute.models import Task

logger = logging.getLogger(__name__)


def clean_value(value):
    """
    A small auxilary function to handle newline characters or backslashes
    in exporting data to PostgreSQL over a COPY command.
    """
    if value is None:
        return "\\N"
    elif "\n" in value or "\\" in value:
        return value.replace("\\", "\\\\").replace("\n", "\\n")
    else:
        return value


class CopyFromGenerator(io.TextIOBase):
    """
    File-like object to handle exporting data to PostgreSQL over
    a copy command.

    Inspired on and copied from:
      https://hakibenita.com/fast-load-data-python-postgresql
    """

    def __init__(self, itr):
        self._iter = itr
        self._buff = ""

    def readable(self):
        return True

    def _read1(self, n=None):
        while not self._buff:
            try:
                self._buff = next(self._iter)
{'fraction_non_alphanumeric': 0.08622418287547624, 'fraction_numerical': 0.00240625626629236, 'mean_word_length': 2.9904, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5189051', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1371, 'n_words': 563}