<commit_before>import re, os, logging, subprocess, platform
from tempfile import NamedTemporaryFile
from django.conf import settings


_log = logging.getLogger(__name__)

NULL_FILE = 'nul' if 'Windows' in platform.system() else '/dev/null'


class BCP(object):

    '''
    https://docs.microsoft.com/en-us/sql/tools/bcp-utility
    '''

    target_model = None
    bcp_path = None

    _command_args_base = None
    _db_args = None
    _table_name = None
    _field_column_map = None

    def __init__(self, target_model, bcp_path='bcp'):
        self.bcp_path = bcp_path
        self.set_target_model(target_model)

    def save(self, rows):

        # Create the bcp FORMAT file from target_model
        bcp_format = self._make_format()

        # Create a temporary file to hold bulk data
        with NamedTemporaryFile(delete=True) as f:
            outfile = '%s_%s.csv' % (f.name, self._table_name)

        # Write bulk data based on FORMAT file
        _log.debug('Writing bulk data file %s', outfile)

        # Open in binary mode so that newlines don't get doubled on Windows: http://stackoverflow.com/a/4025988/193244
        with open(outfile, 'wb') as f:
            for row in rows:
                for field in bcp_format.fields:
                    model_field = self._field_column_map[field.column_name]
                    val = row.get(model_field.name, None) or ''
                    val = getattr(val, 'id', val) # if ForeignKey, we need id
                    if model_field.__class__.__name__ == 'DecimalField':
                        val = ('%.' + str(model_field.decimal_places) + 'f') % float(val or 0)
                    f.write(str(val))
                    f.write(field.delimiter)

        # Do bulk import via bcp
        _log.debug('Calling bcp')
        import_result = _run_cmd(self._command_args_base + ['IN', outfile] + self._db_args + ['-f', bcp_format.filename])<commit_msg>Set correct BCP exe path on Linux<commit_after>import re, os, logging, subprocess, platform
from tempfile import NamedTemporaryFile
from django.conf import settings


_log = logging.getLogger(__name__)

NULL_FILE = 'nul' if 'Windows' in platform.system() else '/dev/null'
BCP_EXE = 'bcp' if 'Windows' in platform.system() else '/opt/mssql-tools/bin/bcp'


class BCP(object):

    '''
    https://docs.microsoft.com/en-us/sql/tools/bcp-utility
    '''

    target_model = None
    bcp_path = None

    _command_args_base = None
    _db_args = None
    _table_name = None
    _field_column_map = None

    def __init__(self, target_model, bcp_path=BCP_EXE):
        self.bcp_path = bcp_path
        self.set_target_model(target_model)

    def save(self, rows):

        # Create the bcp FORMAT file from target_model
        bcp_format = self._make_format()

        # Create a temporary file to hold bulk data
        with NamedTemporaryFile(delete=True) as f:
            outfile = '%s_%s.csv' % (f.name, self._table_name)

        # Write bulk data based on FORMAT file
        _log.debug('Writing bulk data file %s', outfile)

        # Open in binary mode so that newlines don't get doubled on Windows: http://stackoverflow.com/a/4025988/193244
        with open(outfile, 'wb') as f:
            for row in rows:
                for field in bcp_format.fields:
                    model_field = self._field_column_map[field.column_name]
                    val = row.get(model_field.name, None) or ''
                    val = getattr(val, 'id', val) # if ForeignKey, we need id
                    if model_field.__class__.__name__ == 'DecimalField':
                        val = ('%.' + str(model_field.decimal_places) + 'f') % float(val or 0)
                    f.write(str(val))
                    f.write(field.delimiter)

        # Do bulk import via bcp
        _log.debug('Calling bcp')
        import_result = _run_cmd(self._command_args_base + ['IN', outfile] + self._db_args + ['-f', bcp_format.filename])
{'fraction_non_alphanumeric': 0.10324708269913749, 'fraction_numerical': 0.007102993404363267, 'mean_word_length': 2.3471986417657047, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3809854', 'n_tokens_mistral': 1279, 'n_tokens_neox': 1210, 'n_words': 319}