<commit_before>#!/usr/bin/env python3


import os
import argparse
import logging
import numpy as np
import shutil

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


class Runner(object):
    def __init__(self, config):
        self._config = config

    @property
    def Cfg(self):
        return self._config

    @staticmethod
    def add_line(fname, mname, nline=2, pattern=None):
        """
        Add a control line to the file named fname.

        :param fname:  file with type *.1
        :param mname:  model name
        :param nline:  position new line. Default: 2
        :param pattern: string to set the place to rename, as "111001", if it's None [default] rename all substrings.
        :return:
        """
        if nline < 1:
            raise ValueError('nline should be more 0: {}'.format(nline))

        nline -= 1  # shift to zero as array indexes

        # read the file
        with open(fname, "r") as f:
            lines = f.readlines()

        # take pattern as nline
        subs = lines[nline].split()
        if pattern is None:
            pattern = np.ones(len(subs))
        else:
            pattern = pattern.replace(' ', '')
            pattern = [int(p) for p in pattern]

        for i, p in enumerate(pattern):
            if bool(p):
                filename = subs[i]
                if '.' in filename:
                    (prefix, sep1, suffix) = filename.rpartition('.')
                else:
                    (prefix, sep1, suffix) = filename, '', ''
                (predir, sep2, nm) = prefix.rpartition('/')
                subs[i] = '{}{}{}{}{}'.format(predir, sep2, mname, sep1, suffix)
        newline = '  '.join(subs)

        # add new line at nline position
        lines.insert(nline, newline + "\n")
        # write to the file
        with open(fname, "w") as fout:
            for l in lines:
                fout.write(l)
            # fout.writelines(lines)

    @staticmethod
    def cp_sample(fin, fout, mode=1):
        if os.path.exists(fout):
            if mode == 2:
                raise ValueError('The target file: {}  is exist.'.format(fout))
            if mode == 1:
                logger.warning('The target file: {} was exist and left unchanged.'.format(fout))
                return

            logger.warning('The target file: {}  was rewritten.'.format(fout))
        logger.info(' Copy  {} to {}'.format(fin, fout))

        try:
            shutil.copy2(fin, fout)
        except shutil.SameFileError:
            logger.info(' Copy  {} and {} are the same file'.format(fin, fout))
            pass<commit_msg>- fix: log message<commit_after>#!/usr/bin/env python3


import os
import argparse
import logging
import numpy as np
import shutil

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)


class Runner(object):
    def __init__(self, config):
        self._config = config

    @property
    def Cfg(self):
        return self._config

    @staticmethod
    def add_line(fname, mname, nline=2, pattern=None):
        """
        Add a control line to the file named fname.

        :param fname:  file with type *.1
        :param mname:  model name
        :param nline:  position new line. Default: 2
        :param pattern: string to set the place to rename, as "111001", if it's None [default] rename all substrings.
        :return:
        """
        if nline < 1:
            raise ValueError('nline should be more 0: {}'.format(nline))

        nline -= 1  # shift to zero as array indexes

        # read the file
        with open(fname, "r") as f:
            lines = f.readlines()

        # take pattern as nline
        subs = lines[nline].split()
        if pattern is None:
            pattern = np.ones(len(subs))
        else:
            pattern = pattern.replace(' ', '')
            pattern = [int(p) for p in pattern]

        for i, p in enumerate(pattern):
            if bool(p):
                filename = subs[i]
                if '.' in filename:
                    (prefix, sep1, suffix) = filename.rpartition('.')
                else:
                    (prefix, sep1, suffix) = filename, '', ''
                (predir, sep2, nm) = prefix.rpartition('/')
                subs[i] = '{}{}{}{}{}'.format(predir, sep2, mname, sep1, suffix)
        newline = '  '.join(subs)

        # add new line at nline position
        lines.insert(nline, newline + "\n")
        # write to the file
        with open(fname, "w") as fout:
            for l in lines:
                fout.write(l)
            # fout.writelines(lines)

    @staticmethod
    def cp_sample(fin, fout, mode=1):
        if os.path.exists(fout):
            if mode == 2:
                raise ValueError('The target file: {}  is exist.'.format(fout))
            if mode == 1:
                logger.warning('The target file: {} was exist and left unchanged.'.format(fout))
                return

            logger.warning('The target file: {}  was rewritten.'.format(fout))
        logger.info(' Copy  {} to {}'.format(fin, fout))

        try:
            shutil.copy2(fin, fout)
        except shutil.SameFileError:
            logger.info('    {} and {} are the same file'.format(fin, fout))
            pass
{'fraction_non_alphanumeric': 0.11829176114890401, 'fraction_numerical': 0.008314436885865457, 'mean_word_length': 1.7741090146750524, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1916859', 'n_tokens_mistral': 1602, 'n_tokens_neox': 1522, 'n_words': 479}