<commit_before># -*- coding: utf-8 -*-
import fnmatch
import logging
import os
import shutil
from collections import OrderedDict

from crosspm.helpers.archive import Archive


class Package:
    def __init__(self, name, pkg, params, downloader, adapter, parser, params_found=None, params_found_raw=None,
                 stat=None, in_cache=False):
        self.name = name
        self.package_name = name
        self.packed_path = ''
        self.unpacked_path = ''
        self.duplicated = False
        self.packages = OrderedDict()

        if isinstance(pkg, int):
            if pkg == 0:
                self._root = True

        self._raw = []
        self._root = False
        self._params_found = {}
        self._params_found_raw = {}
        self._not_cached = True
        self._log = logging.getLogger('crosspm')

        self._pkg = pkg
        self._params = params
        self._adapter = adapter
        self._parser = parser
        self._downloader = downloader
        self._in_cache = in_cache

        if params_found:
            self._params_found = params_found
        if params_found_raw:
            self._params_found_raw = params_found_raw
        self.stat = stat

    def download(self, force=False):
        """
        Download file containing this package.
        :param force: Force download even if it seems file already exists
        :return: Full path with filename of downloaded package file.
        """
        exists, dest_path = self._downloader.cache.exists_packed(package=self, pkg_path=self.packed_path,
                                                                 check_stat=not self._in_cache)
        if exists and not self.packed_path:
            self.packed_path = dest_path
        unp_exists, unp_path = self._downloader.cache.exists_unpacked(package=self, pkg_path=self.unpacked_path)

        if force or not exists:
            # _packed_path = self._packed_path
            dest_path_tmp = dest_path + ".tmp"
            if os.path.exists(dest_path_tmp):
                os.remove(dest_path_tmp)
            self._adapter.download_package(self._pkg, dest_path_tmp)
            os.rename(dest_path_tmp, dest_path)
            self.packed_path = dest_path
            # if not _packed_path:
            self._not_cached = True
        else:
            if unp_exists and not self.unpacked_path:
                self.unpacked_path = unp_path
                self._not_cached = False

        if self._not_cached and unp_exists:
            shutil.rmtree(unp_path, ignore_errors=True)

        return self.packed_path

    def get_file(self, file_name, unpack_force=True):
        if unpack_force:
            self.unpack()
        _dest_file = self.get_file_path(file_name)
        _dest_file = _dest_file if os.path.isfile(_dest_file) else None
        return _dest_file

    def get_file_path(self, file_name):
        _dest_file = os.path.join(self.unpacked_path, file_name)
        return _dest_file

    def find_dependencies(self, depslock_file_path):
        self._raw = [x for x in self._parser.iter_packages_params(depslock_file_path)]
        self.packages = self._downloader.get_dependency_packages({'raw': self._raw})

    def unpack(self, force=False):
        if self._downloader.solid(self):
            self.unpacked_path = self.packed_path
        else:
            exists, dest_path = self._downloader.cache.exists_unpacked(package=self, pkg_path=self.unpacked_path)
            if exists and not self.unpacked_path:
                self.unpacked_path = dest_path

            # if force or not exists:

            # if not dest_path:
            #     dest_path = self._downloader.unpacked_path
            # temp_path = os.path.realpath(os.path.join(dest_path, self._name))
            # _exists = os.path.exists(temp_path)
            if not self._not_cached:
                self.unpacked_path = dest_path if exists else ''  # temp_path if exists else ''
            if force or self._not_cached or (not exists):
                Archive.extract(self.packed_path, dest_path)  # temp_path)
                self.unpacked_path = dest_path  # temp_path
                self._not_cached = False

    def pack(self, src_path):
        Archive.create(self.packed_path, src_path)

    def print(self, level=0, output=None):
        def do_print(left):
            res_str = ''
            for out_item in output:
                for k, v in out_item.items():
                    cur_str = self._params_found.get(k, '')
                    if not res_str:
                        cur_str = self._params.get(k, '')
                    if not res_str:
                        cur_str = '{}{} '.format(left, cur_str)
                    cur_format = '{}'
                    if v > 0:
                        cur_format = '{:%s}' % (v if len(cur_str) <= v else v + len(left))
                    res_str += cur_format.format(cur_str)
                    break
            self._log.info(res_str)

        _sign = ' '
        if not self._root:
            if self.duplicated:
                _sign = '!'
            elif self.unpacked_path:
                _sign = '+'
            elif self.packed_path:
                _sign = '>'
            else:
                _sign = '-'
        _left = '{}{}'.format(' ' * 4 * level, _sign)
        do_print(_left)
        for _pkg_name in self.packages:
            _pkg = self.packages[_pkg_name]
            if not _pkg:
                _left = '{}-'.format(' ' * 4 * (level + 1))
                self._log.info('{}{}'.format(_left, _pkg_name))
            else:<commit_msg>get_params in print<commit_after># -*- coding: utf-8 -*-
import fnmatch
import logging
import os
import shutil
from collections import OrderedDict

from crosspm.helpers.archive import Archive


class Package:
    def __init__(self, name, pkg, params, downloader, adapter, parser, params_found=None, params_found_raw=None,
                 stat=None, in_cache=False):
        self.name = name
        self.package_name = name
        self.packed_path = ''
        self.unpacked_path = ''
        self.duplicated = False
        self.packages = OrderedDict()

        if isinstance(pkg, int):
            if pkg == 0:
                self._root = True

        self._raw = []
        self._root = False
        self._params_found = {}
        self._params_found_raw = {}
        self._not_cached = True
        self._log = logging.getLogger('crosspm')

        self._pkg = pkg
        self._params = params
        self._adapter = adapter
        self._parser = parser
        self._downloader = downloader
        self._in_cache = in_cache

        if params_found:
            self._params_found = params_found
        if params_found_raw:
            self._params_found_raw = params_found_raw
        self.stat = stat

    def download(self, force=False):
        """
        Download file containing this package.
        :param force: Force download even if it seems file already exists
        :return: Full path with filename of downloaded package file.
        """
        exists, dest_path = self._downloader.cache.exists_packed(package=self, pkg_path=self.packed_path,
                                                                 check_stat=not self._in_cache)
        if exists and not self.packed_path:
            self.packed_path = dest_path
        unp_exists, unp_path = self._downloader.cache.exists_unpacked(package=self, pkg_path=self.unpacked_path)

        if force or not exists:
            # _packed_path = self._packed_path
            dest_path_tmp = dest_path + ".tmp"
            if os.path.exists(dest_path_tmp):
                os.remove(dest_path_tmp)
            self._adapter.download_package(self._pkg, dest_path_tmp)
            os.rename(dest_path_tmp, dest_path)
            self.packed_path = dest_path
            # if not _packed_path:
            self._not_cached = True
        else:
            if unp_exists and not self.unpacked_path:
                self.unpacked_path = unp_path
                self._not_cached = False

        if self._not_cached and unp_exists:
            shutil.rmtree(unp_path, ignore_errors=True)

        return self.packed_path

    def get_file(self, file_name, unpack_force=True):
        if unpack_force:
            self.unpack()
        _dest_file = self.get_file_path(file_name)
        _dest_file = _dest_file if os.path.isfile(_dest_file) else None
        return _dest_file

    def get_file_path(self, file_name):
        _dest_file = os.path.join(self.unpacked_path, file_name)
        return _dest_file

    def find_dependencies(self, depslock_file_path):
        self._raw = [x for x in self._parser.iter_packages_params(depslock_file_path)]
        self.packages = self._downloader.get_dependency_packages({'raw': self._raw})

    def unpack(self, force=False):
        if self._downloader.solid(self):
            self.unpacked_path = self.packed_path
        else:
            exists, dest_path = self._downloader.cache.exists_unpacked(package=self, pkg_path=self.unpacked_path)
            if exists and not self.unpacked_path:
                self.unpacked_path = dest_path

            # if force or not exists:

            # if not dest_path:
            #     dest_path = self._downloader.unpacked_path
            # temp_path = os.path.realpath(os.path.join(dest_path, self._name))
            # _exists = os.path.exists(temp_path)
            if not self._not_cached:
                self.unpacked_path = dest_path if exists else ''  # temp_path if exists else ''
            if force or self._not_cached or (not exists):
                Archive.extract(self.packed_path, dest_path)  # temp_path)
                self.unpacked_path = dest_path  # temp_path
                self._not_cached = False

    def pack(self, src_path):
        Archive.create(self.packed_path, src_path)

    def print(self, level=0, output=None):
        def do_print(left):
            res_str = ''
            for out_item in output:
                for k, v in out_item.items():
                    cur_str = self.get_params(merged=True).get(k, '')
                    if not res_str:
                        cur_str = self._params.get(k, '')
                    if not res_str:
                        cur_str = '{}{} '.format(left, cur_str)
                    cur_format = '{}'
                    if v > 0:
                        cur_format = '{:%s}' % (v if len(cur_str) <= v else v + len(left))
                    res_str += cur_format.format(cur_str)
                    break
            self._log.info(res_str)

        _sign = ' '
        if not self._root:
            if self.duplicated:
                _sign = '!'
            elif self.unpacked_path:
                _sign = '+'
            elif self.packed_path:
                _sign = '>'
            else:
                _sign = '-'
        _left = '{}{}'.format(' ' * 4 * level, _sign)
        do_print(_left)
        for _pkg_name in self.packages:
            _pkg = self.packages[_pkg_name]
            if not _pkg:
                _left = '{}-'.format(' ' * 4 * (level + 1))
                self._log.info('{}{}'.format(_left, _pkg_name))
            else:
{'fraction_non_alphanumeric': 0.09002498215560314, 'fraction_numerical': 0.0012491077801570307, 'mean_word_length': 1.7710754017305315, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4090278', 'n_tokens_mistral': 3300, 'n_tokens_neox': 3172, 'n_words': 713}