<commit_before>import os
import sys
import utils
import logging
import traceback
import datetime
import hash_version
import copy
from sets import Set

class InvalidPlatform(Exception):
    def __init__(self, plat):
        self._plat = plat
    def __repr__(self):
        return "Invalid platform detected: %s" % self._plat

class DontExistsFile(Exception):
    def __init__(self, source_filename):
        self._source_filename = source_filename
    def __repr__(self):
        return 'Dont exists file %s' % self._source_filename

class FailPrepare(Exception):
    def __init__(self, node):
        self._node = node
    def __repr__(self):
        return ('Failing preparing package: %s' % self._node.get_package_name())

class AmbiguationLibs(Exception):
    def __init__(self, kind, package, build_mode):
        self._kind = kind
        self._package = package
        self._build_mode = build_mode
    def __repr__(self):
        return "Ambiguation in %s in %s. Mode: %s. Candidates:" % (self._kind, self._package, self._build_mode)

class NotFoundInDataset(Exception):
    def __init__(self, msg):
        self._msg = msg
    def __repr__(self):
        return "%s" % self._msg

class FailThirdParty(Exception):
    def __init__(self, msg):
        self._msg = msg
    def __repr__(self):
        return "%s" % self._msg

#
# INMUTABLE GLOBALS
#
HTTP_URL_NPSERVER = 'https://artifacts.000webhostapp.com/packages'
CMAKELIB_URL='https://github.com/makiolo/cmaki.git'
prefered = {}
prefered['Debug'] = ['Debug', 'RelWithDebInfo', 'Release']
prefered['RelWithDebInfo'] = ['RelWithDebInfo', 'Release', 'Debug']
prefered['Release'] = ['Release', 'RelWithDebInfo', 'Debug']
magic_invalid_file = '__not_found__'
exceptions_fail_group = (OSError, IOError, )
exceptions_fail_program = (KeyboardInterrupt, )
uncompress_strip_default = '.'
uncompress_prefix_default = '.'
priority_default = 50
build_unittests_foldername = 'unittest'
if sys.platform.startswith("win"):
    somask_id = 'w'
    archs = {'win32': '', 'win64': '64'}
    platforms = ["win32", "win64"]
elif sys.platform.startswith("linux"):  # linux2
    try:
        value = os.environ['EMSCRIPTEN']
        is_emscripten = (str(value) == '1')
    except KeyError:
        is_emscripten = False
    ######
    if is_emscripten:
        print('using emscripten ...')
        somask_id = 'e'
        archs = {'emscripten': ''}
        platforms = ["emscripten"]
    else:
        print('using linux ...')
        somask_id = 'l'
        archs = {"default": '64'}
        platforms = ["default"]
elif sys.platform.startswith("sun"): # sunos5
    somask_id = 's'
    archs = {'solaris_sparc32': ''}
    platforms = ["solaris_sparc32"]
else:
    raise InvalidPlatform(sys.platform)
alias_priority_name = { 10: 'minimal',
                        20: 'tools',
                        30: 'third_party' }
alias_priority_name_inverse = {v: k for k, v in alias_priority_name.items()}


def is_valid(package_name, mask):
    return (mask.find(somask_id) != -1) and (package_name != 'dummy')

def is_blacklisted(blacklist_file, no_blacklist, package_name):
    blacklisted = False
    if os.path.exists(blacklist_file):
        with open(blacklist_file, 'rt') as f:
            for line in f.readlines():
                if line.strip() == package_name:
                    blacklisted = True
                    break
    # --no-blacklist can annular effect of blacklist
    if blacklisted and (package_name in no_blacklist):<commit_msg>use script for generate platform<commit_after>import os
import sys
import utils
import logging
import traceback
import datetime
import hash_version
import copy
from sets import Set

class InvalidPlatform(Exception):
    def __init__(self, plat):
        self._plat = plat
    def __repr__(self):
        return "Invalid platform detected: %s" % self._plat

class DontExistsFile(Exception):
    def __init__(self, source_filename):
        self._source_filename = source_filename
    def __repr__(self):
        return 'Dont exists file %s' % self._source_filename

class FailPrepare(Exception):
    def __init__(self, node):
        self._node = node
    def __repr__(self):
        return ('Failing preparing package: %s' % self._node.get_package_name())

class AmbiguationLibs(Exception):
    def __init__(self, kind, package, build_mode):
        self._kind = kind
        self._package = package
        self._build_mode = build_mode
    def __repr__(self):
        return "Ambiguation in %s in %s. Mode: %s. Candidates:" % (self._kind, self._package, self._build_mode)

class NotFoundInDataset(Exception):
    def __init__(self, msg):
        self._msg = msg
    def __repr__(self):
        return "%s" % self._msg

class FailThirdParty(Exception):
    def __init__(self, msg):
        self._msg = msg
    def __repr__(self):
        return "%s" % self._msg

#
# INMUTABLE GLOBALS
#
HTTP_URL_NPSERVER = 'https://artifacts.000webhostapp.com/packages'
CMAKELIB_URL='https://github.com/makiolo/cmaki.git'
prefered = {}
prefered['Debug'] = ['Debug', 'RelWithDebInfo', 'Release']
prefered['RelWithDebInfo'] = ['RelWithDebInfo', 'Release', 'Debug']
prefered['Release'] = ['Release', 'RelWithDebInfo', 'Debug']
magic_invalid_file = '__not_found__'
exceptions_fail_group = (OSError, IOError, )
exceptions_fail_program = (KeyboardInterrupt, )
uncompress_strip_default = '.'
uncompress_prefix_default = '.'
priority_default = 50
build_unittests_foldername = 'unittest'
if sys.platform.startswith("win"):
    somask_id = 'w'
    archs = {'win32': '', 'win64': '64'}
    platforms = ["win32", "win64"]
elif sys.platform.startswith("linux"):  # linux2
    try:
        value = os.environ['EMSCRIPTEN']
        is_emscripten = (str(value) == '1')
    except KeyError:
        is_emscripten = False
    ######
    if is_emscripten:
        print('using emscripten ...')
        somask_id = 'e'
        archs = {'emscripten': ''}
        platforms = ["emscripten"]
    else:
        print('using linux ...')
        somask_id = 'l'
        for platform in utils.get_stdout(os.path.join(parameters.cmakefiles, 'ci', 'detect_operative_system.sh')):
            archs = {platform: '64'}
            platforms = [platform]
            break
elif sys.platform.startswith("sun"): # sunos5
    somask_id = 's'
    archs = {'solaris_sparc32': ''}
    platforms = ["solaris_sparc32"]
else:
    raise InvalidPlatform(sys.platform)
alias_priority_name = { 10: 'minimal',
                        20: 'tools',
                        30: 'third_party' }
alias_priority_name_inverse = {v: k for k, v in alias_priority_name.items()}


def is_valid(package_name, mask):
    return (mask.find(somask_id) != -1) and (package_name != 'dummy')

def is_blacklisted(blacklist_file, no_blacklist, package_name):
    blacklisted = False
    if os.path.exists(blacklist_file):
        with open(blacklist_file, 'rt') as f:
            for line in f.readlines():
                if line.strip() == package_name:
                    blacklisted = True
                    break
    # --no-blacklist can annular effect of blacklist
    if blacklisted and (package_name in no_blacklist):
{'fraction_non_alphanumeric': 0.12471973094170404, 'fraction_numerical': 0.008688340807174889, 'mean_word_length': 3.156668608037274, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5345766', 'n_tokens_mistral': 2331, 'n_tokens_neox': 2274, 'n_words': 534}