<commit_before># -*- coding:utf-8 -*-
'''
Function for building whole packed version of highlight.js out of
pre-packed modules.
'''

import os
import sys

def build(packed_path, target_path, languages):
    '''
    packed_path -- path to pre-packed .js files including language files and highligh.js itself
    target_path -- path where to put the final 'highlight.pack.js'
    languages -- list of language names to include in the final package
    '''
    ALIASES = {
        'html': ['html-xml'],
        'xml': ['html-xml'],
        'css': ['html-xml', 'css'],
        'django': ['html-xml', 'django'],
    }
    all_files = os.listdir(packed_path)
    all_files = [l for l in all_files if l.endswith('.js') and not l.startswith('highlight.')]
    if languages is not None:
        selected_languages = []
        for l in languages:
            selected_languages += ALIASES.get(l, [l])
        all_languages = set(os.path.splitext(l)[0] for l in all_files)
        languages = all_languages & set(selected_languages)
        files = ['%s.js' % l for l in languages]
    else:
        files = all_files
    f = open(os.path.join(target_path, 'highlight.pack.js'), 'w')
    f.write(open(os.path.join(packed_path, 'highlight.js')).read())
    for file in files:
        f.write(open(os.path.join(packed_path, file)).read())
    f.close()

if __name__ == '__main__':
    path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    packed_path = os.path.join(path, 'packed')
    target_path = os.path.join(path, 'src')
    if len(sys.argv) > 1:
        languages = sys.argv[1:]
    else:
        languages = None
    build(packed_path, target_path, languages)<commit_msg>Рефакторинг функций в build.py для библиотечного использования<commit_after># -*- coding:utf-8 -*-
'''
Function for building whole packed version of highlight.js out of
pre-packed modules.
'''

import os
import sys

ALIASES = {
    'html': ['html-xml'],
    'xml': ['html-xml'],
    'css': ['html-xml', 'css'],
    'django': ['html-xml', 'django'],
}

def language_files(packed_path):
    '''
    Returns a list of all language .js files from packed_path.
    '''
    files = os.listdir(packed_path)
    files = [l for l in files if l.endswith('.js') and not l.startswith('highlight.')]
    return files

def language_names(packed_path):
    '''
    Returns a list of all available language names converting necessary filenames with ALIASES
    '''
    aliased_filenames = set()
    for l in ALIASES.values():
        aliased_filenames |= set(l)
    aliased_languages = ALIASES.keys()
    names = [os.path.splitext(f)[0] for f in language_files(packed_path)]
    return [n for n in names if n not in aliased_filenames or n in aliased_languages]

def build_content(packed_path, languages):
    '''
    Builds content of highlight.pack.js and returns it as a  string.
    '''
    all_files = language_files(packed_path)
    if languages is not None:
        selected_languages = []
        for l in languages:
            selected_languages += ALIASES.get(l, [l])
        all_languages = set(os.path.splitext(l)[0] for l in all_files)
        languages = all_languages & set(selected_languages)
        files = ['%s.js' % l for l in languages]
    else:
        files = all_files
    contents = [open(os.path.join(packed_path, f)).read() for f in files]
    contents.insert(0, open(os.path.join(packed_path, 'highlight.js')).read())
    return ''.join(contents)
    
def build(packed_path, target_path, languages):
    '''
    Builds highlight.pack.js and puts it under target_path.
    
    packed_path -- path to pre-packed .js files including language files and highligh.js itself
    languages -- list of language names to include in the final package.
    '''
    f = open(os.path.join(target_path, 'highlight.pack.js'), 'w')
    f.write(build_content(packed_path, languages))
    f.close()

if __name__ == '__main__':
    path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    packed_path = os.path.join(path, 'packed')
    target_path = os.path.join(path, 'src')
    if len(sys.argv) > 1:
        languages = sys.argv[1:]
    else:
        languages = None
    build(packed_path, target_path, languages)
{'fraction_non_alphanumeric': 0.13311148086522462, 'fraction_numerical': 0.002376990729736154, 'mean_word_length': 3.042267050912584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2828899', 'n_tokens_mistral': 1382, 'n_tokens_neox': 1381, 'n_words': 373}