<commit_before># -*- coding:utf-8 -*-
'''
Function for building whole packed version of highlight.js out of
pre-packed modules.
'''

import os
import sys

ALIASES = {
    'html': ['html-xml'],
    'xml': ['html-xml'],
    'css': ['html-xml', 'css'],
    'django': ['html-xml', 'django'],
}

def language_files(packed_path):
    '''
    Returns a list of all language .js files from packed_path.
    '''
    files = os.listdir(packed_path)
    files = [l for l in files if l.endswith('.js') and not l.startswith('highlight.')]
    return files

def language_names(packed_path):
    '''
    Returns a list of all available language names converting necessary filenames with ALIASES
    '''
    aliased_filenames = set()
    for l in ALIASES.values():
        aliased_filenames |= set(l)
    aliased_languages = ALIASES.keys()
    names = [os.path.splitext(f)[0] for f in language_files(packed_path)]
    return [n for n in names if n not in aliased_filenames or n in aliased_languages]

def build_content(packed_path, languages):
    '''
    Builds content of highlight.pack.js and returns it as a  string.
    '''
    all_files = language_files(packed_path)
    if languages is not None:
        selected_languages = []
        for l in languages:
            selected_languages += ALIASES.get(l, [l])
        all_languages = set(os.path.splitext(l)[0] for l in all_files)
        languages = all_languages & set(selected_languages)
        files = ['%s.js' % l for l in languages]
    else:
        files = all_files
    contents = [open(os.path.join(packed_path, f)).read() for f in files]
    contents.insert(0, open(os.path.join(packed_path, 'highlight.js')).read())
    return ''.join(contents)
    
def build(packed_path, target_path, languages):
    '''<commit_msg>Ручное задание порядка файлов в build.py<commit_after># -*- coding:utf-8 -*-
'''
Function for building whole packed version of highlight.js out of
pre-packed modules.
'''

import os
import sys

ALIASES = {
    'html': ['html-xml'],
    'xml': ['html-xml'],
    'css': ['html-xml', 'css'],
    'django': ['html-xml', 'django'],
}

FILES_ORDER = {
    'html-xml.js': 0,
    'css.js': 1,
    'django.js': 1,
}

def language_files(packed_path):
    '''
    Returns a list of all language .js files from packed_path.
    '''
    files = os.listdir(packed_path)
    files = [l for l in files if l.endswith('.js') and not l.startswith('highlight.')]
    return files

def language_names(packed_path):
    '''
    Returns a list of all available language names converting necessary filenames with ALIASES
    '''
    aliased_filenames = set()
    for l in ALIASES.values():
        aliased_filenames |= set(l)
    aliased_languages = ALIASES.keys()
    names = [os.path.splitext(f)[0] for f in language_files(packed_path)]
    return [n for n in names if n not in aliased_filenames or n in aliased_languages]

def build_content(packed_path, languages):
    '''
    Builds content of highlight.pack.js and returns it as a  string.
    '''
    all_files = language_files(packed_path)
    if languages is not None:
        selected_languages = []
        for l in languages:
            selected_languages += ALIASES.get(l, [l])
        all_languages = set(os.path.splitext(l)[0] for l in all_files)
        languages = all_languages & set(selected_languages)
        files = ['%s.js' % l for l in languages]
    else:
        files = all_files
    ordered_files = [(FILES_ORDER.get(f, 0), f) for f in files]
    ordered_files.sort()
    files = [f for o, f in ordered_files]
    contents = [open(os.path.join(packed_path, f)).read() for f in files]
    contents.insert(0, open(os.path.join(packed_path, 'highlight.js')).read())
    return ''.join(contents)
    
def build(packed_path, target_path, languages):
    '''
{'fraction_non_alphanumeric': 0.1304923239809423, 'fraction_numerical': 0.0031762837480148226, 'mean_word_length': 3.1255458515283845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3046313', 'n_tokens_mistral': 1263, 'n_tokens_neox': 1252, 'n_words': 358}