<commit_before>'''
Module that does most of the heavy lifting for the ``conda build`` command.
'''

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import json
import os
import re
import shutil
import stat
import subprocess
import sys
import tarfile
from io import open
from os.path import exists, isdir, isfile, islink, join
import yaml


import conda.config as cc
import conda.plan as plan
from conda.api import get_index
from conda.compat import PY3
from conda.fetch import fetch_index
from conda.install import prefix_placeholder
from conda.utils import url_path

from conda_build import config, environ, source, tarcheck
from conda_build.scripts import create_entry_points, bin_dirname
from conda_build.post import (post_process, post_build, is_obj,
                              fix_permissions, get_build_metadata)
from conda_build.utils import rm_rf, _check_call
from conda_build.index import update_index
from conda_build.create_test import (create_files, create_shell_files,
                                     create_py_files, create_pl_files)


prefix = config.build_prefix
info_dir = join(prefix, 'info')

broken_dir = join(config.croot, "broken")


def prefix_files():
    '''
    Returns a set of all files in prefix.
    '''
    res = set()
    for root, dirs, files in os.walk(prefix):
        for fn in files:
            res.add(join(root, fn)[len(prefix) + 1:])
        for dn in dirs:
            path = join(root, dn)
            if islink(path):
                res.add(path[len(prefix) + 1:])
    return res


def create_post_scripts(m):
    '''
    Create scripts to run after build step
    '''
    recipe_dir = m.path
    ext = '.bat' if sys.platform == 'win32' else '.sh'
    for tp in 'pre-link', 'post-link', 'pre-unlink':
        src = join(recipe_dir, tp + ext)
        if not isfile(src):
            continue
        dst_dir = join(prefix,
                       'Scripts' if sys.platform == 'win32' else 'bin')
        if not isdir(dst_dir):
            os.makedirs(dst_dir, int('755', 8))
        dst = join(dst_dir, '.%s-%s%s' % (m.name(), tp, ext))
        shutil.copyfile(src, dst)
        os.chmod(dst, int('755', 8))


def have_prefix_files(files):
    '''
    Yields files that contain the current prefix in them, and modifies them
    to replace the prefix with a placeholder.

    :param files: Filenames to check for instances of prefix
    :type files: list of str
    '''
    for f in files:
        if f.endswith(('.pyc', '.pyo', '.a', '.dylib')):
            continue
        path = join(prefix, f)
        if isdir(path):
            continue
        if sys.platform != 'darwin' and islink(path):
            # OSX does not allow hard-linking symbolic links, so we cannot
            # skip symbolic links (as we can on Linux)
            continue
        if sys.platform != 'win32' and is_obj(path):
            continue
        # Open file as binary, since it might have any crazy encoding
        with open(path, 'rb') as fi:
            data = fi.read()
        # Skip files that are truly binary
        if b'\x00' in data:
            continue
        # This may end up mixing encodings, but since paths are usually ASCII,
        # this shouldn't be a problem very often. The only way to completely
        # avoid this would be to use chardet (or cChardet) to detect the
        # encoding on the fly.
        prefix_bytes = prefix.encode('utf-8')
        if prefix_bytes not in data:
            continue
        st = os.stat(path)
        data = data.replace(prefix_bytes, prefix_placeholder.encode('utf-8'))
        # Save as
        with open(path, 'wb') as fo:
            fo.write(data)
        os.chmod(path, stat.S_IMODE(st.st_mode) | stat.S_IWUSR) # chmod u+w
        yield f


def create_info_files(m, files, include_recipe=True):
    '''
    Creates the metadata files that will be stored in the built package.

    :param m: Package metadata
    :type m: Metadata
    :param files: Paths to files to include in package
    :type files: list of str
    :param include_recipe: Whether or not to include the recipe (True by default)
    :type include_recipe: bool
    '''
    recipe_dir = join(info_dir, 'recipe')
    os.makedirs(recipe_dir)

    if include_recipe:
        for fn in os.listdir(m.path):
            if fn.startswith('.'):
                continue
            src_path = join(m.path, fn)
            dst_path = join(recipe_dir, fn)
            if isdir(src_path):
                shutil.copytree(src_path, dst_path)
            else:
                shutil.copy(src_path, dst_path)

    if isfile(join(recipe_dir, 'meta.yaml')):
        shutil.move(join(recipe_dir, 'meta.yaml'),
                    join(recipe_dir, 'meta.yaml.orig'))

    with open(join(recipe_dir, 'meta.yaml'), 'w', encoding='utf-8') as fo:
        yaml.safe_dump(m.meta, fo)

    with open(join(info_dir, 'files'), 'w', encoding='utf-8') as fo:
        for f in files:
            if sys.platform == 'win32':
                f = f.replace('\\', '/')
            fo.write(f + '\n')

    # Deal with Python 2 and 3's different json module type reqs
    mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'}
    with open(join(info_dir, 'index.json'), **mode_dict) as fo:
        json.dump(m.info_index(), fo, indent=2, sort_keys=True)

    with open(join(info_dir, 'recipe.json'), **mode_dict) as fo:
        json.dump(m.meta, fo, indent=2, sort_keys=True)

    files_with_prefix = m.has_prefix_files()
    binary_files_with_prefix = m.binary_has_prefix_files()

    for file in files_with_prefix:
        if file not in files:
            raise RuntimeError("file %s from build/has_prefix_files was "
                               "not found" % file)

    for file in binary_files_with_prefix:
        if file not in files:
            raise RuntimeError("file %s from build/has_prefix_files was "
                               "not found" % file)
        files_with_prefix.append('%s %s %s' % (prefix, 'binary', file))

    files_with_prefix += list(have_prefix_files(files))
    files_with_prefix = sorted(set(files_with_prefix))
    if files_with_prefix:
        with open(join(info_dir, 'has_prefix'), 'w', encoding='utf-8') as fo:
            for f in files_with_prefix:
                fo.write(f + '\n')

    no_link = m.get_value('build/no_link')
    if no_link:
        def w2rx(p):
            return p.replace('.', r'\.').replace('*', r'.*')
        if not isinstance(no_link, list):
            no_link = [no_link]
        rx = '(%s)$' % '|'.join(w2rx(p) for p in no_link)
        pat = re.compile(rx)
        with open(join(info_dir, 'no_link'), 'w', encoding='utf-8') as fo:<commit_msg>Supports unix-style binary_has_prefix paths in windows binaries<commit_after>'''
Module that does most of the heavy lifting for the ``conda build`` command.
'''

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import json
import os
import re
import shutil
import stat
import subprocess
import sys
import tarfile
from io import open
from os.path import exists, isdir, isfile, islink, join
import yaml


import conda.config as cc
import conda.plan as plan
from conda.api import get_index
from conda.compat import PY3
from conda.fetch import fetch_index
from conda.install import prefix_placeholder
from conda.utils import url_path

from conda_build import config, environ, source, tarcheck
from conda_build.scripts import create_entry_points, bin_dirname
from conda_build.post import (post_process, post_build, is_obj,
                              fix_permissions, get_build_metadata)
from conda_build.utils import rm_rf, _check_call
from conda_build.index import update_index
from conda_build.create_test import (create_files, create_shell_files,
                                     create_py_files, create_pl_files)


prefix = config.build_prefix
info_dir = join(prefix, 'info')

broken_dir = join(config.croot, "broken")


def prefix_files():
    '''
    Returns a set of all files in prefix.
    '''
    res = set()
    for root, dirs, files in os.walk(prefix):
        for fn in files:
            res.add(join(root, fn)[len(prefix) + 1:])
        for dn in dirs:
            path = join(root, dn)
            if islink(path):
                res.add(path[len(prefix) + 1:])
    return res


def create_post_scripts(m):
    '''
    Create scripts to run after build step
    '''
    recipe_dir = m.path
    ext = '.bat' if sys.platform == 'win32' else '.sh'
    for tp in 'pre-link', 'post-link', 'pre-unlink':
        src = join(recipe_dir, tp + ext)
        if not isfile(src):
            continue
        dst_dir = join(prefix,
                       'Scripts' if sys.platform == 'win32' else 'bin')
        if not isdir(dst_dir):
            os.makedirs(dst_dir, int('755', 8))
        dst = join(dst_dir, '.%s-%s%s' % (m.name(), tp, ext))
        shutil.copyfile(src, dst)
        os.chmod(dst, int('755', 8))


def have_prefix_files(files):
    '''
    Yields files that contain the current prefix in them, and modifies them
    to replace the prefix with a placeholder.

    :param files: Filenames to check for instances of prefix
    :type files: list of str
    '''
    for f in files:
        if f.endswith(('.pyc', '.pyo', '.a', '.dylib')):
            continue
        path = join(prefix, f)
        if isdir(path):
            continue
        if sys.platform != 'darwin' and islink(path):
            # OSX does not allow hard-linking symbolic links, so we cannot
            # skip symbolic links (as we can on Linux)
            continue
        if sys.platform != 'win32' and is_obj(path):
            continue
        # Open file as binary, since it might have any crazy encoding
        with open(path, 'rb') as fi:
            data = fi.read()
        # Skip files that are truly binary
        if b'\x00' in data:
            continue
        # This may end up mixing encodings, but since paths are usually ASCII,
        # this shouldn't be a problem very often. The only way to completely
        # avoid this would be to use chardet (or cChardet) to detect the
        # encoding on the fly.
        prefix_bytes = prefix.encode('utf-8')
        if prefix_bytes not in data:
            continue
        st = os.stat(path)
        data = data.replace(prefix_bytes, prefix_placeholder.encode('utf-8'))
        # Save as
        with open(path, 'wb') as fo:
            fo.write(data)
        os.chmod(path, stat.S_IMODE(st.st_mode) | stat.S_IWUSR) # chmod u+w
        yield f


def create_info_files(m, files, include_recipe=True):
    '''
    Creates the metadata files that will be stored in the built package.

    :param m: Package metadata
    :type m: Metadata
    :param files: Paths to files to include in package
    :type files: list of str
    :param include_recipe: Whether or not to include the recipe (True by default)
    :type include_recipe: bool
    '''
    recipe_dir = join(info_dir, 'recipe')
    os.makedirs(recipe_dir)

    if include_recipe:
        for fn in os.listdir(m.path):
            if fn.startswith('.'):
                continue
            src_path = join(m.path, fn)
            dst_path = join(recipe_dir, fn)
            if isdir(src_path):
                shutil.copytree(src_path, dst_path)
            else:
                shutil.copy(src_path, dst_path)

    if isfile(join(recipe_dir, 'meta.yaml')):
        shutil.move(join(recipe_dir, 'meta.yaml'),
                    join(recipe_dir, 'meta.yaml.orig'))

    with open(join(recipe_dir, 'meta.yaml'), 'w', encoding='utf-8') as fo:
        yaml.safe_dump(m.meta, fo)

    with open(join(info_dir, 'files'), 'w', encoding='utf-8') as fo:
        for f in files:
            if sys.platform == 'win32':
                f = f.replace('\\', '/')
            fo.write(f + '\n')

    # Deal with Python 2 and 3's different json module type reqs
    mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'}
    with open(join(info_dir, 'index.json'), **mode_dict) as fo:
        json.dump(m.info_index(), fo, indent=2, sort_keys=True)

    with open(join(info_dir, 'recipe.json'), **mode_dict) as fo:
        json.dump(m.meta, fo, indent=2, sort_keys=True)

    files_with_prefix = m.has_prefix_files()
    binary_files_with_prefix = m.binary_has_prefix_files()

    for file in files_with_prefix:
        if file not in files:
            raise RuntimeError("file %s from build/has_prefix_files was "
                               "not found" % file)

    for file in binary_files_with_prefix:
        if file not in files:
            raise RuntimeError("file %s from build/has_prefix_files was "
                               "not found" % file)
        with open(os.path.join(prefix, file), 'rb') as f:
            data = f.read()
        if prefix.encode('utf-8') in data:
            files_with_prefix.append('%s %s %s' % (prefix, 'binary', file))
        elif sys.platform == 'win32':
            # some windows libraries encode prefix with unix path separators
            alt_p = prefix.replace('\\', '/')
            if alt_p.encode('utf-8') in data:
                files_with_prefix.append('%s %s %s' % (alt_p, 'binary', file))
            else:
                print('Warning: prefix %s not found in %s' % (prefix, file))
        else:
            print('Warning: prefix %s not found in %s' % (prefix, file)

    files_with_prefix += list(have_prefix_files(files))
    files_with_prefix = sorted(set(files_with_prefix))
    if files_with_prefix:
        with open(join(info_dir, 'has_prefix'), 'w', encoding='utf-8') as fo:
            for f in files_with_prefix:
                fo.write(f + '\n')

    no_link = m.get_value('build/no_link')
    if no_link:
        def w2rx(p):
            return p.replace('.', r'\.').replace('*', r'.*')
        if not isinstance(no_link, list):
            no_link = [no_link]
        rx = '(%s)$' % '|'.join(w2rx(p) for p in no_link)
        pat = re.compile(rx)
        with open(join(info_dir, 'no_link'), 'w', encoding='utf-8') as fo:
{'fraction_non_alphanumeric': 0.1109144960362401, 'fraction_numerical': 0.005237825594563986, 'mean_word_length': 2.3205640423031726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2088856', 'n_tokens_mistral': 4485, 'n_tokens_neox': 4380, 'n_words': 1355}