<commit_before>from __future__ import print_function, division, absolute_import

import re
import os
import sys
import json
import stat
import shutil
import tarfile
from os.path import exists, isdir, isfile, islink, join, abspath
import subprocess

import conda.config as cc
import conda.plan as plan
from conda.utils import url_path
from conda.api import get_index
from conda.install import prefix_placeholder
from conda.fetch import fetch_index

from conda_build import config
from conda_build import environ
from conda_build import source
from conda_build import tarcheck
from conda_build.scripts import create_entry_points, bin_dirname
from conda_build.post import (post_process, post_build, is_obj,
                              fix_permissions)
from conda_build.utils import rm_rf, _check_call
from conda_build.index import update_index
from conda_build.create_test import (create_files, create_shell_files,
                                     create_py_files)
from conda_build.metadata import MetaData


prefix = config.build_prefix
info_dir = join(prefix, 'info')

broken_dir = join(config.croot, "broken")


def prefix_files():
    res = set()
    for root, dirs, files in os.walk(prefix):
        for fn in files:
            res.add(join(root, fn)[len(prefix) + 1:])
        for dn in dirs:
            path = join(root, dn)
            if islink(path):
                res.add(path[len(prefix) + 1:])
    return res


def create_post_scripts(m):
    recipe_dir = m.path
    ext = '.bat' if sys.platform == 'win32' else '.sh'
    for tp in 'pre-link', 'post-link', 'pre-unlink':
        src = join(recipe_dir, tp + ext)
        if not isfile(src):
            continue
        dst = join(prefix,
                   'Scripts' if sys.platform == 'win32' else 'bin',
                   '.%s-%s%s' % (m.name(), tp, ext))
        shutil.copyfile(src, dst)
        os.chmod(dst, int('755', 8))


def have_prefix_files(files):
    for f in files:
        if f.endswith(('.pyc', '.pyo', '.a', '.dylib')):
            continue
        path = join(prefix, f)
        if isdir(path):
            continue
        if sys.platform != 'darwin' and islink(path):
            # OSX does not allow hard-linking symbolic links, so we cannot
            # skip symbolic links (as we can on Linux)
            continue
        if is_obj(path):
            continue
        if islink(path):
            continue
        try:
            with open(path) as fi:
                data = fi.read()
        except UnicodeDecodeError:
            continue
        if '\x00' in data:
            continue
        if prefix not in data:
            continue
        st = os.stat(path)
        data = data.replace(prefix, prefix_placeholder)
        with open(path, 'w') as fo:
            fo.write(data)
        os.chmod(path, stat.S_IMODE(st.st_mode) | stat.S_IWUSR) # chmod u+w
        yield f


def create_info_files(m, files):
    recipe_dir = join(info_dir, 'recipe')
    os.makedirs(recipe_dir)

    for fn in os.listdir(m.path):
        if fn.startswith('.'):
            continue
        src_path = join(m.path, fn)
        dst_path = join(recipe_dir, fn)
        if isdir(src_path):
            shutil.copytree(src_path, dst_path)
        else:
            shutil.copy(src_path, dst_path)

    with open(join(info_dir, 'files'), 'w') as fo:
        for f in files:
            if sys.platform == 'win32':
                f = f.replace('\\', '/')
            fo.write(f + '\n')

    with open(join(info_dir, 'index.json'), 'w') as fo:
        json.dump(m.info_index(), fo, indent=2, sort_keys=True)

    with open(join(info_dir, 'recipe.json'), 'w') as fo:
        json.dump(m.meta, fo, indent=2, sort_keys=True)

    if sys.platform != 'win32':
        prefix_files = list(have_prefix_files(files))
        if prefix_files:
            with open(join(info_dir, 'has_prefix'), 'w') as fo:
                for f in prefix_files:
                    fo.write(f + '\n')

    no_soft_rx = m.get_value('build/no_softlink')
    if no_soft_rx:
        pat = re.compile(no_soft_rx)
        with open(join(info_dir, 'no_softlink'), 'w') as fo:
            for f in files:
                if pat.match(f):
                    fo.write(f + '\n')

    if m.get_value('source/git_url'):
        with open(join(info_dir, 'git'), 'w') as fo:
            source.git_info(fo)

    if m.get_value('app/icon'):
        shutil.copyfile(join(m.path, m.get_value('app/icon')),
                        join(info_dir, 'icon.png'))


def create_env(pref, specs):
    if not isdir(config.bldpkgs_dir):
        os.makedirs(config.bldpkgs_dir)
    update_index(config.bldpkgs_dir)
    # remove the cache such that a refetch is made,
    # this is necessary because we add the local build repo URL
    fetch_index.cache = {}
    index = get_index([url_path(config.croot)])

    cc.pkgs_dirs = cc.pkgs_dirs[:1]
    actions = plan.install_actions(pref, index, specs)
    plan.display_actions(actions, index)
    plan.execute_actions(actions, index, verbose=True)
    # ensure prefix exists, even if empty, i.e. when specs are empty
    if not isdir(pref):
        os.makedirs(pref)

def rm_pkgs_cache(dist):
    cc.pkgs_dirs = cc.pkgs_dirs[:1]
    rmplan = ['RM_FETCHED %s' % dist,
              'RM_EXTRACTED %s' % dist]
    plan.execute_plan(rmplan)

def bldpkg_path(m):
    return join(config.bldpkgs_dir, '%s.tar.bz2' % m.dist())


def build(m, get_src=True):
    rm_rf(prefix)
    try_again = True
    while try_again:
        try:
            create_env(prefix, [ms.spec for ms in m.ms_depends('build')])
        except RuntimeError as e:
            error_str = str(e)
            if error_str.startswith('No packages found matching:'):
                # Build dependency if recipe exists
                recipe_dir = error_str.split(': ')[1]
                if exists(recipe_dir):
                    print(("Missing dependency {0}, but found recipe " +
                           "directory, so building " +
                           "{0} first").format(recipe_dir))
                    dep_m = MetaData(os.path.abspath(recipe_dir))
                    dep_m.check_fields()
                    build(dep_m)
                    # Now try again
                    try_again = True
                else:
                    raise
            else:
                raise
        else:
            try_again = False<commit_msg>Drop os.path prefix from os.path.abspath, since it's imported.<commit_after>from __future__ import print_function, division, absolute_import

import re
import os
import sys
import json
import stat
import shutil
import tarfile
from os.path import exists, isdir, isfile, islink, join, abspath
import subprocess

import conda.config as cc
import conda.plan as plan
from conda.utils import url_path
from conda.api import get_index
from conda.install import prefix_placeholder
from conda.fetch import fetch_index

from conda_build import config
from conda_build import environ
from conda_build import source
from conda_build import tarcheck
from conda_build.scripts import create_entry_points, bin_dirname
from conda_build.post import (post_process, post_build, is_obj,
                              fix_permissions)
from conda_build.utils import rm_rf, _check_call
from conda_build.index import update_index
from conda_build.create_test import (create_files, create_shell_files,
                                     create_py_files)
from conda_build.metadata import MetaData


prefix = config.build_prefix
info_dir = join(prefix, 'info')

broken_dir = join(config.croot, "broken")


def prefix_files():
    res = set()
    for root, dirs, files in os.walk(prefix):
        for fn in files:
            res.add(join(root, fn)[len(prefix) + 1:])
        for dn in dirs:
            path = join(root, dn)
            if islink(path):
                res.add(path[len(prefix) + 1:])
    return res


def create_post_scripts(m):
    recipe_dir = m.path
    ext = '.bat' if sys.platform == 'win32' else '.sh'
    for tp in 'pre-link', 'post-link', 'pre-unlink':
        src = join(recipe_dir, tp + ext)
        if not isfile(src):
            continue
        dst = join(prefix,
                   'Scripts' if sys.platform == 'win32' else 'bin',
                   '.%s-%s%s' % (m.name(), tp, ext))
        shutil.copyfile(src, dst)
        os.chmod(dst, int('755', 8))


def have_prefix_files(files):
    for f in files:
        if f.endswith(('.pyc', '.pyo', '.a', '.dylib')):
            continue
        path = join(prefix, f)
        if isdir(path):
            continue
        if sys.platform != 'darwin' and islink(path):
            # OSX does not allow hard-linking symbolic links, so we cannot
            # skip symbolic links (as we can on Linux)
            continue
        if is_obj(path):
            continue
        if islink(path):
            continue
        try:
            with open(path) as fi:
                data = fi.read()
        except UnicodeDecodeError:
            continue
        if '\x00' in data:
            continue
        if prefix not in data:
            continue
        st = os.stat(path)
        data = data.replace(prefix, prefix_placeholder)
        with open(path, 'w') as fo:
            fo.write(data)
        os.chmod(path, stat.S_IMODE(st.st_mode) | stat.S_IWUSR) # chmod u+w
        yield f


def create_info_files(m, files):
    recipe_dir = join(info_dir, 'recipe')
    os.makedirs(recipe_dir)

    for fn in os.listdir(m.path):
        if fn.startswith('.'):
            continue
        src_path = join(m.path, fn)
        dst_path = join(recipe_dir, fn)
        if isdir(src_path):
            shutil.copytree(src_path, dst_path)
        else:
            shutil.copy(src_path, dst_path)

    with open(join(info_dir, 'files'), 'w') as fo:
        for f in files:
            if sys.platform == 'win32':
                f = f.replace('\\', '/')
            fo.write(f + '\n')

    with open(join(info_dir, 'index.json'), 'w') as fo:
        json.dump(m.info_index(), fo, indent=2, sort_keys=True)

    with open(join(info_dir, 'recipe.json'), 'w') as fo:
        json.dump(m.meta, fo, indent=2, sort_keys=True)

    if sys.platform != 'win32':
        prefix_files = list(have_prefix_files(files))
        if prefix_files:
            with open(join(info_dir, 'has_prefix'), 'w') as fo:
                for f in prefix_files:
                    fo.write(f + '\n')

    no_soft_rx = m.get_value('build/no_softlink')
    if no_soft_rx:
        pat = re.compile(no_soft_rx)
        with open(join(info_dir, 'no_softlink'), 'w') as fo:
            for f in files:
                if pat.match(f):
                    fo.write(f + '\n')

    if m.get_value('source/git_url'):
        with open(join(info_dir, 'git'), 'w') as fo:
            source.git_info(fo)

    if m.get_value('app/icon'):
        shutil.copyfile(join(m.path, m.get_value('app/icon')),
                        join(info_dir, 'icon.png'))


def create_env(pref, specs):
    if not isdir(config.bldpkgs_dir):
        os.makedirs(config.bldpkgs_dir)
    update_index(config.bldpkgs_dir)
    # remove the cache such that a refetch is made,
    # this is necessary because we add the local build repo URL
    fetch_index.cache = {}
    index = get_index([url_path(config.croot)])

    cc.pkgs_dirs = cc.pkgs_dirs[:1]
    actions = plan.install_actions(pref, index, specs)
    plan.display_actions(actions, index)
    plan.execute_actions(actions, index, verbose=True)
    # ensure prefix exists, even if empty, i.e. when specs are empty
    if not isdir(pref):
        os.makedirs(pref)

def rm_pkgs_cache(dist):
    cc.pkgs_dirs = cc.pkgs_dirs[:1]
    rmplan = ['RM_FETCHED %s' % dist,
              'RM_EXTRACTED %s' % dist]
    plan.execute_plan(rmplan)

def bldpkg_path(m):
    return join(config.bldpkgs_dir, '%s.tar.bz2' % m.dist())


def build(m, get_src=True):
    rm_rf(prefix)
    try_again = True
    while try_again:
        try:
            create_env(prefix, [ms.spec for ms in m.ms_depends('build')])
        except RuntimeError as e:
            error_str = str(e)
            if error_str.startswith('No packages found matching:'):
                # Build dependency if recipe exists
                recipe_dir = error_str.split(': ')[1]
                if exists(recipe_dir):
                    print(("Missing dependency {0}, but found recipe " +
                           "directory, so building " +
                           "{0} first").format(recipe_dir))
                    dep_m = MetaData(abspath(recipe_dir))
                    dep_m.check_fields()
                    build(dep_m)
                    # Now try again
                    try_again = True
                else:
                    raise
            else:
                raise
        else:
            try_again = False
{'fraction_non_alphanumeric': 0.10853801169590643, 'fraction_numerical': 0.0037426900584795323, 'mean_word_length': 2.0581783500238435, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5589873', 'n_tokens_mistral': 4163, 'n_tokens_neox': 4053, 'n_words': 1058}