<commit_before>"""Build CPython from source."""

from collections import namedtuple
from pathlib import Path
import logging

from foreman import define_parameter, get_relpath, rule

from garage import scripts

from templates import apps
from templates.common import define_archive, define_distro_packages
from templates.utils import tapeout_files


LOG = logging.getLogger(__name__)


### CPython build parameters


Version = namedtuple('Version', 'major minor')
(define_parameter.namedtuple_typed(Version, 'version')
 .with_doc('CPython version.')
 .with_default(Version('3', '6')))


define_archive(
    uri='https://www.python.org/ftp/python/3.6.1/Python-3.6.1rc1.tar.xz',
    filename='Python-3.6.1rc1.tar.xz',
    output='Python-3.6.1rc1',
    checksum='md5-5919c290d3727d81c3472e6c46fd78b6',
)


define_parameter.path_typed('prefix').with_default(Path('/usr/local'))


define_parameter.list_typed('configuration').with_default([
    '--enable-ipv6',
    '--enable-loadable-sqlite-extensions',
    '--with-computed-gotos',
    '--with-dbmliborder=bdb:gdbm',
    '--with-fpectl',
    '--with-system-expat',
    '--with-system-ffi',
    '--with-system-libmpdec',
])


### Dependencies


# This list is derived from running `apt-get build-dep python3.6` on
# Ubuntu 16.10 (not all packages are included)
define_distro_packages([
    # Build tool
    'autoconf',
    'automake',
    'autotools-dev',
    'binutils',
    'cpp',
    'g++',
    'gcc',
    'libtool',
    'm4',
    'make',
    'pkg-config',
    # Libraries
    'libatomic1',
    'libc6-dev',
    'libmpc3',
    'libmpdec-dev',
    'libmpfr4',
    'libmpx2',
    'libsigsegv2',
    # Compression
    'libbz2-dev',
    'liblzma-dev',
    'zlib1g-dev',
    # Console
    'libncursesw5-dev',
    'libreadline-dev',
    # Database
    'libdb-dev',
    'libgdbm-dev',
    'libsqlite3-dev',
    # FFI
    'libffi-dev',
    # Network
    'libssl-dev',
    # XML
    'libexpat1-dev',
    'libxml2',  # Do we really need this?
])


### Build artifacts


(define_parameter.path_typed('python')
 .with_doc('Path to CPython interpreter.')
 .with_derive(lambda ps: ps['prefix'] / ('bin/python%s.%s' % ps['version'])))


(define_parameter.path_typed('pip')
 .with_doc('Path to pip.')
 .with_derive(lambda ps: ps['prefix'] / ('bin/pip%s.%s' % ps['version'])))


(define_parameter.path_typed('modules')
 .with_doc('Path to site-packages directory.')
 .with_derive(lambda ps: ps['prefix'] / ('lib/python%s.%s' % ps['version'])))


### Build rules


@rule
@rule.depend('//base:build')
@rule.depend('install_packages')
@rule.depend('download')
def build(parameters):
    """Build CPython from source."""
    drydock_src = (parameters['//base:drydock'] / get_relpath() /
                   parameters['archive_info'].output)
    with scripts.directory(drydock_src):

        if not (drydock_src / 'Makefile').exists():
            LOG.info('configure cpython build')
            cmd = ['./configure', '--prefix', parameters['prefix']]
            cmd.extend(parameters['configuration'])
            if parameters['//base:release']:
                cmd.extend(['--enable-optimizations', '--with-lto'])
            scripts.execute(cmd)

        if not (drydock_src / 'python').exists():
            LOG.info('build cpython')
            scripts.execute(['make'])

            LOG.info('install cpython')
            with scripts.using_sudo():
                # (Probably a bug?) When optimizations are enabled, this
                # will re-run `make run_profile_task`
                scripts.execute(['make', 'install'])


@rule
@rule.depend('build')
@rule.reverse_depend('//base:tapeout')
def tapeout(parameters):
    """Tape-out CPython.

       NOTE: All Python module's `tapeout` rules should reverse depend
       on this rule.
    """

    LOG.info('tapeout cpython modules')
    modules = parameters['modules']
    excludes = [
        # Exclude idlelib, lib2to3, tkinter, and turtledemo<commit_msg>Symlink header directory from pythonX.Y to pythonX.Ym<commit_after>"""Build CPython from source."""

from collections import namedtuple
from pathlib import Path
import logging

from foreman import define_parameter, get_relpath, rule

from garage import scripts

from templates import apps
from templates.common import define_archive, define_distro_packages
from templates.utils import tapeout_files


LOG = logging.getLogger(__name__)


### CPython build parameters


Version = namedtuple('Version', 'major minor')
(define_parameter.namedtuple_typed(Version, 'version')
 .with_doc('CPython version.')
 .with_default(Version('3', '6')))


define_archive(
    uri='https://www.python.org/ftp/python/3.6.1/Python-3.6.1rc1.tar.xz',
    filename='Python-3.6.1rc1.tar.xz',
    output='Python-3.6.1rc1',
    checksum='md5-5919c290d3727d81c3472e6c46fd78b6',
)


define_parameter.path_typed('prefix').with_default(Path('/usr/local'))


define_parameter.list_typed('configuration').with_default([
    '--enable-ipv6',
    '--enable-loadable-sqlite-extensions',
    '--with-computed-gotos',
    '--with-dbmliborder=bdb:gdbm',
    '--with-fpectl',
    '--with-system-expat',
    '--with-system-ffi',
    '--with-system-libmpdec',
])


### Dependencies


# This list is derived from running `apt-get build-dep python3.6` on
# Ubuntu 16.10 (not all packages are included)
define_distro_packages([
    # Build tool
    'autoconf',
    'automake',
    'autotools-dev',
    'binutils',
    'cpp',
    'g++',
    'gcc',
    'libtool',
    'm4',
    'make',
    'pkg-config',
    # Libraries
    'libatomic1',
    'libc6-dev',
    'libmpc3',
    'libmpdec-dev',
    'libmpfr4',
    'libmpx2',
    'libsigsegv2',
    # Compression
    'libbz2-dev',
    'liblzma-dev',
    'zlib1g-dev',
    # Console
    'libncursesw5-dev',
    'libreadline-dev',
    # Database
    'libdb-dev',
    'libgdbm-dev',
    'libsqlite3-dev',
    # FFI
    'libffi-dev',
    # Network
    'libssl-dev',
    # XML
    'libexpat1-dev',
    'libxml2',  # Do we really need this?
])


### Build artifacts


(define_parameter.path_typed('python')
 .with_doc('Path to CPython interpreter.')
 .with_derive(lambda ps: ps['prefix'] / ('bin/python%s.%s' % ps['version'])))


(define_parameter.path_typed('pip')
 .with_doc('Path to pip.')
 .with_derive(lambda ps: ps['prefix'] / ('bin/pip%s.%s' % ps['version'])))


(define_parameter.path_typed('modules')
 .with_doc('Path to site-packages directory.')
 .with_derive(lambda ps: ps['prefix'] / ('lib/python%s.%s' % ps['version'])))


### Build rules


@rule
@rule.depend('//base:build')
@rule.depend('install_packages')
@rule.depend('download')
def build(parameters):
    """Build CPython from source."""
    drydock_src = (parameters['//base:drydock'] / get_relpath() /
                   parameters['archive_info'].output)
    with scripts.directory(drydock_src):

        if not (drydock_src / 'Makefile').exists():
            LOG.info('configure cpython build')
            cmd = ['./configure', '--prefix', parameters['prefix']]
            cmd.extend(parameters['configuration'])
            if parameters['//base:release']:
                cmd.extend(['--enable-optimizations', '--with-lto'])
            scripts.execute(cmd)

        if not (drydock_src / 'python').exists():
            LOG.info('build cpython')
            scripts.execute(['make'])

            LOG.info('install cpython')
            with scripts.using_sudo():
                # (Probably a bug?) When optimizations are enabled, this
                # will re-run `make run_profile_task`
                scripts.execute(['make', 'install'])

    # Custom-built Python sometimes creates "pythonX.Ym" rather than
    # "pythonX.Y" header directory
    header_dir = (
        parameters['prefix'] / 'include' /
        ('python%s.%s' % parameters['version'])
    )
    if not header_dir.exists():
        alt_header_dir = header_dir.with_name(
            'python%s.%sm' % parameters['version'])
        scripts.ensure_directory(alt_header_dir)
        LOG.info('symlink cpython headers')
        with scripts.using_sudo():
            scripts.symlink(alt_header_dir.name, header_dir)


@rule
@rule.depend('build')
@rule.reverse_depend('//base:tapeout')
def tapeout(parameters):
    """Tape-out CPython.

       NOTE: All Python module's `tapeout` rules should reverse depend
       on this rule.
    """

    LOG.info('tapeout cpython modules')
    modules = parameters['modules']
    excludes = [
        # Exclude idlelib, lib2to3, tkinter, and turtledemo
{'fraction_non_alphanumeric': 0.15909895034791838, 'fraction_numerical': 0.014860242953178442, 'mean_word_length': 3.108527131782946, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2536568', 'n_tokens_mistral': 2947, 'n_tokens_neox': 2804, 'n_words': 539}