<commit_before>#!/usr/bin/env python
"""Pex building script for Please."""

import argparse
import contextlib
import json
import os
import pkg_resources
import py_compile
import shutil
import sys
import tempfile
import zipfile

from third_party.python.pex.compatibility import to_bytes
from third_party.python.pex.pex_builder import PEXBuilder
from third_party.python.pex.interpreter import PythonInterpreter


def dereference_symlinks(src):
    """
    Resolve all symbolic references that `src` points to.  Note that this
    is different than `os.path.realpath` as path components leading up to
    the final location may still be symbolic links.
    """
    while os.path.islink(src):
        src = os.path.join(os.path.dirname(src), os.readlink(src))
    return src


def write_file(f, contents):
    """Writes contents to given file object.

    TODO(pebers):  This is mostly an effort to get around various encoding issues. Would be
                   nice to solve this in a less brutal way but not sure how best to resolve
                   python2 / python3 / pkg_resources :(
    """
    try:
        f.write(contents)
    except (UnicodeDecodeError, UnicodeEncodeError, TypeError):
        try:
            f.write(contents.decode('utf-8'))
        except (UnicodeDecodeError, UnicodeEncodeError):
            f.write(contents.decode('latin_1'))  # This is desperate peasantry of course.


def add_directory(root_path, path, pex_builder):
    """Recursively adds the contents of a directory to the pex."""
    base = os.path.join(root_path, path)
    for filename in os.listdir(base):
        src = dereference_symlinks(os.path.join(base, filename))
        dst = os.path.join(path, filename)
        if os.path.isdir(src):
            add_directory(root_path, dst, pex_builder)
        elif filename.endswith('.py'):
            # Assume this is source code and add as such.
            # TODO(pebers): this is possibly a bit simplistic...
            pex_builder.add_source(src, dst)
        elif not filename.endswith('.pyc'):
            pex_builder.add_resource(src, dst)


def add_test_files(test_sources, out_dir):
    """Add files required for running a test pex."""
    contents = pkg_resources.resource_string('src.build.python', 'test_main.py').decode('utf-8')
    remove_ext = lambda x: x[:-3] if x.endswith('.py') else x
    test_modules = ','.join(remove_ext(src.replace('/', '.')) for src in test_sources.split(','))
    contents = contents.replace('__TEST_NAMES__', test_modules)
    with open(os.path.join(out_dir, 'test_main.py'), 'w') as f:
        write_file(f, contents)
    # Extract the xmlrunner and coverage files too
    extract_directory('third_party.python', 'xmlrunner', out_dir, '.bootstrap/xmlrunner', True)
    extract_file('third_party.python', os.path.join(out_dir, '.bootstrap'), 'six.py', True)
    extract_directory('third_party.python', 'coverage', out_dir, '.bootstrap/coverage', True)


def add_main(module_dir, entry_point, out_dir):
    """Add pex_main.py as the entry point to a pex."""
    contents = pkg_resources.resource_string('src.build.python', 'pex_main.py').decode('utf-8')
    contents = contents.replace('__MODULE_DIR__', module_dir).replace('__ENTRY_POINT__', entry_point)
    with open(os.path.join(out_dir, 'pex_main.py'), 'w') as f:
        write_file(f, contents)
    return 'pex_main'


def extract_directory(in_pkg, in_req, out_dir, out_path, duplicates_allowed=False):
    """Extracts a directory from the pex builder and adds it to the staging directory."""
    target_dir = os.path.join(out_dir, out_path)
    if os.path.exists(target_dir) and duplicates_allowed:
        return
    os.makedirs(target_dir)
    full_dir = '%s.%s' % (in_pkg, in_req)
    for filename in pkg_resources.resource_listdir(in_pkg, in_req):
        if not pkg_resources.resource_isdir(full_dir, filename):
            extract_file(full_dir, target_dir, filename)


def extract_file(in_pkg, out_dir, filename, duplicates_allowed=False):
    """Extracts a single file from the pex builder and adds it to the staging directory."""
    target = os.path.join(out_dir, filename)
    if os.path.exists(target) and duplicates_allowed:
        return
    with open(target, 'w') as f:
        write_file(f, pkg_resources.resource_string(in_pkg, filename))


def compile_bytecode():
    """Walks the temp dir and precompiles bytecode for all .py files there."""
    for dirpath, dirnames, filenames in os.walk('.'):
        for filename in filenames:
            if filename.endswith('.py'):
                filename = os.path.join(dirpath, filename)
                py_compile.compile(filename, doraise=True)
                # Overwrite the timestamp in the .pyc file with 2000-01-01 so it's deterministic.
                extension = 'o' if sys.flags.optimize else 'c'
                with open(filename + extension, 'r+b') as f:
                    f.seek(4)
                    f.write(b'\\x80Cm8')


def main(args):
    # If --compile is given, we just need to precompile bytecode.
    if args.compile:
        compile_bytecode()
        sys.exit(0)
<commit_msg>python3 fix for rewriting .pyc files when they're in a different place.<commit_after>#!/usr/bin/env python
"""Pex building script for Please."""

import argparse
import contextlib
import json
import os
import pkg_resources
import py_compile
import shutil
import sys
import tempfile
import zipfile

from third_party.python.pex.compatibility import to_bytes
from third_party.python.pex.pex_builder import PEXBuilder
from third_party.python.pex.interpreter import PythonInterpreter


def dereference_symlinks(src):
    """
    Resolve all symbolic references that `src` points to.  Note that this
    is different than `os.path.realpath` as path components leading up to
    the final location may still be symbolic links.
    """
    while os.path.islink(src):
        src = os.path.join(os.path.dirname(src), os.readlink(src))
    return src


def write_file(f, contents):
    """Writes contents to given file object.

    TODO(pebers):  This is mostly an effort to get around various encoding issues. Would be
                   nice to solve this in a less brutal way but not sure how best to resolve
                   python2 / python3 / pkg_resources :(
    """
    try:
        f.write(contents)
    except (UnicodeDecodeError, UnicodeEncodeError, TypeError):
        try:
            f.write(contents.decode('utf-8'))
        except (UnicodeDecodeError, UnicodeEncodeError):
            f.write(contents.decode('latin_1'))  # This is desperate peasantry of course.


def add_directory(root_path, path, pex_builder):
    """Recursively adds the contents of a directory to the pex."""
    base = os.path.join(root_path, path)
    for filename in os.listdir(base):
        src = dereference_symlinks(os.path.join(base, filename))
        dst = os.path.join(path, filename)
        if os.path.isdir(src):
            add_directory(root_path, dst, pex_builder)
        elif filename.endswith('.py'):
            # Assume this is source code and add as such.
            # TODO(pebers): this is possibly a bit simplistic...
            pex_builder.add_source(src, dst)
        elif not filename.endswith('.pyc'):
            pex_builder.add_resource(src, dst)


def add_test_files(test_sources, out_dir):
    """Add files required for running a test pex."""
    contents = pkg_resources.resource_string('src.build.python', 'test_main.py').decode('utf-8')
    remove_ext = lambda x: x[:-3] if x.endswith('.py') else x
    test_modules = ','.join(remove_ext(src.replace('/', '.')) for src in test_sources.split(','))
    contents = contents.replace('__TEST_NAMES__', test_modules)
    with open(os.path.join(out_dir, 'test_main.py'), 'w') as f:
        write_file(f, contents)
    # Extract the xmlrunner and coverage files too
    extract_directory('third_party.python', 'xmlrunner', out_dir, '.bootstrap/xmlrunner', True)
    extract_file('third_party.python', os.path.join(out_dir, '.bootstrap'), 'six.py', True)
    extract_directory('third_party.python', 'coverage', out_dir, '.bootstrap/coverage', True)


def add_main(module_dir, entry_point, out_dir):
    """Add pex_main.py as the entry point to a pex."""
    contents = pkg_resources.resource_string('src.build.python', 'pex_main.py').decode('utf-8')
    contents = contents.replace('__MODULE_DIR__', module_dir).replace('__ENTRY_POINT__', entry_point)
    with open(os.path.join(out_dir, 'pex_main.py'), 'w') as f:
        write_file(f, contents)
    return 'pex_main'


def extract_directory(in_pkg, in_req, out_dir, out_path, duplicates_allowed=False):
    """Extracts a directory from the pex builder and adds it to the staging directory."""
    target_dir = os.path.join(out_dir, out_path)
    if os.path.exists(target_dir) and duplicates_allowed:
        return
    os.makedirs(target_dir)
    full_dir = '%s.%s' % (in_pkg, in_req)
    for filename in pkg_resources.resource_listdir(in_pkg, in_req):
        if not pkg_resources.resource_isdir(full_dir, filename):
            extract_file(full_dir, target_dir, filename)


def extract_file(in_pkg, out_dir, filename, duplicates_allowed=False):
    """Extracts a single file from the pex builder and adds it to the staging directory."""
    target = os.path.join(out_dir, filename)
    if os.path.exists(target) and duplicates_allowed:
        return
    with open(target, 'w') as f:
        write_file(f, pkg_resources.resource_string(in_pkg, filename))


def compile_bytecode():
    """Walks the temp dir and precompiles bytecode for all .py files there."""
    for dirpath, dirnames, filenames in os.walk('.'):
        for filename in filenames:
            if filename.endswith('.py'):
                filename = os.path.join(dirpath, filename)
                path = py_compile.compile(filename, doraise=True)
                if not path:
                    # In python3 we already have a path to the .pyc file. py_compile in python2
                    # does not return anything so we have to work it out ourselves.
                    path = filename + ('o' if sys.flags.optimize else 'c')
                # Overwrite the timestamp in the .pyc file with 2000-01-01 so it's deterministic.
                with open(path, 'r+b') as f:
                    f.seek(4)
                    f.write(b'\\x80Cm8')


def main(args):
    # If --compile is given, we just need to precompile bytecode.
    if args.compile:
        compile_bytecode()
        sys.exit(0)

{'fraction_non_alphanumeric': 0.10907704042715484, 'fraction_numerical': 0.0040999237223493515, 'mean_word_length': 3.126278520849725, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5289644', 'n_tokens_mistral': 3255, 'n_tokens_neox': 3132, 'n_words': 919}