<commit_before>#!/usr/bin/env python3
# This script support python 3.6 and later

import argparse
import logging
import lzma
import os
import re
import subprocess
from pathlib import Path
from shutil import copyfile

AMD64_LINUX_MUSL = 'x86_64-unknown-linux-musl'
ARM64_LINUX_MUSL = 'aarch64-unknown-linux-musl'
ARM_LINUX_MUSL = 'arm-unknown-linux-musleabi'

ARMV7_LINUX_MUSL = 'armv7-unknown-linux-musleabihf'

SCRIPT_DIR = Path(os.path.realpath(__file__)).parent
GIT_ROOT_DIR = SCRIPT_DIR
DIST_DIR = GIT_ROOT_DIR / 'dist'
TARGET_DIR = GIT_ROOT_DIR / 'target'

TARGET_TRIPLE = {
    'amd64': AMD64_LINUX_MUSL,
    'arm64': ARM64_LINUX_MUSL,
    # We don't use hard-float so it fine to build without it
    'arm': ARM_LINUX_MUSL,
    'armv7': ARMV7_LINUX_MUSL,
}


def build_parser():
    parser = argparse.ArgumentParser(prog='build.py')
    parser.add_argument('-v', '--verbose',
                        dest='verbosity',
                        action='count',
                        default=0,
                        help='Request verbose logging')
    parser.add_argument('--lto',
                        dest='lto',
                        action='store_true',
                        default=False,
                        help="Sets link-time optimization to true")
    parser.add_argument('--strip',
                        dest='strip',
                        action='store_true',
                        default=False,
                        help="Strip build")
    parser.add_argument('--dist',
                        dest='dist',
                        action='store_true',
                        default=False,
                        help="Copy to dist/")
    parser.add_argument('--target',
                        dest='target',
                        default='amd64',
                        choices=TARGET_TRIPLE.keys(),
                        help="Set build target")
    parser.add_argument('args', nargs=argparse.REMAINDER)
    return parser


def invalidate_cache():
    """
    Touch some files to force invalidate cache.

    We need to do this to get correct build time & commit id embedded in binary.
    """
    target_source_files = [
        'src/bin/nmk/main.rs',
        'src/bin/nmkup/main.rs',
        # lib doesn't need to invalidate because it doesn't use any env!/option_env!
        # 'src/nmk/lib.rs',
    ]
    logging.debug("Invalidating cache")
    for file in target_source_files:
        os.utime(file)


def build_rust_flags(target, strip):
    flags = []
    if strip:
        flags += ['-C', 'link-arg=-s']
    if target == ARM64_LINUX_MUSL:
        # See https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
        flags += ['-C', 'link-arg=-lgcc', '-C', 'target-feature=+crt-static']
    return " ".join(flags)


def build_release(target, strip=False, lto=False, commit_id=None):
    rust_flags = build_rust_flags(target, strip=strip)
    env = {
        'RUSTFLAGS': rust_flags
    }
    if lto:
        env['CARGO_PROFILE_RELEASE_LTO'] = 'true'
    if commit_id:
        env['GIT_SHORT_SHA'] = commit_id
    args = ['cross', 'build', '--release', '--target', target]<commit_msg>Remove workaround in build script<commit_after>#!/usr/bin/env python3
# This script support python 3.6 and later

import argparse
import logging
import lzma
import os
import re
import subprocess
from pathlib import Path
from shutil import copyfile

AMD64_LINUX_MUSL = 'x86_64-unknown-linux-musl'
ARM64_LINUX_MUSL = 'aarch64-unknown-linux-musl'
# We build non hard-float to maximize compatibility in one binary
ARM_LINUX_MUSL = 'arm-unknown-linux-musleabi'

ARMV7_LINUX_MUSL = 'armv7-unknown-linux-musleabihf'

SCRIPT_DIR = Path(os.path.realpath(__file__)).parent
GIT_ROOT_DIR = SCRIPT_DIR
DIST_DIR = GIT_ROOT_DIR / 'dist'
TARGET_DIR = GIT_ROOT_DIR / 'target'

TARGET_TRIPLE = {
    'amd64': AMD64_LINUX_MUSL,
    'arm64': ARM64_LINUX_MUSL,
    'arm': ARM_LINUX_MUSL,
    'armv7': ARMV7_LINUX_MUSL,
}


def build_parser():
    parser = argparse.ArgumentParser(prog='build.py')
    parser.add_argument('-v', '--verbose',
                        dest='verbosity',
                        action='count',
                        default=0,
                        help='Request verbose logging')
    parser.add_argument('--lto',
                        dest='lto',
                        action='store_true',
                        default=False,
                        help="Sets link-time optimization to true")
    parser.add_argument('--strip',
                        dest='strip',
                        action='store_true',
                        default=False,
                        help="Strip build")
    parser.add_argument('--dist',
                        dest='dist',
                        action='store_true',
                        default=False,
                        help="Copy to dist/")
    parser.add_argument('--target',
                        dest='target',
                        default='amd64',
                        choices=TARGET_TRIPLE.keys(),
                        help="Set build target")
    parser.add_argument('args', nargs=argparse.REMAINDER)
    return parser


def invalidate_cache():
    """
    Touch some files to force invalidate cache.

    We need to do this to get correct build time & commit id embedded in binary.
    """
    target_source_files = [
        'src/bin/nmk/main.rs',
        'src/bin/nmkup/main.rs',
        # lib doesn't need to invalidate because it doesn't use any env!/option_env!
        # 'src/nmk/lib.rs',
    ]
    logging.debug("Invalidating cache")
    for file in target_source_files:
        os.utime(file)


def build_rust_flags(strip):
    flags = []
    if strip:
        flags += ['-C', 'link-arg=-s']
    return " ".join(flags)


def build_release(target, strip=False, lto=False, commit_id=None):
    rust_flags = build_rust_flags(strip=strip)
    env = {
        'RUSTFLAGS': rust_flags
    }
    if lto:
        env['CARGO_PROFILE_RELEASE_LTO'] = 'true'
    if commit_id:
        env['GIT_SHORT_SHA'] = commit_id
    args = ['cross', 'build', '--release', '--target', target]
{'fraction_non_alphanumeric': 0.11561071195297191, 'fraction_numerical': 0.011757021554539516, 'mean_word_length': 2.0442345924453282, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3249083', 'n_tokens_mistral': 1974, 'n_tokens_neox': 1865, 'n_words': 398}