<commit_before>#!/usr/bin/env python
"""
Creates an APK file, merging OAT-optimized odex files as needed. Given a
directory containing

    GoogleServicesFramework.apk (without classes.dex)
    arm/GoogleServicesFramework.odex

this program create a classes.dex file from arm/GoogleServicesFramework.odex and
put it in the original GoogleServicesFramework.apk file. If a classes.dex file
already exists, no action is taken. (As an implementation detail, the file
arm/GoogleServicesFramework.dex is also created.)

Set the OAT2DEX environment variable to the location of the oat2dex.jar file.
"""
__author__ = "<NAME>"
__email__ = "<EMAIL>"
__license__ = "MIT"

import argparse, sys, zipfile, os, subprocess, logging
_logger = logging.getLogger("odex2apk")

# Path to oat2dex.jar (from https://github.com/testwhat/SmaliEx.git)
# If not set, try to use a jar file next to the file.
_dirname = os.path.dirname(__file__)
OAT2DEX = os.getenv("OAT2DEX", os.path.join(_dirname, "oat2dex.jar"))

# Supported architectures (first match will be used)
architectures = ["arm", "arm64", "x86", "x86_64"]

def find_odex_for_apk(apk_path):
    """
    Given a filename "Foo.apk", look for a matching odex file such as
    "Foo/Foo.odex". Return the path to that file.
    """
    dirname, filename = os.path.split(apk_path)
    odex_filename = "%s.odex" % os.path.splitext(filename)[0]

    # Look for first available architecture
    for arch in architectures:
        odex_path = os.path.join(dirname, arch, odex_filename)
        if os.path.exists(odex_path):
            return odex_path

    raise FileNotFoundError("No .odex file found!")

def odex_to_dex(odex_path):
    """
    Converts an .odex file to a .dex file, returning its path on success.
    """
    dex_path = "%s.dex" % os.path.splitext(odex_path)[0]
    # Output directory for the dex file
    cwd = os.path.dirname(odex_path)

    # remove old files first
    try:
        os.remove(dex_path)
    except FileNotFoundError:
        pass # ignore missing files

    # Try to create a file. Check for the file existence as the exit code is
    # still 0 even for errors...
    cmd = ["java", "-jar", os.path.abspath(OAT2DEX), "odex",
           os.path.abspath(odex_path)]
    _logger.debug("Executing: %s", cmd)
    output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT)
    if not os.path.exists(dex_path):
        _logger.debug("Program output: %s", output.decode())
        raise RuntimeError("Failed to convert odex to dex")

    # File is generated! Accept it!
    return dex_path

def add_classes_dex(apk_path, dex_path):
    """
    Adds the file specified by dex_path to an APK file (specified by apk_path).
    """
    with zipfile.ZipFile(apk_path, "a", zipfile.ZIP_DEFLATED) as z:
        z.write(dex_path, "classes.dex")

def process_apk(apk_path):
    # Sanity check...
    if not apk_path.endswith(".apk"):
        raise RuntimeError("File %s is not an APK file!" % apk_path)

    # Scan for classes.dex in file list
    file_list = zipfile.ZipFile(apk_path).namelist()
    if "classes.dex" not in file_list:
        # Not found? Try to find odex file...
        odex_path = find_odex_for_apk(apk_path)

        # convert it to a dex file...
        dex_path = odex_to_dex(odex_path)

        # and add it as a classes.dex file
        add_classes_dex(apk_path, dex_path)
        _logger.info("Added %s to %s as classes.dex", dex_path, apk_path)

    # Ready!
    _logger.info("%s is ready!", apk_path)

parser = argparse.ArgumentParser("odex2apk.py", description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("-d", "--debug", action="store_true",
    help="Enable verbose debug logging")
parser.add_argument("files", nargs="+", help="Paths to APK files")

def main():
    args = parser.parse_args()
    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO,
            format="%(name)s: %(message)s")
    for file_path in args.files:
        try:
            process_apk(file_path)
        except:
            _logger.exception("Failed to process %s", file_path)
            sys.exit(1)

if __name__ == "__main__":
    main()<commit_msg>odex2apk: fix oat2dex invocation<commit_after>#!/usr/bin/env python
"""
Creates an APK file, merging OAT-optimized odex files as needed. Given a
directory containing

    framework/arm/boot.oat
    priv-app/GoogleServicesFramework/GoogleServicesFramework.apk
    priv-app/GoogleServicesFramework/arm/GoogleServicesFramework.odex

this program will:

 1. Deoptimize framework/arm/boot.oat to framework/arm/odex/
 2. Deoptimize GoogleServicesFramework.odex to GoogleServicesFramework.dex
 3. Add GoogleServicesFramework.dex as classes.dex to the APK file.

Set the OAT2DEX environment variable to the location of the oat2dex.jar file.
"""
__author__ = "<NAME>"
__email__ = "<EMAIL>"
__license__ = "MIT"

import argparse, sys, zipfile, os, subprocess, logging
_logger = logging.getLogger("odex2apk")

# Path to oat2dex.jar (from https://github.com/testwhat/SmaliEx.git)
# If not set, try to use a jar file next to the file.
_dirname = os.path.dirname(__file__)
OAT2DEX = os.getenv("OAT2DEX", os.path.join(_dirname, "oat2dex.jar"))

# Supported architectures (first match will be used)
architectures = ["arm", "arm64", "x86", "x86_64"]

def detect_arch(dirname):
    # Look for first available architecture (as subdir)
    for arch in architectures:
        arch_path = os.path.join(dirname, arch)
        if os.path.isdir(arch_path):
            return arch
    raise FileNotFoundError("Unable to detect arch, must specify --arch")

def find_odex_for_apk(apk_path, arch):
    """
    Given a filename "Foo.apk", look for a matching odex file such as
    "arm/Foo.odex". Return the path to that file.
    """
    dirname, filename = os.path.split(apk_path)
    odex_filename = "%s.odex" % os.path.splitext(filename)[0]

    odex_path = os.path.join(dirname, arch, odex_filename)
    if os.path.exists(odex_path):
        return odex_path

    raise FileNotFoundError("No .odex file found!")

def odex_to_dex(odex_path, boot_odex_path):
    """
    Converts an .odex file to a .dex file, returning its path on success.
    """
    dex_path = "%s.dex" % os.path.splitext(odex_path)[0]
    # Output directory for the dex file
    cwd = os.path.dirname(odex_path)

    # remove old files first
    try:
        os.remove(dex_path)
    except FileNotFoundError:
        pass # ignore missing files

    # Try to create a file. Check for the file existence as the exit code is
    # still 0 even for errors...
    cmd = ["java", "-jar", os.path.abspath(OAT2DEX),
           os.path.abspath(odex_path), os.path.abspath(boot_odex_path)]
    _logger.debug("Executing: %s", cmd)
    output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT)
    if not os.path.exists(dex_path):
        _logger.debug("Program output: %s", output.decode())
        raise RuntimeError("Failed to convert odex to dex")

    # File is generated! Accept it!
    return dex_path

def add_classes_dex(apk_path, dex_path):
    """
    Adds the file specified by dex_path to an APK file (specified by apk_path).
    """
    with zipfile.ZipFile(apk_path, "a", zipfile.ZIP_DEFLATED) as z:
        z.write(dex_path, "classes.dex")

def process_apk(apk_path, arch, boot_odex_path):
    # Sanity check...
    if not apk_path.endswith(".apk"):
        raise RuntimeError("File %s is not an APK file!" % apk_path)

    # Scan for classes.dex in file list
    file_list = zipfile.ZipFile(apk_path).namelist()
    if "classes.dex" not in file_list:
        # Not found? Try to find odex file...
        odex_path = find_odex_for_apk(apk_path, arch)

        # convert it to a dex file...
        dex_path = odex_to_dex(odex_path, boot_odex_path)

        # and add it as a classes.dex file
        add_classes_dex(apk_path, dex_path)
        _logger.info("Added %s to %s as classes.dex", dex_path, apk_path)

    # Ready!
    _logger.info("%s is ready!", apk_path)

def process_boot(boot_odex_path):
    """
    If the boot odex path does not exist, create it based on ../boot.oat
    (relative to the given path).
    """
    # If the optimized dir cannot be found, try to create it.
    if not os.path.isdir(boot_odex_path):
        # Assume ../arch/odex and find ../arch/boot.oat.
        framework_arch_dir = os.path.dirname(boot_odex_path)
        boot_oat_path = os.path.join(framework_arch_dir, "boot.oat")

        cmd = ["java", "-jar", OAT2DEX, "boot", boot_oat_path]
        _logger.info("Processing boot directory, may take a minute...")
        _logger.debug("Executing: %s", cmd)
        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        if not os.path.exists(boot_odex_path):
            _logger.debug("Program output: %s", output.decode())
            raise RuntimeError("Failed to deoptimize boot.oat")

parser = argparse.ArgumentParser("odex2apk.py", description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("-d", "--debug", action="store_true",
    help="Enable verbose debug logging")
parser.add_argument("-a", "--arch",
    help="""
    Set architecture for all APKs (auto-detected from boot path if omitted).
    """)
parser.add_argument("-f", "--framework", dest="framework_path",
    help="""
    Directory that contains (arch)/boot.oat (if omitted, use ../../framework
    relative to the first given APK file).
    """)
parser.add_argument("files", nargs="+", help="Paths to APK files")

# TODO: this is ugly, maybe split it...
def detect_paths(apk_file, arch=None, framework_path=None):
    first_apk_dir = os.path.dirname(apk_file)

    # Default to ../../framework/boot relative if not given.
    if not framework_path:
        framework_path = os.path.join(first_apk_dir, "..", "..", "framework")

    # Detect architecture based on the APK files.
    if not arch:
        try:
            arch = detect_arch(framework_path)
        except FileNotFoundError:
            _logger.error("Cannot detect architecture")
            sys.exit(1)

    # Assumed boot path, could be non-existing and be created later.
    boot_odex_path = os.path.join(framework_path, arch, "odex")

    return arch, boot_odex_path

def main():
    args = parser.parse_args()
    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO,
            format="%(name)s: %(message)s")

    arch, boot_odex_path = detect_paths(args.files[0],
            args.arch, args.framework_path)

    # Validate boot path and try to optimize these files (needed for APK steps).
    process_boot(boot_odex_path)

    for file_path in args.files:
        try:
            process_apk(file_path, arch, boot_odex_path)
        except:
            _logger.exception("Failed to process %s", file_path)
            sys.exit(1)

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.11472398273169836, 'fraction_numerical': 0.004592633416000735, 'mean_word_length': 3.2332814930015554, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6904348', 'n_tokens_mistral': 3642, 'n_tokens_neox': 3485, 'n_words': 998}