<commit_before># Copyright (C) 2014-2016 DNAnexus, Inc.
#
# This file is part of dx-toolkit (DNAnexus platform client libraries).
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
#   use this file except in compliance with the License. You may obtain a copy
#   of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.

'''
App Unbuilder
+++++++++++++

Contains utility methods for "decompiling" an applet from the platform
and generating an equivalent applet source directory.

'''

from __future__ import print_function, unicode_literals, division, absolute_import

import collections
import json
import os
import subprocess
import sys
import tarfile

import dxpy
from .. import get_handler, download_dxfile
from ..compat import open
from ..exceptions import err_exit, DXError
from .pretty_print import flatten_json_array

def _recursive_cleanup(foo):
    """
    Aggressively cleans up things that look empty.
    """
    if isinstance(foo, dict):
        for (key, val) in foo.items():
            if isinstance(val, dict):
                _recursive_cleanup(val)
            if val == "" or val == [] or val == {}:
                del foo[key]

def dump_executable(executable, destination_directory, omit_resources=False, describe_output=[]):
    """
    Reconstitutes executable into a directory that would create a
    functionally identical executable if "dx build" were run on it.
    destination_directory will be the root source directory for the
    applet.

    :param executable: executable, i.e. app or applet,  to be dumped
    :type executable: DXExecutable (only DXApp or DXApplet now)
    :param destination_directory: an existing, empty, and writable directory
    :type destination_directory: str
    """

    old_cwd = os.getcwd()
    os.chdir(destination_directory)

    try:
        info = executable.get()

        if info["runSpec"]["interpreter"] == "bash":
            suffix = "sh"
        elif info["runSpec"]["interpreter"] == "python2.7":
            suffix = "py"
        else:
            print('Sorry, I don\'t know how to get executables with interpreter ' +
                  info["runSpec"]["interpreter"] + '\n', file=sys.stderr)
            sys.exit(1)

        # Entry point script
        script = "src/code.%s" % (suffix,)
        os.mkdir("src")
        with open(script, "w") as f:
            f.write(info["runSpec"]["code"])

        # Get all the asset bundles
        asset_depends = []
        deps_to_remove = []

        # When an applet is built bundledDepends are added in the following order:
        # 1. bundledDepends explicitly specified in the dxapp.json
        # 2. resources (contents of resources directory added as bundledDepends)
        # 3. assetDepends (translated into bundledDepends)
        #
        # Therefore while translating bundledDepends to assetDepends, we are traversing the
        # list in reverse order and exiting when we can't find the "AssetBundle" property
        # with the tarball file.
        #
        # NOTE: If last item (and contiguous earlier items) of bundledDepends (#1 above) refers to an
        # AssetBundle tarball, those items will be converted to assetDepends.
        #
        # TODO: The bundledDepends should be annotated with another field called {"asset": true}
        # to distinguish it from non assets. It will be needed to annotate the bundleDepends,
        # when the wrapper record object is no more accessible.

        for dep in reversed(info["runSpec"]["bundledDepends"]):
            file_handle = get_handler(dep["id"])
            if isinstance(file_handle, dxpy.DXFile):
                asset_record_id = file_handle.get_properties().get("AssetBundle")
                asset_record = None
                if asset_record_id:
                    asset_record = dxpy.DXRecord(asset_record_id)
                    if asset_record:
                        try:
                            asset_depends.append({"name": asset_record.describe().get("name"),
                                                  "project": asset_record.get_proj_id(),
                                                  "folder": asset_record.describe().get("folder"),
                                                  "version": asset_record.describe(fields={"properties": True}
                                                                                   )["properties"]["version"]
                                                  })
                            deps_to_remove.append(dep)
                        except DXError:
                            print("Describe failed on the assetDepends record object with ID - " +
                                  asset_record_id + "\n", file=sys.stderr)
                            pass
                else:
                    break
        # Reversing the order of the asset_depends[] so that original order is maintained
        asset_depends.reverse()
        # resources/ directory
        created_resources_directory = False
        if not omit_resources:
            for dep in info["runSpec"]["bundledDepends"]:
                if dep in deps_to_remove:
                    continue
                handler = get_handler(dep["id"])
                if isinstance(handler, dxpy.DXFile):
                    if not created_resources_directory:
                        os.mkdir("resources")
                        created_resources_directory = True
                    handler_id = handler.get_id()
                    fname = "resources/%s.tar.gz" % (handler_id)
                    download_dxfile(handler_id, fname)
                    print("Unpacking resources", file=sys.stderr)
                    tar = tarfile.open(fname)
                    tar.extractall("resources")
                    tar.close()<commit_msg>DEVEX-1 - Fix 'dx get' untar issue<commit_after># Copyright (C) 2014-2016 DNAnexus, Inc.
#
# This file is part of dx-toolkit (DNAnexus platform client libraries).
#
#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
#   use this file except in compliance with the License. You may obtain a copy
#   of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#   License for the specific language governing permissions and limitations
#   under the License.

'''
App Unbuilder
+++++++++++++

Contains utility methods for "decompiling" an applet from the platform
and generating an equivalent applet source directory.

'''

from __future__ import print_function, unicode_literals, division, absolute_import

import collections
import json
import os
import subprocess
import sys
import tarfile

import dxpy
from .. import get_handler, download_dxfile
from ..compat import open
from ..exceptions import err_exit, DXError
from .pretty_print import flatten_json_array

def _recursive_cleanup(foo):
    """
    Aggressively cleans up things that look empty.
    """
    if isinstance(foo, dict):
        for (key, val) in foo.items():
            if isinstance(val, dict):
                _recursive_cleanup(val)
            if val == "" or val == [] or val == {}:
                del foo[key]

def dump_executable(executable, destination_directory, omit_resources=False, describe_output=[]):
    """
    Reconstitutes executable into a directory that would create a
    functionally identical executable if "dx build" were run on it.
    destination_directory will be the root source directory for the
    applet.

    :param executable: executable, i.e. app or applet,  to be dumped
    :type executable: DXExecutable (only DXApp or DXApplet now)
    :param destination_directory: an existing, empty, and writable directory
    :type destination_directory: str
    """

    old_cwd = os.getcwd()
    os.chdir(destination_directory)

    try:
        info = executable.get()

        if info["runSpec"]["interpreter"] == "bash":
            suffix = "sh"
        elif info["runSpec"]["interpreter"] == "python2.7":
            suffix = "py"
        else:
            print('Sorry, I don\'t know how to get executables with interpreter ' +
                  info["runSpec"]["interpreter"] + '\n', file=sys.stderr)
            sys.exit(1)

        # Entry point script
        script = "src/code.%s" % (suffix,)
        os.mkdir("src")
        with open(script, "w") as f:
            f.write(info["runSpec"]["code"])

        # Get all the asset bundles
        asset_depends = []
        deps_to_remove = []

        # When an applet is built bundledDepends are added in the following order:
        # 1. bundledDepends explicitly specified in the dxapp.json
        # 2. resources (contents of resources directory added as bundledDepends)
        # 3. assetDepends (translated into bundledDepends)
        #
        # Therefore while translating bundledDepends to assetDepends, we are traversing the
        # list in reverse order and exiting when we can't find the "AssetBundle" property
        # with the tarball file.
        #
        # NOTE: If last item (and contiguous earlier items) of bundledDepends (#1 above) refers to an
        # AssetBundle tarball, those items will be converted to assetDepends.
        #
        # TODO: The bundledDepends should be annotated with another field called {"asset": true}
        # to distinguish it from non assets. It will be needed to annotate the bundleDepends,
        # when the wrapper record object is no more accessible.

        for dep in reversed(info["runSpec"]["bundledDepends"]):
            file_handle = get_handler(dep["id"])
            if isinstance(file_handle, dxpy.DXFile):
                asset_record_id = file_handle.get_properties().get("AssetBundle")
                asset_record = None
                if asset_record_id:
                    asset_record = dxpy.DXRecord(asset_record_id)
                    if asset_record:
                        try:
                            asset_depends.append({"name": asset_record.describe().get("name"),
                                                  "project": asset_record.get_proj_id(),
                                                  "folder": asset_record.describe().get("folder"),
                                                  "version": asset_record.describe(fields={"properties": True}
                                                                                   )["properties"]["version"]
                                                  })
                            deps_to_remove.append(dep)
                        except DXError:
                            print("Describe failed on the assetDepends record object with ID - " +
                                  asset_record_id + "\n", file=sys.stderr)
                            pass
                else:
                    break
        # Reversing the order of the asset_depends[] so that original order is maintained
        asset_depends.reverse()
        # resources/ directory
        created_resources_directory = False
        if not omit_resources:
            for dep in info["runSpec"]["bundledDepends"]:
                if dep in deps_to_remove:
                    continue
                handler = get_handler(dep["id"])
                if isinstance(handler, dxpy.DXFile):
                    if not created_resources_directory:
                        os.mkdir("resources")
                        created_resources_directory = True
                    handler_id = handler.get_id()
                    fname = "resources/%s.tar.gz" % (handler_id)
                    download_dxfile(handler_id, fname)
                    print("Unpacking resources", file=sys.stderr)

                    def untar_strip_leading_slash(tarfname, path):
                        t = tarfile.open(tarfname)
                        for m in t.getmembers():
                            if m.name.startswith("/"):
                                m.name = m.name[1:]
                            t.extract(m, path)
                        t.close()

                    untar_strip_leading_slash(fname, "resources")
{'fraction_non_alphanumeric': 0.08304334071496362, 'fraction_numerical': 0.0031635558367605187, 'mean_word_length': 1.9095720202485045, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4027065', 'n_tokens_mistral': 3241, 'n_tokens_neox': 3073, 'n_words': 1048}