<commit_before># Python modules
from __future__ import division
import os
import tarfile
import zipfile as zipfile_module
import hashlib

"""A developer tool that builds bundles (a tarballs and a ZIP file) and
hash files suitable for distribution to end users.

This works by examining the list of prebuilt directories on the fly combining
them into an all-in-one mega bundle that contains the binaries for all
platforms.

It might be nicer to create platform-specific bundles rather than cramming
all of them into one, and it wouldn't be difficult code to write. I chose
one mega binary because it's simpler code here and (more importantly) simpler
for the user.

Python's distutils package has some support for this kind of packaging via
the sdist command. I tried to use it, and shoehorning it into PyGAMMA's
setup.py alongside the install command was more trouble than it was worth.
"""

# TARGET_DIR is where the bundles & hash files get created.
TARGET_DIR = "dist"

VERSION = open(os.path.join("..", "VERSION")).read().strip()

BUNDLE_DIR = "pygamma-%s" % VERSION

# standard files are included regardless of what platform the bundle is
# being built for.
STANDARD = ["setup.py",
            "init.py",
            "homemade",
            os.path.join("..", "LICENSE"),
            os.path.join("..", "README"),
            os.path.join("..", "VERSION"),
            ]


def list_files(path):
    """Given a path, returns a list of the files found there. The files
    returned are just names; they're not fully qualified.

    Directories, files that start with '.' and .pyc files are excluded.

    This function doesn't travel into subdirectories.
    """
    filenames = [ ]

    for filename in os.listdir(path):
        # Exclude non-files, files that start with a dot, and compiled Python
        # files
        if os.path.isfile(os.path.join(path, filename))     and \
           not filename.startswith(".")                     and \
           not filename.endswith(".pyc"):
            filenames.append(filename)

    return filenames


def build_bundles(target_path, filenames):
    """Given a target filename and a list of files, builds a tarball & ZIP
    file containing the files named in the filenames param. Also creates
    MD5 and SHA1 hash files for each tarball & ZIP file.

    target_path should be the path (fully qualified or relative) and name of
    the tarball/ZIP file minus the extension, e.g. "pygamma-4.2.0",
    "dist/pygamma-4.2.0.linux-32-py25" or "pygamma-4.2.0.all".

    The filenames in files must be relative to the current working directory.
    """
    print "Building the tarball and ZIP file..."

    tarball_name = target_path + ".tar.gz"
    zipfile_name = target_path + ".zip"

    # Clean up old stuff to ensure that I don't append to an existing
    # tarball or zip file.
    if os.path.exists(tarball_name):
        os.remove(tarball_name)
    if os.path.exists(zipfile_name):
        os.remove(zipfile_name)

    zipfile = zipfile_module.ZipFile(zipfile_name, "w",
                                     zipfile_module.ZIP_DEFLATED)
    tarball = tarfile.open(tarball_name, "w:gz")

    for filename in filenames:
        print "Adding %s..." % filename

        # Note that both ZIP files and tarballs support giving the file a
        # different name in the bundle. Most of our files don't need to change
        # name, but a few do.
        # Specifically, of our files come from the directory above current,
        # and neither tar nor zip support unbundling files above the
        # current directory.
        bundle_name = filename

        if bundle_name.startswith(".."):
            # This is ../VERSION or one of its siblings. Once in the bundle,
            # these should be in the root so I trim the reference to "../".
            bundle_name = bundle_name[3:]

        tarball.add(filename, bundle_name, False)

        # ZIP files require a little more work.
        if bundle_name != filename:
            # This is one of the repositioned files from my parent dir.
            # The zipfile interface doesn't support renaming them via
            # zipfile.write(), but this works.
            zipfile.writestr(bundle_name, open(filename).read())
        else:
            if os.path.isdir(filename):
                # Trouble here. Either the ZIP format or Python's interface
                # to it doesn't really support empty directories. The
                # workaround is to add a trailing path separator to the
                # filename and write it as an empty string.
                # ref: http://bytes.com/topic/python/answers/19053-add-empty-directory-using-zipfile
                zipfile.writestr(filename + os.sep, "")
            else:
                # This is the simple case. =)
                zipfile.write(filename)

    zipfile.close()
    tarball.close()<commit_msg>Fixed a bug in build_bundle that was being obscured by a helpful feature in the OS X Archive Manager.<commit_after># Python modules
from __future__ import division
import os
import tarfile
import zipfile as zipfile_module
import hashlib

"""A developer tool that builds bundles (a tarballs and a ZIP file) and
hash files suitable for distribution to end users.

This works by examining the list of prebuilt directories on the fly combining
them into an all-in-one mega bundle that contains the binaries for all
platforms.

It might be nicer to create platform-specific bundles rather than cramming
all of them into one, and it wouldn't be difficult code to write. I chose
one mega binary because it's simpler code here and (more importantly) simpler
for the user.

Python's distutils package has some support for this kind of packaging via
the sdist command. I tried to use it, and shoehorning it into PyGAMMA's
setup.py alongside the install command was more trouble than it was worth.
"""

# TARGET_DIR is where the bundles & hash files get created.
TARGET_DIR = "dist"

VERSION = open(os.path.join("..", "VERSION")).read().strip()

BUNDLE_DIR = "pygamma-%s" % VERSION

# standard files are included regardless of what platform the bundle is
# being built for.
STANDARD = ["setup.py",
            "init.py",
            "homemade",
            os.path.join("..", "LICENSE"),
            os.path.join("..", "README"),
            os.path.join("..", "VERSION"),
            ]


def list_files(path):
    """Given a path, returns a list of the files found there. The files
    returned are just names; they're not fully qualified.

    Directories, files that start with '.' and .pyc files are excluded.

    This function doesn't travel into subdirectories.
    """
    filenames = [ ]

    for filename in os.listdir(path):
        # Exclude non-files, files that start with a dot, and compiled Python
        # files
        if os.path.isfile(os.path.join(path, filename))     and \
           not filename.startswith(".")                     and \
           not filename.endswith(".pyc"):
            filenames.append(filename)

    return filenames


def build_bundles(target_path, filenames):
    """Given a target filename and a list of files, builds a tarball & ZIP
    file containing the files named in the filenames param. Also creates
    MD5 and SHA1 hash files for each tarball & ZIP file.

    target_path should be the path (fully qualified or relative) and name of
    the tarball/ZIP file minus the extension, e.g. "pygamma-4.2.0",
    "dist/pygamma-4.2.0.linux-32-py25" or "pygamma-4.2.0.all".

    The filenames in files must be relative to the current working directory.
    """
    print "Building the tarball and ZIP file..."

    tarball_name = target_path + ".tar.gz"
    zipfile_name = target_path + ".zip"

    # Clean up old stuff to ensure that I don't append to an existing
    # tarball or zip file.
    if os.path.exists(tarball_name):
        os.remove(tarball_name)
    if os.path.exists(zipfile_name):
        os.remove(zipfile_name)

    zipfile = zipfile_module.ZipFile(zipfile_name, "w",
                                     zipfile_module.ZIP_DEFLATED)
    tarball = tarfile.open(tarball_name, "w:gz")

    for filename in filenames:
        print "Adding %s..." % filename

        # Note that both ZIP files and tarballs support giving the file a
        # different name in the bundle. Most of our files don't need to change
        # name, but a few do.
        # Specifically, of our files come from the directory above current,
        # and neither tar nor zip support unbundling files above the
        # current directory.
        bundle_name = filename

        if bundle_name.startswith(".."):
            # This is ../VERSION or one of its siblings. Once in the bundle,
            # these should be in the root so I trim the reference to "../".
            bundle_name = bundle_name[3:]
            
        bundle_name = os.path.join(BUNDLE_DIR, bundle_name)

        tarball.add(filename, bundle_name, False)

        # ZIP files are a little more difficult.
        if os.path.isdir(filename):
            # Trouble here. Either the ZIP format or Python's interface
            # to it doesn't really support empty directories. The
            # workaround is to add a trailing path separator to the
            # filename and write it as an empty string.
            # ref: http://bytes.com/topic/python/answers/19053-add-empty-directory-using-zipfile
            zipfile.writestr(bundle_name + os.sep, "")
        else:
            zipfile.writestr(bundle_name, open(filename).read())

    zipfile.close()
    tarball.close()
{'fraction_non_alphanumeric': 0.08246128087065718, 'fraction_numerical': 0.004395144411887819, 'mean_word_length': 2.4968898646176365, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '779750', 'n_tokens_mistral': 2730, 'n_tokens_neox': 2595, 'n_words': 1077}