<commit_before>import yaml
import os
import pathlib2
import itertools
import sys
import argparse
import logging
import docker_images

TRAVIS_BUILD_DIR = os.environ.get("TRAVIS_BUILD_DIR", "")
CONFIG_FILE_PATH = pathlib2.Path(TRAVIS_BUILD_DIR, "util", "parsefiles_config.yml")
LOGGER = logging.getLogger(__name__)

def pack_shards(used_images, num_shards):
    """
    Determines an approximation of the optimal way to pack the images into a given number of shards so as to
    equalize the execution time amongst the shards.

    Input:
    used_images: A set of Docker images and their ranks
    num_shards: A number of shards amongst which to distribute the Docker images
    """

    # sorts used containers in descending order on the weight
    sorted_images = sorted(used_images, key = lambda x: x[1], reverse=True) 

    shards = []

    # for the number of shards
    for i in range(0, num_shards):
        # initialize initial dict
        shards.append({"images": [], "sum": 0})
<commit_msg>Silently fail on missing requirements<commit_after>import argparse
import logging
import os
import sys

try:
    # This script is used by docker.mk at parse-time, which means when you run
    # "make requirements" to install the required Python packages, this script
    # runs before its requirements are installed. That means this import will
    # fail.  To prevent a successful installation from having irrelevant error
    # messages, we catch the failure and exit silently.
    import pathlib2
except ImportError:
    sys.exit(1)

import docker_images


TRAVIS_BUILD_DIR = os.environ.get("TRAVIS_BUILD_DIR", "")
CONFIG_FILE_PATH = pathlib2.Path(TRAVIS_BUILD_DIR, "util", "parsefiles_config.yml")
LOGGER = logging.getLogger(__name__)

def pack_shards(used_images, num_shards):
    """
    Determines an approximation of the optimal way to pack the images into a given number of shards so as to
    equalize the execution time amongst the shards.

    Input:
    used_images: A set of Docker images and their ranks
    num_shards: A number of shards amongst which to distribute the Docker images
    """

    # sorts used containers in descending order on the weight
    sorted_images = sorted(used_images, key = lambda x: x[1], reverse=True)

    shards = []

    # for the number of shards
    for i in range(0, num_shards):
        # initialize initial dict
        shards.append({"images": [], "sum": 0})

{'fraction_non_alphanumeric': 0.07128056361375881, 'fraction_numerical': 0.0045586406962287605, 'mean_word_length': 3.8183632734530937, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '88827', 'n_tokens_mistral': 704, 'n_tokens_neox': 684, 'n_words': 274}