<commit_before>"""deploy.py"""

HELPTEXT = """
                  ----- Deploy Module -----

The deploy module reads a deployment JSON file.

Usage: deploy.py <deployment_file>

"""

import sys, os, urllib2, requests
from multiprocessing import Value
from maestro.internal import module
from maestro.aws import s3

import unpack

HELP_KEYS = ["h", "help"]

class DeploymentModule(module.AsyncModule):
    # Required ID of this module
    id = "deploy"

    def help(self):
        print self.help_text
        exit(0)

    def run(self,kwargs):
        import json, time
        data = None

        with open(sys.argv[1]) as f:
            data = json.load(f)

        downloaders = list()

        for repo in data:
            repo_url = repo["url"]
            for item in repo["items"]:
                name = item["name"]
                version = item["version"]
                platform = item["platform"]
                destination = item["destination"]

                full_url = s3.join_s3_url(repo_url, platform, name, version)
                downloader = s3.AsyncS3Downloader(None)
                downloader.source_url = full_url
                downloader.destination_path = destination
                print "Downloading " + full_url
                downloader.start(None)
                downloaders.append(downloader)

        unpacker_count = 0
        unpackers = list()

        downloader_count = 0
        while downloader_count < len(downloaders):
            downloader_count = 0
            for downloader in downloaders:
                if downloader.status == module.DONE:
                    for item in downloader.result:
                        if item.endswith(".tar.gz"):<commit_msg>deploy: Minor changes before merge into develop<commit_after>"""deploy.py"""

HELPTEXT = """
                  ----- Deploy Module -----

The deploy module reads a deployment JSON file.

Usage: deploy.py <deployment_file>

"""

import sys, os, json
from maestro.internal import module
from maestro.aws import s3

#Local modules
import unpack

HELP_KEYS = ["h", "help"]

class DeploymentModule(module.AsyncModule):
    # Required ID of this module
    id = "deploy"

    def help(self):
        print self.help_text
        exit(0)

    def run(self,kwargs):
        import json, time
        data = None

        with open(sys.argv[1]) as f:
            data = json.load(f)

        fetchers = list()

        for repo in data:
            repo_url = repo["url"]
            for item in repo["items"]:
                name = item["name"]
                version = item["version"]
                platform = item["platform"]
                destination = item["destination"]

                full_url = s3.join_s3_url(repo_url, platform, name, version)
                
                downloader = s3.AsyncS3Downloader(None)
                downloader.source_url = full_url
                downloader.destination_path = destination
                print "Downloading " + full_url
                downloader.start(None)
                downloaders.append(downloader)

        unpacker_count = 0
        unpackers = list()

        downloader_count = 0
        while downloader_count < len(downloaders):
            downloader_count = 0
            for downloader in downloaders:
                if downloader.status == module.DONE:
                    for item in downloader.result:
                        if item.endswith(".tar.gz"):
{'fraction_non_alphanumeric': 0.09241019698725376, 'fraction_numerical': 0.00608342989571263, 'mean_word_length': 1.7557861133280128, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 33, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7086119', 'n_tokens_mistral': 968, 'n_tokens_neox': 910, 'n_words': 250}