<commit_before>import os
import boto3
import time
import copy
import subprocess
import threading
import multiprocessing

MAX_WORKERS    = 1
SRC_ACCESS_KEY = os.environ['SRC_ACCESS_KEY']
SRC_SECRET_KEY = os.environ['SRC_SECRET_KEY']
SRC_REGION     = os.environ['SRC_REGION']
DST_ACCESS_KEY = os.environ['DST_ACCESS_KEY']
DST_SECRET_KEY = os.environ['DST_SECRET_KEY']
DST_REGION     = os.environ['DST_REGION']
DST_REPO_URI   = os.environ['DST_REPO_URI']

class Repository:
    def __init__(self, name, uri, images):
        self.name = name
        self.uri = uri
        self.images = images

def create_dst_tag(repoName, imageTag):
    return '{0}/{1}:{2}'.format(DST_REPO_URI, repoName, imageTag)

def get_catalog(client):
    catalog = {}

    repos = client.describe_repositories()
    for repo in repos['repositories']:
        name = repo['repositoryName']
        uri = repo['repositoryUri']

        images = []
        paginator = client.get_paginator('list_images')
        for output in paginator.paginate(repositoryName=name, filter={'tagStatus': 'TAGGED'}):
            images = images + output['imageIds']

        catalog[name] = Repository(name, uri, images)

    return catalog

def calculate_catalog_diff(src, dst):
    diff = {}
 
    for name, repo in src.iteritems():
        if not name in dst:
            diff[name] = repo
            continue

        image_diff = copy.deepcopy(repo.images)
        for src_image in repo.images:
            src_tag = src_image['imageTag']
            <commit_msg>Update migrate.py<commit_after>import os
import boto3
import time
import copy
import subprocess
import threading
import multiprocessing

MAX_WORKERS    = 1
SRC_ACCESS_KEY = os.environ['SRC_ACCESS_KEY']
SRC_SECRET_KEY = os.environ['SRC_SECRET_KEY']
SRC_REGION     = os.environ['SRC_REGION']
DST_ACCESS_KEY = os.environ['DST_ACCESS_KEY']
DST_SECRET_KEY = os.environ['DST_SECRET_KEY']
DST_REGION     = os.environ['DST_REGION']
DST_REPO_URI   = os.environ['DST_REPO_URI']

class Repository:
    def __init__(self, name, uri, images):
        self.name = name
        self.uri = uri
        self.images = images

def create_dst_tag(repoName, imageTag):
    return '{0}/{1}:{2}'.format(DST_REPO_URI, repoName, imageTag)

def get_catalog(client):
    catalog = {}

    repos = client.describe_repositories()
    for repo in repos['repositories']:
        name = repo['repositoryName']
        uri = repo['repositoryUri']

        images = []
        paginator = client.get_paginator('list_images')
        for output in paginator.paginate(repositoryName=name, filter={'tagStatus': 'TAGGED'}):
            images = images + output['imageIds']

        catalog[name] = Repository(name, uri, images)

    return catalog

def calculate_catalog_diff(src, dst):
    diff = {}
 
    for name, repo in src.iteritems():
        if not name in dst:
            diff[name] = repo
            continue

        image_diff = copy.deepcopy(repo.images)
        for src_image in repo.images:
            src_tag = src_image['imageTag']
{'fraction_non_alphanumeric': 0.11037891268533773, 'fraction_numerical': 0.0032948929159802307, 'mean_word_length': 3.216666666666667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2308793', 'n_tokens_mistral': 997, 'n_tokens_neox': 993, 'n_words': 213}