<commit_before>from __future__ import absolute_import
import os
from six.moves.urllib.parse import urljoin
from tempfile import mkdtemp
import click
import requests
from shutil import rmtree

from shub.config import get_target
from shub.fetch_eggs import fetch_eggs
from shub.utils import decompress_egg_files, _deploy_dependency_egg

SHORT_HELP = "Sync eggs from one project with other project"

HELP = SHORT_HELP + """

Fetch all eggs from one project and upload them to other project. This allows
you to easily clone requirements from an old project into a new one."""


@click.command(help=HELP, short_help=SHORT_HELP)
@click.option("--source_project", prompt="From which projects should I download eggs?")
@click.option("--new_project", prompt="To which project should I upload eggs?")
@click.option("-m", "--copy-main", default=False, is_flag=True, help="copy main Scrapy project egg")
def cli(source_project, new_project, copy_main):
    project, endpoint, apikey = get_target(source_project)
    new_project, new_endpoint, new_apikey = get_target(new_project)
    copy_eggs(project, endpoint, apikey, new_project, new_endpoint, new_apikey, copy_main)


def copy_eggs(project, endpoint, apikey, new_project, new_endpoint, new_apikey, copy_main):

    egg_versions = get_eggs_versions(project, endpoint, apikey)
    temp_dir = mkdtemp()
    destfile = os.path.join(temp_dir, 'eggs-%s.zip' % project)
    fetch_eggs(project, endpoint, apikey, destfile)

    # this will decompress egg containing other eggs, not eggs from source project
    decompress_egg_files(directory=temp_dir)
    destdir = os.path.join(temp_dir, "eggs-{}".format(project))
    for egg_name in os.listdir(destdir):
        if egg_name == "__main__.egg" and not copy_main:
            continue<commit_msg>Use new config API in copy-eggs<commit_after>from __future__ import absolute_import
import os
from six.moves.urllib.parse import urljoin
from tempfile import mkdtemp
import click
import requests
from shutil import rmtree

from shub.config import get_target_conf
from shub.fetch_eggs import fetch_eggs
from shub.utils import decompress_egg_files, _deploy_dependency_egg

SHORT_HELP = "Sync eggs from one project with other project"

HELP = SHORT_HELP + """

Fetch all eggs from one project and upload them to other project. This allows
you to easily clone requirements from an old project into a new one."""


@click.command(help=HELP, short_help=SHORT_HELP)
@click.option("--source_project", prompt="From which projects should I download eggs?")
@click.option("--new_project", prompt="To which project should I upload eggs?")
@click.option("-m", "--copy-main", default=False, is_flag=True, help="copy main Scrapy project egg")
def cli(source_project, new_project, copy_main):
    source = get_target_conf(source_project)
    target = get_target_conf(new_project)
    copy_eggs(source.project_id, source.endpoint, source.apikey,
              target.project_id, target.endpoint, target.apikey,
              copy_main)


def copy_eggs(project, endpoint, apikey, new_project, new_endpoint, new_apikey, copy_main):

    egg_versions = get_eggs_versions(project, endpoint, apikey)
    temp_dir = mkdtemp()
    destfile = os.path.join(temp_dir, 'eggs-%s.zip' % project)
    fetch_eggs(project, endpoint, apikey, destfile)

    # this will decompress egg containing other eggs, not eggs from source project
    decompress_egg_files(directory=temp_dir)
    destdir = os.path.join(temp_dir, "eggs-{}".format(project))
    for egg_name in os.listdir(destdir):
        if egg_name == "__main__.egg" and not copy_main:
            continue
{'fraction_non_alphanumeric': 0.09116098642283181, 'fraction_numerical': 0.0, 'mean_word_length': 4.966942148760331, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1828276', 'n_tokens_mistral': 1129, 'n_tokens_neox': 1106, 'n_words': 336}