<commit_before>from __future__ import print_function
from os.path import isfile, exists
from os import mknod, remove
from time import sleep
from yaml import load
from reflector.util import *

KEY_TITLE = {'xml': 'title', 'json': 'Id'}
KEY_CONTENT = 'content'
KEY_SRC = 'src'
KEY_REL = 'rel'
KEY_HREF = 'href'
VALUE_NEXT = {'xml': 'next', 'json': '__next'}


class Config(object):
    def __init__(self):
        with open('config/config.yaml') as data_file:
            c = load(data_file)
            self.remote_url = c['remote']['url'].rstrip('/')
            self.update_feed = c['remote']['update_feed']
            self.remote_json_api = c['remote']['json_api']
            self.local_url = c['local']['url'].rstrip('/')
            self.local_json_api = c['local']['json_api']
            self.local_api_key = c['local']['api_key']
            self.package_storage_path = c['local']['package_storage_path'].rstrip('/').rstrip('\\')
            self.hash_verify_downloads = c['hash']['verify_downloads']
            self.hash_verify_uploaded = c['hash']['verify_uploaded']
            self.dotnet_path = c['local']['dotnet_path']
            if not self.dotnet_path or not isfile(self.dotnet_path):
                raise EnvironmentError('DotNot CLI executable is not configured or the path specified does not exist!')


class Mirror(object):
    def __init__(self,
                 remote_url,
                 update_feed,
                 remote_json_api,
                 local_url,
                 local_json_api,
                 package_storage_path,
                 local_api_key,
                 dotnet_path,
                 verify_downloads=True,
                 verify_uploaded=True
                 ):
        """
        :param remote_url:
        :param update_feed:
        :param remote_json_api:
        :param local_url:
        :param local_json_api:
        :param package_storage_path:
        :param local_api_key:
        :param dotnet_path:
        :param verify_downloads:
        :param verify_uploaded:
        """
        self.remote_api_url = '/'.join([remote_url, 'api/v2'])
        self.update_feed = update_feed
        self.remote_json_api = remote_json_api
        self.remote_packages_url = '/'.join([self.remote_api_url, 'Packages'])
        self.local_api_url = '/'.join([local_url, 'api/v2'])
        self.local_json_api = local_json_api
        self.local_api_upload_url = '/'.join([self.local_api_url, 'package'])
        self.local_packages_url = '/'.join([self.local_api_url, 'Packages'])
        self.package_storage_path = package_storage_path
        self.local_api_key = local_api_key
        self.dotnet_path = dotnet_path
        self.verify_downloads = verify_downloads
        self.verify_uploaded = verify_uploaded

    def __sync(self, content_url, save_to, package_name, version,
               source_hash=None,
               source_hash_method=None,
               dl_reties=0,
               up_retries=0,
               force_dl=False,
               force_up=False
               ):
        """
        :param content_url: 
        :param save_to: 
        :param package_name: 
        :param version: 
        :param source_hash: 
        :param source_hash_method: 
        :param dl_reties:
        :param up_retries:
        :param force_dl:
        :param force_up: 
        :return: 
        """
        # Is the package already uploaded? Pull it from the target API
        pull_request = pull_package(package_name, version, self.local_packages_url, self.local_json_api)

        if not pull_request:
            print('Unable to sync package.')
            print(pull_request)
            exit(111)
            return False

        # What did the target api return
        if pull_request.status_code == 404 or pull_request.status_code == 200 or force_dl or not isfile(save_to):
            # Download the file if we are forcing or it was not already uploaded or cached
            if pull_request.status_code == 404 or not isfile(save_to) or force_dl:
                if not download_file(content_url, save_to):
                    print('Package does not exists after download!?!')
                    return False

            # Did we get a source hash when this was called<commit_msg>added general exception catch<commit_after>from __future__ import print_function
from os.path import isfile, exists
from os import mknod, remove
from time import sleep
from yaml import load
from reflector.util import *

KEY_TITLE = {'xml': 'title', 'json': 'Id'}
KEY_CONTENT = 'content'
KEY_SRC = 'src'
KEY_REL = 'rel'
KEY_HREF = 'href'
VALUE_NEXT = {'xml': 'next', 'json': '__next'}


class Config(object):
    def __init__(self):
        with open('config/config.yaml') as data_file:
            c = load(data_file)
            self.remote_url = c['remote']['url'].rstrip('/')
            self.update_feed = c['remote']['update_feed']
            self.remote_json_api = c['remote']['json_api']
            self.local_url = c['local']['url'].rstrip('/')
            self.local_json_api = c['local']['json_api']
            self.local_api_key = c['local']['api_key']
            self.package_storage_path = c['local']['package_storage_path'].rstrip('/').rstrip('\\')
            self.hash_verify_downloads = c['hash']['verify_downloads']
            self.hash_verify_uploaded = c['hash']['verify_uploaded']
            self.dotnet_path = c['local']['dotnet_path']
            if not self.dotnet_path or not isfile(self.dotnet_path):
                raise EnvironmentError('DotNot CLI executable is not configured or the path specified does not exist!')


class Mirror(object):
    def __init__(self,
                 remote_url,
                 update_feed,
                 remote_json_api,
                 local_url,
                 local_json_api,
                 package_storage_path,
                 local_api_key,
                 dotnet_path,
                 verify_downloads=True,
                 verify_uploaded=True
                 ):
        """
        :param remote_url:
        :param update_feed:
        :param remote_json_api:
        :param local_url:
        :param local_json_api:
        :param package_storage_path:
        :param local_api_key:
        :param dotnet_path:
        :param verify_downloads:
        :param verify_uploaded:
        """
        self.remote_api_url = '/'.join([remote_url, 'api/v2'])
        self.update_feed = update_feed
        self.remote_json_api = remote_json_api
        self.remote_packages_url = '/'.join([self.remote_api_url, 'Packages'])
        self.local_api_url = '/'.join([local_url, 'api/v2'])
        self.local_json_api = local_json_api
        self.local_api_upload_url = '/'.join([self.local_api_url, 'package'])
        self.local_packages_url = '/'.join([self.local_api_url, 'Packages'])
        self.package_storage_path = package_storage_path
        self.local_api_key = local_api_key
        self.dotnet_path = dotnet_path
        self.verify_downloads = verify_downloads
        self.verify_uploaded = verify_uploaded

    def __sync(self, content_url, save_to, package_name, version,
               source_hash=None,
               source_hash_method=None,
               dl_reties=0,
               up_retries=0,
               force_dl=False,
               force_up=False
               ):
        """
        :param content_url: 
        :param save_to: 
        :param package_name: 
        :param version: 
        :param source_hash: 
        :param source_hash_method: 
        :param dl_reties:
        :param up_retries:
        :param force_dl:
        :param force_up: 
        :return: 
        """
        # Is the package already uploaded? Pull it from the target API
        pull_request = pull_package(package_name, version, self.local_packages_url, self.local_json_api)

        if not pull_request.status_code:
            print('Unable to sync package.')
            exit(111)
            return False

        # What did the target api return
        if pull_request.status_code == 404 or pull_request.status_code == 200 or force_dl or not isfile(save_to):
            # Download the file if we are forcing or it was not already uploaded or cached
            if pull_request.status_code == 404 or not isfile(save_to) or force_dl:
                if not download_file(content_url, save_to):
                    print('Package does not exists after download!?!')
                    return False

            # Did we get a source hash when this was called
{'fraction_non_alphanumeric': 0.09948441528005625, 'fraction_numerical': 0.0037497070541363956, 'mean_word_length': 2.1564349112426036, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4081375', 'n_tokens_mistral': 2500, 'n_tokens_neox': 2427, 'n_words': 561}