<commit_before>from base64 import urlsafe_b64encode
import glob
import json
import logging
import os
import re
import requests
import time
import uuid

import utils
import centinel.client

logging.getLogger("requests").setLevel(logging.WARNING)

class User:
    def __init__(self, config):
        self.config = config
        self.verify = self.config['server']['verify']
        # check for login file
        if os.path.isfile(config['server']['login_file']):
            with open(config['server']['login_file']) as login_fh:
                login_details = json.load(login_fh)
                self.username = login_details.get('username')
                self.password = login_details.get('password')
                self.typeable_handle = login_details.get('typeable_handle')
                self.auth     = (self.username, self.password)
        else:
            self.create_user()

    def request(self, slug):
        url = "%s/%s" % (self.config['server']['server_url'], slug)
        try:
            req = requests.get(url, auth=self.auth,
                               proxies=self.config['proxy']['proxy'],
                               verify=self.verify)
            req.raise_for_status()
            return req.json()
        except Exception as exp:
            logging.error("Exception trying to make request - %s for URL: %s" %
                          (exp, url))
            raise exp

    @property
    def recommended_version(self):
        try:
            return int(self.request("version")["version"])
        except Exception as exp:
            logging.error("Exception trying to get recommended version: %s " %
                          (exp))
            raise exp

    @property
    def experiments(self):
        try:
            return self.request("experiments")["experiments"]
        except Exception as exp:
            logging.error("Error trying to get experiments: %s " % (exp))
            raise exp

    @property
    def input_files(self):
        try:
            return self.request("input_files")["inputs"]
        except Exception as exp:
            logging.error("Error trying to get input files: %s " % (exp))
            raise exp

    @property
    def results(self):
        try:
            return self.request("results")
        except Exception as exp:
            logging.error("Error trying to get results: %s " % (exp))
            raise exp

    @property
    def clients(self):
        try:
            return self.request("clients")
        except Exception as exp:
            logging.error("Error trying to get clients: %s " % (exp))
            raise exp

    def submit_result(self, file_name):
        logging.info("Uploading result file - %s", file_name)

        with open(file_name) as result_file:
            files   = {'result': result_file}
            url     = "%s/%s" % (self.config['server']['server_url'],
                                 "results")
            timeout = self.config['server']['req_timeout']
            try:
                req = requests.post(url, files=files, auth=self.auth,
                                    proxies=self.config['proxy']['proxy'],
                                    timeout=timeout,
                                    verify=self.verify)
                req.raise_for_status()
                if ('delete_after_sync' in self.config['results'].keys()
                   and self.config['results']['delete_after_sync']):
                    os.remove(file_name)
            except Exception as exp:
                logging.error("Error trying to submit result: %s" % exp)
                raise exp

    def sync_scheduler(self):
        """Download the scheduler.info file and perform a smart comparison
        with what we currently have so that we don't overwrite the
        last_run timestamp

        To do a smart comparison, we go over each entry in the
        server's scheduler file. If a scheduler entry is not present
        in the server copy, we delete it in the client copy and if the
        scheduler entry is present in the server copy, then we
        overwrite the frequency count in the client copy

        """
        # get the server scheduler.info file
        url = "%s/%s/%s" % (self.config['server']['server_url'],
                            "experiments", "scheduler.info")
        try:
            req = requests.get(url, proxies=self.config['proxy']['proxy'],
                               auth=self.auth,
                               verify=self.verify)
            req.raise_for_status()
        except Exception as exp:
            logging.error("Error trying to download scheduler.info: %s" % exp)
            raise exp

        server_sched = json.loads(req.content)
        sched_filename = os.path.join(self.config['dirs']['experiments_dir'],
                                      'scheduler.info')
        if not os.path.exists(sched_filename):
            with open(sched_filename, 'w') as file_p:
                json.dump(server_sched, file_p)
            return

        with open(sched_filename, 'r') as file_p:
            client_sched = json.load(file_p)

        # delete any scheduled tasks as necessary
        for exp in client_sched:
            if exp not in server_sched:
                del client_sched[exp]
        # and update all the other frequencies
        for exp in server_sched:
            if exp in client_sched:
                client_sched[exp]['frequency'] = server_sched[exp]['frequency']
            else:
                client_sched[exp] = server_sched[exp]

        # write out the results
        with open(sched_filename, 'w') as file_p:
            json.dump(client_sched, file_p)

    def download_experiment(self, name):
        logging.info("Downloading experiment - %s", name)

        url = "%s/%s/%s" % (self.config['server']['server_url'],
                            "experiments", name)<commit_msg>fixed bug in scheduler updater<commit_after>from base64 import urlsafe_b64encode
import glob
import json
import logging
import os
import re
import requests
import time
import uuid

import utils
import centinel.client

logging.getLogger("requests").setLevel(logging.WARNING)

class User:
    def __init__(self, config):
        self.config = config
        self.verify = self.config['server']['verify']
        # check for login file
        if os.path.isfile(config['server']['login_file']):
            with open(config['server']['login_file']) as login_fh:
                login_details = json.load(login_fh)
                self.username = login_details.get('username')
                self.password = login_details.get('password')
                self.typeable_handle = login_details.get('typeable_handle')
                self.auth     = (self.username, self.password)
        else:
            self.create_user()

    def request(self, slug):
        url = "%s/%s" % (self.config['server']['server_url'], slug)
        try:
            req = requests.get(url, auth=self.auth,
                               proxies=self.config['proxy']['proxy'],
                               verify=self.verify)
            req.raise_for_status()
            return req.json()
        except Exception as exp:
            logging.error("Exception trying to make request - %s for URL: %s" %
                          (exp, url))
            raise exp

    @property
    def recommended_version(self):
        try:
            return int(self.request("version")["version"])
        except Exception as exp:
            logging.error("Exception trying to get recommended version: %s " %
                          (exp))
            raise exp

    @property
    def experiments(self):
        try:
            return self.request("experiments")["experiments"]
        except Exception as exp:
            logging.error("Error trying to get experiments: %s " % (exp))
            raise exp

    @property
    def input_files(self):
        try:
            return self.request("input_files")["inputs"]
        except Exception as exp:
            logging.error("Error trying to get input files: %s " % (exp))
            raise exp

    @property
    def results(self):
        try:
            return self.request("results")
        except Exception as exp:
            logging.error("Error trying to get results: %s " % (exp))
            raise exp

    @property
    def clients(self):
        try:
            return self.request("clients")
        except Exception as exp:
            logging.error("Error trying to get clients: %s " % (exp))
            raise exp

    def submit_result(self, file_name):
        logging.info("Uploading result file - %s", file_name)

        with open(file_name) as result_file:
            files   = {'result': result_file}
            url     = "%s/%s" % (self.config['server']['server_url'],
                                 "results")
            timeout = self.config['server']['req_timeout']
            try:
                req = requests.post(url, files=files, auth=self.auth,
                                    proxies=self.config['proxy']['proxy'],
                                    timeout=timeout,
                                    verify=self.verify)
                req.raise_for_status()
                if ('delete_after_sync' in self.config['results'].keys()
                   and self.config['results']['delete_after_sync']):
                    os.remove(file_name)
            except Exception as exp:
                logging.error("Error trying to submit result: %s" % exp)
                raise exp

    def sync_scheduler(self):
        """Download the scheduler.info file and perform a smart comparison
        with what we currently have so that we don't overwrite the
        last_run timestamp

        To do a smart comparison, we go over each entry in the
        server's scheduler file. If a scheduler entry is not present
        in the server copy, we delete it in the client copy and if the
        scheduler entry is present in the server copy, then we
        overwrite the frequency count in the client copy

        """
        # get the server scheduler.info file
        url = "%s/%s/%s" % (self.config['server']['server_url'],
                            "experiments", "scheduler.info")
        try:
            req = requests.get(url, proxies=self.config['proxy']['proxy'],
                               auth=self.auth,
                               verify=self.verify)
            req.raise_for_status()
        except Exception as exp:
            logging.error("Error trying to download scheduler.info: %s" % exp)
            raise exp

        server_sched = json.loads(req.content)
        sched_filename = os.path.join(self.config['dirs']['experiments_dir'],
                                      'scheduler.info')
        if not os.path.exists(sched_filename):
            with open(sched_filename, 'w') as file_p:
                json.dump(server_sched, file_p)
            return

        with open(sched_filename, 'r') as file_p:
            client_sched = json.load(file_p)

        # delete any scheduled tasks as necessary
        #
        # Note: this looks ugly, but we can't modify dictionaries
        # while we iterate over them
        client_exp_keys = client_sched.keys()
        for exp in client_exp_keys:
            if exp not in server_sched:
                del client_sched[exp]
        # and update all the other frequencies
        for exp in server_sched:
            if exp in client_sched:
                client_sched[exp]['frequency'] = server_sched[exp]['frequency']
            else:
                client_sched[exp] = server_sched[exp]

        # write out the results
        with open(sched_filename, 'w') as file_p:
            json.dump(client_sched, file_p)

    def download_experiment(self, name):
        logging.info("Downloading experiment - %s", name)

        url = "%s/%s/%s" % (self.config['server']['server_url'],
                            "experiments", name)
{'fraction_non_alphanumeric': 0.09959197268715131, 'fraction_numerical': 0.0006661670413856274, 'mean_word_length': 1.77111213659437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7277082', 'n_tokens_mistral': 3113, 'n_tokens_neox': 3020, 'n_words': 873}