<commit_before>import os, requests, json, datetime, time, sys, re, random, string
from time import sleep
from requests import Timeout
import logging

logger = logging.getLogger("ti.fakes")
logger.setLevel(logging.INFO)

requests_log = logging.getLogger("requests").setLevel(
    logging.WARNING) # Requests' logging is too noisy


# setup external services
webapp_url = "http://" + os.getenv("WEBAPP_ROOT")
api_url = "http://" + os.getenv("API_ROOT")

''' Test classes
*****************************************************************************'''

class Importer:
    '''Emulates a single importer on the create_collection page;

    feed it a provider name like "github" at instantiation, then run run a query
    like jasonpriem to get all the aliases associated with that account
    '''
    query = None

    def __init__(self, provider_name):
        self.provider_name = provider_name

    def get_aliases(self, query):
        query_url = "{api_url}/provider/{provider_name}/memberitems/{query}?method=sync".format(
            api_url=api_url,
            provider_name=self.provider_name,
            query=query
        )
        start = time.time()
        logger.info("getting aliases from the {provider} importer, using url '{url}'".format(
                provider=self.provider_name, url=query_url))
        r = requests.get(query_url)

        try:
            response = json.loads(r.text)
            aliases = response["memberitems"]
            logger.debug("got some aliases from the http call: " + str(aliases))
        except ValueError:
            logger.warning("{provider} importer returned no json for {query}".format(
                    provider=self.provider_name, query="query"))
            aliases = []

        # annoyingly, some providers return lists-as-IDs, which must be joined with a comma
        aliases = [(namespace, id) if isinstance(id, str)
                   else (namespace, ",".join(id)) for namespace, id in aliases]

        logger.info("{provider} importer got {num_aliases} aliases with username '{q}' in {elapsed} seconds.".format(
                provider=self.provider_name, num_aliases=len(aliases), q=query, elapsed=round(time.time() - start, 2)))

        return aliases


class ReportPage:
    def __init__(self, collection_id):
        self.collection_id = collection_id
        start = time.time()
        logger.info("loading the report page for collection '{collection_id}'.".format(
                collection_id=collection_id))
        request_url = "{webapp_url}/collection/{collection_id}".format(
            webapp_url=webapp_url,
            collection_id=collection_id
        )
        resp = requests.get(request_url)
        if resp.status_code == 200:
            self.collectionId = self._get_collectionId(resp.text)
            elapsed = time.time() - start
            logger.info("loaded the report page for '{collection_id}' in {elapsed} seconds.".format(
                    collection_id=collection_id, elapsed=elapsed))
        else:
            logger.warning("report page for '{collection_id}' failed to load! ({url})".format(
                    collection_id=collection_id, url=request_url))


    def _get_collectionId(self, text):
        """gets id of the collection to poll. in the real report page, this is done
        via a 'collectionId' javascript var that's placed there when the view constructs
        the page. this method parses the page to get them...it's brittle, though,
        since if the report page changes this breaks."""

        m = re.search('var collectionId = "([^"]+)"', text)
        collectionId = m.group(1)
        return collectionId


    def poll(self, max_time=60):
        logger.info("polling collection '{collection_id}'".format(
                collection_id=self.collection_id))

        still_updating = True
        tries = 0
        start = time.time()
        while still_updating:
            url = api_url + "/collection/" + self.collectionId
            resp = requests.get(url, config={'verbose': None})
            try:
                items = json.loads(resp.text)["items"]
            except ValueError:
                items = []
                logger.warning("get '{url}' returned no json, only '{resp}') ".format(
                        url=url, resp=resp.text))

            tries += 1

            currently_updating_flags = [True for item in items if
                                        item["currently_updating"]]
            num_currently_updating = len(currently_updating_flags)
            num_finished_updating = len(items) - num_currently_updating

            logger.info("{num_done} of {num_total} items done updating after {tries} requests.".format(
                    num_done=num_finished_updating, num_total=len(items), tries=tries))
            logger.debug("got these items back: " + str(items))

            elapsed = time.time() - start
            if resp.status_code == 200:
                logger.info("collection '{id}' with {num_items} items finished updating in {elapsed} seconds.".format(
                        id=self.collection_id, num_items=len(items), elapsed=round(elapsed, 2)))
                return True
            elif elapsed > max_time:
                raise Exception(
                    "max polling time ({max} secs) exceeded for collection {id}. These items didn't update: {item_ids}".format(
                        max=max_time,
                        id=self.collection_id,
                        item_ids=", ".join([item["_id"] for item in items if
                                            item["currently_updating"]])))
                return False

            sleep(0.5)


class CreateCollectionPage:
    def __init__(self):
        self.reload()

    def reload(self):
        start = time.time()
        logger.info("loading the create-collection page")
        resp = requests.get(webapp_url + "/create")
        if resp.status_code == 200:
            elapsed = time.time() - start
            logger.info("loaded the create-collection page in {elapsed} seconds.".format(
                elapsed=elapsed))
        else:
            logger.warning("create-collection page for '{collection_id}' failed to load!".format(
                    collection_id=collection_id))
        self.aliases = []
        self.collection_name = "My collection"

    def set_collection_name(self, collection_name):
        self.collection_name = collection_name

    def enter_aliases_directly(self, aliases):
        self.aliases = self.aliases + aliases
        return self.aliases

    def get_aliases_with_importers(self, provider_name, query):
        importer = Importer(provider_name)
        aliases_from_this_importer = importer.get_aliases(query)
        self.aliases = self.aliases + aliases_from_this_importer
        return self.aliases

    def press_go_button(self):
        logger.info("user has pressed the 'go' button on the create-collection page.")
        if len(self.aliases) == 0:
            raise ValueError("Trying to create a collection with no aliases.")
<commit_msg>don't throw error when no collection_id in fakes log<commit_after>import os, requests, json, datetime, time, sys, re, random, string
from time import sleep
from requests import Timeout
import logging

logger = logging.getLogger("ti.fakes")
logger.setLevel(logging.INFO)

requests_log = logging.getLogger("requests").setLevel(
    logging.WARNING) # Requests' logging is too noisy


# setup external services
webapp_url = "http://" + os.getenv("WEBAPP_ROOT")
api_url = "http://" + os.getenv("API_ROOT")

''' Test classes
*****************************************************************************'''

class Importer:
    '''Emulates a single importer on the create_collection page;

    feed it a provider name like "github" at instantiation, then run run a query
    like jasonpriem to get all the aliases associated with that account
    '''
    query = None

    def __init__(self, provider_name):
        self.provider_name = provider_name

    def get_aliases(self, query):
        query_url = "{api_url}/provider/{provider_name}/memberitems/{query}?method=sync".format(
            api_url=api_url,
            provider_name=self.provider_name,
            query=query
        )
        start = time.time()
        logger.info("getting aliases from the {provider} importer, using url '{url}'".format(
                provider=self.provider_name, url=query_url))
        r = requests.get(query_url)

        try:
            response = json.loads(r.text)
            aliases = response["memberitems"]
            logger.debug("got some aliases from the http call: " + str(aliases))
        except ValueError:
            logger.warning("{provider} importer returned no json for {query}".format(
                    provider=self.provider_name, query="query"))
            aliases = []

        # annoyingly, some providers return lists-as-IDs, which must be joined with a comma
        aliases = [(namespace, id) if isinstance(id, str)
                   else (namespace, ",".join(id)) for namespace, id in aliases]

        logger.info("{provider} importer got {num_aliases} aliases with username '{q}' in {elapsed} seconds.".format(
                provider=self.provider_name, num_aliases=len(aliases), q=query, elapsed=round(time.time() - start, 2)))

        return aliases


class ReportPage:
    def __init__(self, collection_id):
        self.collection_id = collection_id
        start = time.time()
        logger.info("loading the report page for collection '{collection_id}'.".format(
                collection_id=collection_id))
        request_url = "{webapp_url}/collection/{collection_id}".format(
            webapp_url=webapp_url,
            collection_id=collection_id
        )
        resp = requests.get(request_url)
        if resp.status_code == 200:
            self.collectionId = self._get_collectionId(resp.text)
            elapsed = time.time() - start
            logger.info("loaded the report page for '{collection_id}' in {elapsed} seconds.".format(
                    collection_id=collection_id, elapsed=elapsed))
        else:
            logger.warning("report page for '{collection_id}' failed to load! ({url})".format(
                    collection_id=collection_id, url=request_url))


    def _get_collectionId(self, text):
        """gets id of the collection to poll. in the real report page, this is done
        via a 'collectionId' javascript var that's placed there when the view constructs
        the page. this method parses the page to get them...it's brittle, though,
        since if the report page changes this breaks."""

        m = re.search('var collectionId = "([^"]+)"', text)
        collectionId = m.group(1)
        return collectionId


    def poll(self, max_time=60):
        logger.info("polling collection '{collection_id}'".format(
                collection_id=self.collection_id))

        still_updating = True
        tries = 0
        start = time.time()
        while still_updating:
            url = api_url + "/collection/" + self.collectionId
            resp = requests.get(url, config={'verbose': None})
            try:
                items = json.loads(resp.text)["items"]
            except ValueError:
                items = []
                logger.warning("get '{url}' returned no json, only '{resp}') ".format(
                        url=url, resp=resp.text))

            tries += 1

            currently_updating_flags = [True for item in items if
                                        item["currently_updating"]]
            num_currently_updating = len(currently_updating_flags)
            num_finished_updating = len(items) - num_currently_updating

            logger.info("{num_done} of {num_total} items done updating after {tries} requests.".format(
                    num_done=num_finished_updating, num_total=len(items), tries=tries))
            logger.debug("got these items back: " + str(items))

            elapsed = time.time() - start
            if resp.status_code == 200:
                logger.info("collection '{id}' with {num_items} items finished updating in {elapsed} seconds.".format(
                        id=self.collection_id, num_items=len(items), elapsed=round(elapsed, 2)))
                return True
            elif elapsed > max_time:
                raise Exception(
                    "max polling time ({max} secs) exceeded for collection {id}. These items didn't update: {item_ids}".format(
                        max=max_time,
                        id=self.collection_id,
                        item_ids=", ".join([item["_id"] for item in items if
                                            item["currently_updating"]])))
                return False

            sleep(0.5)


class CreateCollectionPage:
    def __init__(self):
        self.reload()

    def reload(self):
        start = time.time()
        logger.info("loading the create-collection page")
        resp = requests.get(webapp_url + "/create")
        if resp.status_code == 200:
            elapsed = time.time() - start
            logger.info("loaded the create-collection page in {elapsed} seconds.".format(
                elapsed=elapsed))
        else:
            logger.warning("create-collection page failed to load!")
        self.aliases = []
        self.collection_name = "My collection"

    def set_collection_name(self, collection_name):
        self.collection_name = collection_name

    def enter_aliases_directly(self, aliases):
        self.aliases = self.aliases + aliases
        return self.aliases

    def get_aliases_with_importers(self, provider_name, query):
        importer = Importer(provider_name)
        aliases_from_this_importer = importer.get_aliases(query)
        self.aliases = self.aliases + aliases_from_this_importer
        return self.aliases

    def press_go_button(self):
        logger.info("user has pressed the 'go' button on the create-collection page.")
        if len(self.aliases) == 0:
            raise ValueError("Trying to create a collection with no aliases.")

{'fraction_non_alphanumeric': 0.11237623762376238, 'fraction_numerical': 0.0026874115983026876, 'mean_word_length': 2.3406567446255613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '728940', 'n_tokens_mistral': 3881, 'n_tokens_neox': 3706, 'n_words': 1017}