<commit_before>from Queue import Queue

from .. import User, API


class Parser(object):

    def __init__(self):
        self.apis = API.load_all()
        self.queue = Queue()
        for api in self.apis:
            self.queue.put(api)

    def api_getter(func):
        def debug_wrapper(*args, **kwargs):
            self = args[0]
            if not self.queue.empty():
                api = self.queue.get()
                kwargs['api'] = api
                result = func(*args, **kwargs)
                self.queue.put(api)
                return result
            return None
        return debug_wrapper

    @api_getter
    def get_user_followers(self, user_id, max_id="", api=None):
        if api == None:
            assert(not "No api.")
        api.getUserFollowers(user_id, maxid=max_id)
        result = api.LastJson
        users = result["users"]
        if result["big_list"] is False:
            return (users, None)

        next_max_id = result["next_max_id"]
        return (users, next_max_id)


    # def get_total_followers(self, user_id):
    #
    #
    #
    #     followers = []
    #     next_max_id = ''
    #     self.getUsernameInfo(usernameId)
    #     if "user" in self.LastJson:
    #         total_followers = self.LastJson["user"]['follower_count']
    #     else:
    #         return False
    #     with tqdm(total=total_followers, desc="Getting followers", leave=False) as pbar:
    #         while True:
    #             self.getUserFollowers(usernameId, next_max_id)
    #             temp = self.LastJson
    #             try:
    #                 pbar.update(len(temp["users"]))
    #                 for item in temp["users"]:
    #                     followers.append(item)
    #                 if len(temp["users"]) == 0 or len(followers) >= total_followers:
    #                     return followers[:total_followers]
    #             except:
    #                 return followers[:total_followers]
    #             if temp["big_list"] is False:
    #                 return followers[:total_followers]
    #             next_max_id = temp["next_max_id"]
<commit_msg>add get_user_info and get_user_followers methods to the parser<commit_after>import time
import warnings
from Queue import Queue
from tqdm import tqdm

from .. import User, API


class Parser(object):

    def __init__(self):
        self.apis = API.load_all()
        self.queue = Queue()
        for api in self.apis:
            self.queue.put(api)

    def api_getter(func):
        def debug_wrapper(*args, **kwargs):
            self = args[0]
            if not self.queue.empty():
                api = self.queue.get()
                kwargs['api'] = api
                result = func(*args, **kwargs)
                self.queue.put(api)
                return result
            return None
        return debug_wrapper

    def error_handler(func):
        def error_handler_wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception as e:
                warnings.warn("PARSER: " + str(e))
                time.sleep(2)
        return error_handler_wrapper

    @error_handler
    @api_getter
    def _get_user_followers(self, user_id, max_id="", api=None):
        if api is None:
            raise ("No API instance was passed")
        if not api.getUserFollowers(user_id, maxid=max_id):
            raise ("Broken API")
        result = api.LastJson
        if result["big_list"] is False:
            return (result["users"], None)

        next_max_id = result["next_max_id"]
        return (result["users"], next_max_id)

    @error_handler
    @api_getter
    def _get_user_info(self, user_id, api=None):
        if api is None:
            raise ("No API instance was passed")
        if not api.getUsernameInfo(user_id):
            raise ("Broken API")
        if "user" in api.LastJson:
            return api.LastJson["user"]
        return None

    def get_user_info(self, user_id):
        return self._get_user_info(user_id)

    def get_user_followers(self, user_id):
        result = []
        max_id = ""
        user_info = self._get_user_info(user_id)
        if user_info is None:
            return []

        total_followers = user_info["follower_count"]
        with tqdm(total=total_followers, desc="Getting followers", leave=False) as pbar:
            while True:
                resp = self._get_user_followers(user_id, max_id)
                if resp is None:
                    continue
                users, max_id = resp

                pbar.update(len(users))
                result.extend(users)

                if max_id is None:
                    break

        result = result[:total_followers]
        # assert(len(result) == len(set(result)))
        return result



    # def get_total_followers(self, user_id):
    #
    #
    #
    #     followers = []
    #     next_max_id = ''
    #     self.getUsernameInfo(usernameId)
    #     if "user" in self.LastJson:
    #         total_followers = self.LastJson["user"]['follower_count']
    #     else:
    #         return False
    #     with tqdm(total=total_followers, desc="Getting followers", leave=False) as pbar:
    #         while True:
    #             self.getUserFollowers(usernameId, next_max_id)
    #             temp = self.LastJson
    #             try:
    #                 pbar.update(len(temp["users"]))
    #                 for item in temp["users"]:
    #                     followers.append(item)
    #                 if len(temp["users"]) == 0 or len(followers) >= total_followers:
    #                     return followers[:total_followers]
    #             except:
    #                 return followers[:total_followers]
    #             if temp["big_list"] is False:
    #                 return followers[:total_followers]
    #             next_max_id = temp["next_max_id"]

{'fraction_non_alphanumeric': 0.10275477437890823, 'fraction_numerical': 0.0008450228156160216, 'mean_word_length': 1.6729900632339656, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4706913', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1699, 'n_words': 381}