<commit_before>import logging
from typing import Optional
import queue
import threading
import sys
import pytz
from datetime import datetime, timedelta
from bot.lib.statusdb import Addstatus

from conversation.models import Treedj, Tweetdj
from conversation.models import create_leaf, create_tree
from community.helpers import get_community_twitter_tweepy_api
from community.models import Community
from bot.addstatusdj import addstatus
from conversation.tree.utils import get_community_roots

logger = logging.getLogger(__name__)

class ReplyCrawler(object):
    """
    A class used to represent an Twitter tree crawler based on the search API

    ...

    Attributes
    ----------
    community : Community
        a Community object, the crawler will build statuse trees of this community
    api : Tweepy API object
        Tweepy API object
    root_id : int
        status id of the root of the tree
    q : PriorityQueue
        unused
    since_id: int
        search will start from this id
    tree_init: List[int]
        list of ids of descendants already known for this tree root

    Methods
    -------
    TODO

    """
    max_count = 100
    
    def __init__(self, community):
        self.community = community
        self.api = get_community_twitter_tweepy_api(
            self.community,
            backend=False
        )
        #self.root_id = root_id
        #self.q = queue.PriorityQueue(maxsize=0)
        #self.since_id = root_id
        #self.tree_init = ReplyCrawler.root_descendants(
        #    root_id,
        #    include_self=True
        #)

    @staticmethod
    def root_descendants(root_id, include_self: bool = True):
        """Return list of descendants' ids
        """
        try:
            root = Treedj.objects.get(statusid=root_id)
        except Treedj.DoesNotExist:
            root = create_tree(root_id)
        if root:
            return sorted( 
                list(
                    root.get_descendants(include_self=include_self)
                    .values_list("statusid", flat=True)
                )
            )

    def get_screen_name(self, status_id):
        try:
            status = Tweetdj.objects.get(statusid=status_id)
            return status.json["user"]["screen_name"]
        except Tweetdj.DoesNotExist:
            logger.debug(f"Tweet {status_id} is not present in database.")
            addstatus(status_id, self.community.account.username)
            if Tweetdj.objects.filter(statusid=status_id).exists():
                return self.get_screen_name(status_id)
        except TypeError:
            if status.json is None:
                logger.debug(f"Tweet {status_id} json field is null.")
            return
        except KeyError:
            logger.debug(f"Tweet {status_id} json is buggy.")
            return

    def get_replies(self, status_id, since_id=None):
        screen_name = self.get_screen_name(status_id)
        if not since_id:
            since_id = status_id
        replies = self.search_reply_to(
            screen_name,
            community=self.community,
            since_id = since_id
        )
        logger.debug(f"{[(r.id, r.text) for r in replies]}")
        if len(replies) == ReplyCrawler.max_count:
            while True:
                hi = 0
                for x in (reply['id'] for reply in replies):
                    hi = max(x,hi)<commit_msg>fix comprehension bug<commit_after>import logging
from typing import Optional
import queue
import threading
import sys
import pytz
from datetime import datetime, timedelta
from bot.lib.statusdb import Addstatus

from conversation.models import Treedj, Tweetdj
from conversation.models import create_leaf, create_tree
from community.helpers import get_community_twitter_tweepy_api
from community.models import Community
from bot.addstatusdj import addstatus
from conversation.tree.utils import get_community_roots

logger = logging.getLogger(__name__)

class ReplyCrawler(object):
    """
    A class used to represent an Twitter tree crawler based on the search API

    ...

    Attributes
    ----------
    community : Community
        a Community object, the crawler will build statuse trees of this community
    api : Tweepy API object
        Tweepy API object
    root_id : int
        status id of the root of the tree
    q : PriorityQueue
        unused
    since_id: int
        search will start from this id
    tree_init: List[int]
        list of ids of descendants already known for this tree root

    Methods
    -------
    TODO

    """
    max_count = 100
    
    def __init__(self, community):
        self.community = community
        self.api = get_community_twitter_tweepy_api(
            self.community,
            backend=False
        )
        #self.root_id = root_id
        #self.q = queue.PriorityQueue(maxsize=0)
        #self.since_id = root_id
        #self.tree_init = ReplyCrawler.root_descendants(
        #    root_id,
        #    include_self=True
        #)

    @staticmethod
    def root_descendants(root_id, include_self: bool = True):
        """Return list of descendants' ids
        """
        try:
            root = Treedj.objects.get(statusid=root_id)
        except Treedj.DoesNotExist:
            root = create_tree(root_id)
        if root:
            return sorted( 
                list(
                    root.get_descendants(include_self=include_self)
                    .values_list("statusid", flat=True)
                )
            )

    def get_screen_name(self, status_id):
        try:
            status = Tweetdj.objects.get(statusid=status_id)
            return status.json["user"]["screen_name"]
        except Tweetdj.DoesNotExist:
            logger.debug(f"Tweet {status_id} is not present in database.")
            addstatus(status_id, self.community.account.username)
            if Tweetdj.objects.filter(statusid=status_id).exists():
                return self.get_screen_name(status_id)
        except TypeError:
            if status.json is None:
                logger.debug(f"Tweet {status_id} json field is null.")
            return
        except KeyError:
            logger.debug(f"Tweet {status_id} json is buggy.")
            return

    def get_replies(self, status_id, since_id=None):
        screen_name = self.get_screen_name(status_id)
        if not since_id:
            since_id = status_id
        replies = self.search_reply_to(
            screen_name,
            community=self.community,
            since_id = since_id
        )
        logger.debug(f"{[(r.id, r.text) for r in replies]}")
        if len(replies) == ReplyCrawler.max_count:
            while True:
                hi = 0
                for x in [reply['id'] for reply in replies]:
                    hi = max(x,hi)
{'fraction_non_alphanumeric': 0.07615407233166716, 'fraction_numerical': 0.001470155836518671, 'mean_word_length': 2.0602789023841654, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7571073', 'n_tokens_mistral': 1978, 'n_tokens_neox': 1895, 'n_words': 513}