<commit_before>class Tweet():
    def __init__(self, status):
        self.status = status
        self.rt = hasattr(self.status, 'retweeted_status')

        self.tid = status.id
        if self.rt:
            self.status = status.retweeted_status
            self.o_status = status

        self.text = self.status.text
        self.user = status.user
        self._list_ressource()

    def _list_ressource(self):
        self.ent = {}
        self.ent['vid'] = []
        self.ent['pic'] = []
        self.ent['url'] = []
        self.ent['profile'] = [self.status.user.profile_image_url_https.
                               replace('normal', 'bigger')]
        if self.rt:
            self.ent['profile'].\
                    append(self.o_status.user.profile_image_url_https.
                           replace('normal', 'bigger'))

        if hasattr(self.status, 'extended_entities'):
            for i in self.status.extended_entities['media']:
                if (i['type'] == 'animated_gif' or i['type'] == 'video'):
                    self.ent['vid'].append((i['url'], self.choose_video(i)))
                else:
                    self.ent['pic'].append((i['url'],
                                            i['media_url_https']+':orig'))
        if 'urls' in self.status.entities:
            for i in self.status.entities['urls']:
                self.ent['url'].append((i['indices'], i['display_url'],
                                       i['expanded_url']))

    def __lt__(self, other):
        return self.tid < other.tid

    def choose_video(self, vids):
        chosen = vids['expanded_url']
        max_br = -1
        for i in vids['video_info']['variants']:
            try:
                if i['bitrate'] > max_br:<commit_msg>Bugfix to get original size images<commit_after>class Tweet():
    def __init__(self, status):
        self.status = status
        self.rt = hasattr(self.status, 'retweeted_status')

        self.tid = status.id
        if self.rt:
            self.status = status.retweeted_status
            self.o_status = status

        self.text = self.status.text
        self.user = status.user
        self._list_ressource()

    def _list_ressource(self):
        self.ent = {}
        self.ent['vid'] = []
        self.ent['pic'] = []
        self.ent['url'] = []
        self.ent['profile'] = [self.status.user.profile_image_url_https.
                               replace('_normal', '')]
        if self.rt:
            self.ent['profile'].\
                    append(self.o_status.user.profile_image_url_https.
                           replace('normal', ''))

        if hasattr(self.status, 'extended_entities'):
            for i in self.status.extended_entities['media']:
                if (i['type'] == 'animated_gif' or i['type'] == 'video'):
                    self.ent['vid'].append((i['url'], self.choose_video(i)))
                else:
                    self.ent['pic'].append((i['url'],
                                            i['media_url_https']+':orig'))
        if 'urls' in self.status.entities:
            for i in self.status.entities['urls']:
                self.ent['url'].append((i['indices'], i['display_url'],
                                       i['expanded_url']))

    def __lt__(self, other):
        return self.tid < other.tid

    def choose_video(self, vids):
        chosen = vids['expanded_url']
        max_br = -1
        for i in vids['video_info']['variants']:
            try:
                if i['bitrate'] > max_br:
{'fraction_non_alphanumeric': 0.14544425580079232, 'fraction_numerical': 0.0005659309564233164, 'mean_word_length': 1.62630014858841, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2522381', 'n_tokens_mistral': 1032, 'n_tokens_neox': 1010, 'n_words': 180}