<commit_before>import pprint
import re
import requests

from github import Github, UnknownObjectException

class APIGrabber():
    def __init__(self, api_key=None):
        self.gh = ''
        self.token = ''
        self.user = 'getpelican'
        self.repo_name = 'pelican-themes'
        self.main_repo = None
        self.api_key = api_key

    def process(self):
        """Extracts a list of dicts from the Pelican Themes repo.  Each dict
        provides basic info about a Pelican theme."""

        if self.api_key:
            self.gh = Github(self.api_key)
        else:
            print "NO API KEY"

        theme_list = self.fetch_content()
        theme_list = self.fetch_related_images(theme_list)

        return theme_list

    def fetch_content(self):
        """
        Gets the list of themes from the pelican themes repo, then gets
        possible images from each theme.
        """

        self.main_repo = self.gh.get_repo('{}/{}'.format(self.user, self.repo_name))
        contents = self.main_repo.get_contents('/').raw_data

        theme_list = []
        # Get theme location, whether directory or submodule
        for theme in contents:
            push = False
            if theme['type'] == 'dir':
                push = True
            elif theme['type'] == 'file' and theme['size'] == 0:
                push = True

            if push:
                theme_list.append(self.info_from_content(theme))

        return theme_list

    def fetch_related_images(self, theme_list):
        """ Given info about a repo/path on github, get any .jpg or .png files
        listed in that location."""
        failures = []

        theme_list = theme_list[0:30]

        for t in theme_list:
            print 'tick..'
            contents = []

            this_repo = self.get_repo(t['user'], t['repo'])
            if this_repo is not None:
                contents = this_repo.get_contents(t['path']).raw_data

            # Find image files for this repo / path
            for c in contents:
                m = re.search('.png|.jpg$', c['name'], re.IGNORECASE)
                if m:
                    t['image_urls'].append(c['download_url'])<commit_msg>Prevent theme list from being truncated.<commit_after>import pprint
import re
import requests

from github import Github, UnknownObjectException

class APIGrabber():
    def __init__(self, api_key=None):
        self.gh = ''
        self.token = ''
        self.user = 'getpelican'
        self.repo_name = 'pelican-themes'
        self.main_repo = None
        self.api_key = api_key

    def process(self):
        """Extracts a list of dicts from the Pelican Themes repo.  Each dict
        provides basic info about a Pelican theme."""

        if self.api_key:
            self.gh = Github(self.api_key)
        else:
            print "NO API KEY"

        theme_list = self.fetch_content()
        theme_list = self.fetch_related_images(theme_list)

        return theme_list

    def fetch_content(self):
        """
        Gets the list of themes from the pelican themes repo, then gets
        possible images from each theme.
        """

        self.main_repo = self.gh.get_repo('{}/{}'.format(self.user, self.repo_name))
        contents = self.main_repo.get_contents('/').raw_data

        theme_list = []
        # Get theme location, whether directory or submodule
        for theme in contents:
            push = False
            if theme['type'] == 'dir':
                push = True
            elif theme['type'] == 'file' and theme['size'] == 0:
                push = True

            if push:
                theme_list.append(self.info_from_content(theme))

        return theme_list

    def fetch_related_images(self, theme_list):
        """ Given info about a repo/path on github, get any .jpg or .png files
        listed in that location."""
        failures = []

        for t in theme_list:
            print 'tick..'
            contents = []

            this_repo = self.get_repo(t['user'], t['repo'])
            if this_repo is not None:
                contents = this_repo.get_contents(t['path']).raw_data

            # Find image files for this repo / path
            for c in contents:
                m = re.search('.png|.jpg$', c['name'], re.IGNORECASE)
                if m:
                    t['image_urls'].append(c['download_url'])
{'fraction_non_alphanumeric': 0.10232452142206017, 'fraction_numerical': 0.0011394712853236098, 'mean_word_length': 1.8592833876221497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '426557', 'n_tokens_mistral': 1258, 'n_tokens_neox': 1214, 'n_words': 354}