<commit_before>from mrjob.job import MRJob

class FollowerHistogram(MRJob):
    """
    A Map/Reduce job to count the number of followers each user has
    and then create a count of how many users have how many followers.
    
    Input::
    
        <user_id>\t<follower_id>
    
    Output::
    
        <num_followers>\t<num_users_who_have_that_mahy_followers>
    
    For example, for the following input data::
    
        1\t2
        1\t3
        1\t3
        2\t1
        3\t1
        3\t2
        3\t3
    
    The following would be output::
    
        2\t2
        3\t1
    
    That is to say that for this data set, there are 2 users who have 2
    followers and 1 user that has 3 followers.
    """
    # Map Step 1: Read the file in <user_id>\t<follower_id> and increment
    # the follower count for that <user_id>.
    def increment_followers(self, key, line):
        yield int(line.split('\t')[0]), 1

    # Reduce Step 1: Sum the number of followers for each <user_id>
    def sum_followers(self, id, followers):
        yield id, sum(followers)

    # Map Step 2: Increment the number of users who have the given number
    # of followers.
    def increment_users_per_follow_count(self, id, followers):
        yield followers, 1

    # Reduce Step 2: Sum the follower count for each number of followers
    def sum_users_per_follow_count(self, follow_count, users_count):
        yield follow_count, sum(users_count)

    def steps(self):
        return [self.mr(self.increment_followers, self.sum_followers),
                self.mr(self.increment_users_per_follow_count, 
                self.sum_users_per_follow_count)]

if __name__ == '__main__':
    FollowerHistogram.run()<commit_msg>Potential performance improvement for PyPy though it did not yield a speed boost in a one-off test.<commit_after>from mrjob.job import MRJob

class FollowerHistogram(MRJob):
    """
    A Map/Reduce job to count the number of followers each user has
    and then create a count of how many users have how many followers.
    
    Input::
    
        <user_id>\t<follower_id>
    
    Output::
    
        <num_followers>\t<num_users_who_have_that_mahy_followers>
    
    For example, for the following input data::
    
        1\t2
        1\t3
        1\t3
        2\t1
        3\t1
        3\t2
        3\t3
    
    The following would be output::
    
        2\t2
        3\t1
    
    That is to say that for this data set, there are 2 users who have 2
    followers and 1 user that has 3 followers.
    """
    # Map Step 1: Read the file in <user_id>\t<follower_id> and increment
    # the follower count for that <user_id>.
    def increment_followers(self, key, line):
        yield int(line.split('\t')[0]), 1

    # Reduce Step 1: Sum the number of followers for each <user_id>
    def sum_followers(self, id, followers):
        x = 0
        for f in followers:
            x += f
        yield id, x

    # Map Step 2: Increment the number of users who have the given number
    # of followers.
    def increment_users_per_follow_count(self, id, followers):
        yield followers, 1

    # Reduce Step 2: Sum the follower count for each number of followers
    def sum_users_per_follow_count(self, follow_count, users_count):
        x = 0
        for c in users_count:
            x += c
        yield follow_count, x

    def steps(self):
        return [self.mr(self.increment_followers, self.sum_followers),
                self.mr(self.increment_users_per_follow_count, 
                self.sum_users_per_follow_count)]

if __name__ == '__main__':
    FollowerHistogram.run()
{'fraction_non_alphanumeric': 0.07344008834897847, 'fraction_numerical': 0.016565433462175594, 'mean_word_length': 2.261026102610261, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2197377', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1117, 'n_words': 377}