<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Twitter influencer scraping utility.

Scrape profile usernames of the most influencial Twitter accounts from
a website and then save the output. The usernames can be stored in a database
and used to lookup tweets by those users.

The source is webpages on socialblade.com, which covers the profiles with
highest followers, highest following count, most tweets or most engagements.

These Twitter influencers tend to be policitians, companies, musicians, actors
and so on. They are likely to talk to each other and to possibly talk
about trending topics - they may even be the reason that a topic becomes
trending or they may simply be sharing opinion on what is already a trending
topic.
"""
import requests
from bs4 import BeautifulSoup


CATEGORIES = ['followers', 'following', 'tweets', 'engagements']


def getUsernamesInCategory(category, count=100):
    """
    Get top Twitter usernames from website for a given category.
<commit_msg>Update influencerScraper.py     Make executable.     Add usage documentation.<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Twitter influencer scraping utility.

Usage:
    $ ./utils/influencerScraper.py > var/influencers.txt

Scrape profile usernames of the most influencial Twitter accounts from
a website and then save the output. The usernames can be stored in a database
and used to lookup tweets by those users.

The source is webpages on socialblade.com, which covers the profiles with
highest followers, highest following count, most tweets or most engagements.

These Twitter influencers tend to be policitians, companies, musicians, actors
and so on. They are likely to talk to each other and to possibly talk
about trending topics - they may even be the reason that a topic becomes
trending or they may simply be sharing opinion on what is already a trending
topic.
"""
import requests
from bs4 import BeautifulSoup


CATEGORIES = ['followers', 'following', 'tweets', 'engagements']


def getUsernamesInCategory(category, count=100):
    """
    Get top Twitter usernames from website for a given category.

{'fraction_non_alphanumeric': 0.06159589360709286, 'fraction_numerical': 0.0055996266915538965, 'mean_word_length': 4.469387755102041, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4368439', 'n_tokens_mistral': 586, 'n_tokens_neox': 564, 'n_words': 287}