<commit_before>#!/usr/bin/env python

import argparse
import getpass
from github import Github
import logging
import sys
import urllib2


ORG_NAME = 'openstack'
REQ_FILES = ['requirements.txt',
             'test-requirements.txt',
             'other-requirements.txt']
REPO_RAW_URL = "https://raw.githubusercontent.com/" + ORG_NAME


def create_parser():
    """Returns argument parser."""

    parser = argparse.ArgumentParser(
        description='Identify which OpenStack projects use specific library.')
    parser.add_argument(
        '--library', dest='library', help='Library name')
    parser.add_argument(
        '--user', dest='user', help='Github username')
    parser.add_argument(
        '--debug', dest='debug', action='store_true', help='Turn on debug')

    return parser


def check_library_used(repo, lib):
    """Checks if the library is used by the project.

    :param repo: Github repo object
    :param library: the library name.
    """

    found = False
    for req_file in REQ_FILES:
        try:
            req_list = urllib2.urlopen(
                REPO_RAW_URL +
                "/{}/master/{}".format(repo.name, req_file)).read()
        except Exception:
            req_list = ""

        if req_list.find(lib) != -1:
            found = True

    if found:
        logging.debug("{} is using {} :)".format(repo.name, lib))
    else:
        logging.debug("{} is not using {} :(".format(repo.name, lib))

    return found


def summary(repos_counter, lib, repos_use_library):
    """Print summary."""

    logging.info("=========== Summary ===========")
    logging.info("Scanned {} of repos".format(repos_counter))
    if not repos_use_library:
        logging.info("Couldn't find any repos that use this library")
    else:
        logging.info("Repos using '{}':".format(lib))
        for repo in repos_use_library:
            logging.info("Project: {}".format(repo))


def main():
    """Miko main loop."""

    repos_counter = 0
    repos_use_library = []

    parser = create_parser()
    args = parser.parse_args()
    
    if not args.debug:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.DEBUG)

    if not args.library:
        parser.error("Must specify library name")

    if not args.user:
        gith = Github()
    else:
        password = <PASSWORD>("Enter password: ")
        gith = Github(args.user, password)
    openstack_org = gith.get_organization(ORG_NAME)

    logging.info("Start scanning. This might take a while!")
    for repo in openstack_org.get_repos():
        repos_counter += 1
        if check_library_used(repo, args.library):
            repos_use_library.append(repo.name)

    summary(repos_counter, args.library, repos_use_library)


if __name__ == '__main__':

    sys.exit(main())
<commit_msg>Use eventlet for faster processing<commit_after>#!/usr/bin/env python

import argparse
import eventlet
from eventlet.green import urllib2
import getpass
from github import Github
import logging
import sys


ORG_NAME = 'openstack'
REQ_FILES = ['requirements.txt',
             'test-requirements.txt',
             'other-requirements.txt']
REPO_RAW_URL_PREFIX = "https://raw.githubusercontent.com/" + ORG_NAME


def create_parser():
    """Returns argument parser."""

    parser = argparse.ArgumentParser(
        description='Identify which OpenStack projects use specific library.')
    parser.add_argument(
        '--library', dest='library', help='Library name')
    parser.add_argument(
        '--user', dest='user', help='Github username')
    parser.add_argument(
        '--debug', dest='debug', action='store_true', help='Turn on debug')

    return parser


def fetch_lib_list(url):
    """Returns the content of the url.

    :param url: the url which contains the libraries list
    """
    try:
        return urllib2.urlopen(url).read()
    except Exception:
        logging.debug("No such page: {}".format(url))
        return None


def check_project(project):
    """Return the libraries list and the name of the project.

    :param project: Github repo object
    """

    logging.info("Searching in {}".format(project.keys()[0]))
    pool = eventlet.GreenPool()
    for lib_list in pool.imap(fetch_lib_list, project.values()[0]):
        return lib_list, project.keys()[0]


def get_projects_using_library(repos, lib):
    """Returns list of projects which using the specific library.

    :param repos: list of Github repo objects
    :param lib: the name of the chosen library
    """
    pool = eventlet.GreenPool()
    projects_using_lib = []

    for lib_list, project in pool.imap(check_project, repos):
        if lib_list:
            if lib_list.find(lib) != -1:
                projects_using_lib.append(project)

    return projects_using_lib


def summary(repos_counter, lib, proj_using_lib):
    """Output summary."""

    logging.info("=========== Summary ===========")
    logging.info("Scanned {} of repos".format(repos_counter))
    for project in proj_using_lib:
            logging.info("{} is using {}".format(project, lib))


def main():
    """Miko main loop."""

    repos = []

    parser = create_parser()
    args = parser.parse_args()

    if not args.debug:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.DEBUG)

    if not args.library:
        parser.error("Must specify library name")

    if not args.user:
        gith = Github()
    else:
        password = <PASSWORD>("Enter password: ")
        gith = Github(args.user, password)
    openstack_org = gith.get_organization(ORG_NAME)

    for repo in openstack_org.get_repos():
        repos.append({repo.name:
                      [REPO_RAW_URL_PREFIX +
                       "/{}/master/{}".format(
                           repo.name, req_file) for req_file in REQ_FILES]})

    logging.info("Start scanning. This might take a while!")
    proj_using_lib = get_projects_using_library(repos, args.library)

    summary(len(repos), args.library, proj_using_lib)

if __name__ == '__main__':

    sys.exit(main())

{'fraction_non_alphanumeric': 0.12755853938103814, 'fraction_numerical': 0.0018012117242508596, 'mean_word_length': 2.8246712586098934, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3530739', 'n_tokens_mistral': 1884, 'n_tokens_neox': 1786, 'n_words': 466}