<commit_before>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import codecs
import json
import re
import tqdm
import urllib
from urllib2 import urlopen

popular_packages = []

def get_popular_packages():
    limit = 200 #10000 Packages
    package = 1
    seed = 'http://pypi-ranking.info/alltime?page='
    print("Getting Popular Packages ...")
    for i in tqdm.tqdm(range(1, limit+1)):
        data = urlopen(seed + str(i)).read()
        soup = BeautifulSoup(data, "html.parser")
        spans = soup.find_all('span', {'class' : 'list_title'})
        packages = [span.get_text() for span in spans]
        for element in packages:
            if element not in popular_packages:
                popular_packages.append(element.encode('utf-8'))
    main()<commit_msg>Increased number of pypi packages from pypi-rankings to 15,000<commit_after>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import codecs
import json
import re
import tqdm
import urllib
from urllib2 import urlopen

popular_packages = []

def get_popular_packages():
    limit = 300 #15000 Packages
    package = 1
    seed = 'http://pypi-ranking.info/alltime?page='
    print("Getting Popular Packages ...")
    for i in tqdm.tqdm(range(1, limit+1)):
        data = urlopen(seed + str(i)).read()
        soup = BeautifulSoup(data, "html.parser")
        spans = soup.find_all('span', {'class' : 'list_title'})
        packages = [span.get_text() for span in spans]
        for element in packages:
            if element not in popular_packages:
                popular_packages.append(element.encode('utf-8'))
    main()
{'fraction_non_alphanumeric': 0.12432106216053108, 'fraction_numerical': 0.023536511768255886, 'mean_word_length': 3.004830917874396, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4409457', 'n_tokens_mistral': 564, 'n_tokens_neox': 523, 'n_words': 143}