<commit_before>__author__ = 'jkessel'

from bs4 import BeautifulSoup
import os
from urllib.error import HTTPError
import urllib.request

# Be sure to create the folder beforehand
output_folder = 'c:\\microsoft_books'

books_url = ('http://blogs.msdn.com/b/mssmallbiz/archive/2015/07/07/i-m-giving-away-millions-of-free-microsoft-ebooks-'
'again-including-windows-10-windows-8-1-windows-8-windows-7-office-2013-office-365-sharepoint-2013-dynamics-crm-'
'powershell-exchange-server-lync-2013-system-center-azure-clo.aspx')
request = urllib.request.urlopen(books_url)
page = request.read()

soup = BeautifulSoup(page, 'html.parser')

table = soup.find_all('table')[2]
paragraphs = table.find_all('p')

print('Starting download of books...')

i = 1

for paragraph in paragraphs:
    no_of_books = len(paragraphs)
    anchors = paragraph.find_all('a')
    print('Downloading book', i, 'of', no_of_books)
    for anchor in anchors:
        href = anchor.get('href')
#        print(href)
        try:
            download = urllib.request.urlopen(href)
        except HTTPError as err:
            print('HTTP Error: {0}'.format(err))

        url = download.geturl()
        filename = url.rsplit('/', 1)
        converted_filename = urllib.parse.unquote_plus(filename[1])
        if not os.path.isfile(output_folder + '\\' + converted_filename):
            try:
                file = open(output_folder + '\\' + converted_filename, mode='wb')
                print('     Downloading file', converted_filename, '...')
                file.write(download.read())
                file.close()
            except OSError as err:<commit_msg>Changed URL, started to add file size functionality.<commit_after>__author__ = 'jkessel'

from bs4 import BeautifulSoup
import os
from urllib.error import HTTPError
import urllib.request

# Be sure to create the folder beforehand
output_folder = 'c:\\microsoft_books'
if not os.path.exists(output_folder):
    os.makedirs(output_folder)

"""
books_url = ('http://blogs.msdn.com/b/mssmallbiz/archive/2015/07/07/i-m-giving-away-millions-of-free-microsoft-ebooks-'
             'again-including-windows-10-windows-8-1-windows-8-windows-7-office-2013-office-365-sharepoint-2013-'
             'dynamics-crm-powershell-exchange-server-lync-2013-system-center-azure-clo.aspx')
"""
books_url = ('https://blogs.msdn.microsoft.com/mssmallbiz/2016/07/10/free-thats-right-im-giving-away-millions-of-free-'
             'microsoft-ebooks-again-including-windows-10-office-365-office-2016-power-bi-azure-windows-8-1-office-2013'
             '-sharepoint-2016-sha/')
request = urllib.request.urlopen(books_url)
page = request.read()

soup = BeautifulSoup(page, 'html.parser')

#table = soup.find_all('table')[2]
table = soup.find_all('table')[0]

paragraphs = table.find_all('p')

print('Starting download of books...')

i = 1

for paragraph in paragraphs:
    no_of_books = len(paragraphs)
    anchors = paragraph.find_all('a')
    print('Downloading book', i, 'of', no_of_books)
    for anchor in anchors:
        href = anchor.get('href')
#        print(href)
        try:
            download = urllib.request.urlopen(href)
            content_length = download.info()['Content-Length']
            print('Filesize:', content_length)
        except HTTPError as err:
            print('HTTP Error: {0}'.format(err))

        url = download.geturl()
        filename = url.rsplit('/', 1)
        converted_filename = urllib.parse.unquote_plus(filename[1])
        if not os.path.isfile(output_folder + '\\' + converted_filename):
            try:
                file = open(output_folder + '\\' + converted_filename, mode='wb')
                print('     Downloading file', converted_filename, '...')
                file.write(download.read())
                file.close()
            except OSError as err:
{'fraction_non_alphanumeric': 0.13931646230106967, 'fraction_numerical': 0.025567440647012784, 'mean_word_length': 2.9162410623084782, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3251146', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1226, 'n_words': 220}