<commit_before>import argparse
import contextlib
import cStringIO
import errno
import itertools
import json
import os
import re
import shutil
import sys
import zipfile

import bs4
import requests

BASE_URL = 'http://www.mathworks.com/matlabcentral/fileexchange'
TAG_REGEX = re.compile(r'([\w\s]+)(?:\(\d+\))?')


def get_soup(*args, **kwargs):
    return bs4.BeautifulSoup(requests.get(*args, **kwargs).text)


# From http://stackoverflow.com/a/600612/281108
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != errno.EEXIST or not os.path.isdir(path):
            raise


def extractall(archive, path):
    "A version of ZipFile.extractall that doesn't choke on utf-8 filenames."
    for filename in archive.namelist():
        dest_filename = os.path.join(path, filename.decode('cp437'))
        if dest_filename.endswith('/'):
            mkdir_p(dest_filename)
            continue
        mkdir_p(os.path.dirname(dest_filename))
        with archive.open(filename) as src, open(dest_filename, 'w') as dest:
            shutil.copyfileobj(src, dest)


class Project(object):
    def __init__(self, url_or_name):
        if url_or_name.startswith(BASE_URL):
            self.url = url_or_name
        else:
            self.url = '%s/%s' % (BASE_URL, url_or_name)

    def __repr__(self):
        return '<Project: %s>' % self.name

    @property
    def name(self):
        return self.url.split('/')[-1]

    def download(self, path='.', extract_archives=True):
        download_url = '%s?download=true' % self.url
        response = requests.get(download_url)
        download_filename = response.url.split('/')[-1]
        if extract_archives and download_filename.endswith('.zip'):
            try:
                with zipfile.ZipFile(cStringIO.StringIO(response.content)) as f:
                    extractall(f, path)
            except zipfile.BadZipfile:
                return False
        else:
            with open(os.path.join(path, download_filename), 'w') as f:
                f.write(response.text.encode('utf-8'))
        return True

    def get_metadata(self):
        metadata = {}
        soup = get_soup(self.url)

        tags_div = soup.find('div', id='all_tags')
        raw_tags = tags_div.get_text().strip().split('\n\n')[0].split(', ')
        tags = []
        for tag in raw_tags:
            match = TAG_REGEX.match(tag)
            if match:
                tags.append(match.group(1))
        metadata['tags'] = tags

        details = soup.find('div', id='details')

        metadata['title'] = details.h1.text
        metadata['summary'] = details.find('p', id='summary').text

        author_link = details.find('p', id='author').a<commit_msg>Also swallow ConnectionErrors from requests.<commit_after>import argparse
import contextlib
import cStringIO
import errno
import itertools
import json
import os
import re
import shutil
import sys
import zipfile

import bs4
import requests

BASE_URL = 'http://www.mathworks.com/matlabcentral/fileexchange'
TAG_REGEX = re.compile(r'([\w\s]+)(?:\(\d+\))?')


def get_soup(*args, **kwargs):
    return bs4.BeautifulSoup(requests.get(*args, **kwargs).text)


# From http://stackoverflow.com/a/600612/281108
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != errno.EEXIST or not os.path.isdir(path):
            raise


def extractall(archive, path):
    "A version of ZipFile.extractall that doesn't choke on utf-8 filenames."
    for filename in archive.namelist():
        dest_filename = os.path.join(path, filename.decode('cp437'))
        if dest_filename.endswith('/'):
            mkdir_p(dest_filename)
            continue
        mkdir_p(os.path.dirname(dest_filename))
        with archive.open(filename) as src, open(dest_filename, 'w') as dest:
            shutil.copyfileobj(src, dest)


class Project(object):
    def __init__(self, url_or_name):
        if url_or_name.startswith(BASE_URL):
            self.url = url_or_name
        else:
            self.url = '%s/%s' % (BASE_URL, url_or_name)

    def __repr__(self):
        return '<Project: %s>' % self.name

    @property
    def name(self):
        return self.url.split('/')[-1]

    def download(self, path='.', extract_archives=True):
        download_url = '%s?download=true' % self.url
        try:
            response = requests.get(download_url)
        except requests.exceptions.ConnectionError:
            return False
        download_filename = response.url.split('/')[-1]
        if extract_archives and download_filename.endswith('.zip'):
            try:
                with zipfile.ZipFile(cStringIO.StringIO(response.content)) as f:
                    extractall(f, path)
            except zipfile.BadZipfile:
                return False
        else:
            with open(os.path.join(path, download_filename), 'w') as f:
                f.write(response.text.encode('utf-8'))
        return True

    def get_metadata(self):
        metadata = {}
        soup = get_soup(self.url)

        tags_div = soup.find('div', id='all_tags')
        raw_tags = tags_div.get_text().strip().split('\n\n')[0].split(', ')
        tags = []
        for tag in raw_tags:
            match = TAG_REGEX.match(tag)
            if match:
                tags.append(match.group(1))
        metadata['tags'] = tags

        details = soup.find('div', id='details')

        metadata['title'] = details.h1.text
        metadata['summary'] = details.find('p', id='summary').text

        author_link = details.find('p', id='author').a
{'fraction_non_alphanumeric': 0.13146127705240565, 'fraction_numerical': 0.008585226256483634, 'mean_word_length': 2.453983940704138, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3155644', 'n_tokens_mistral': 1804, 'n_tokens_neox': 1716, 'n_words': 372}