<commit_before>#!/usr/bin/env python

"""Software Carpentry Windows Installer

Helps mimic a *nix environment on Windows with as little work as possible.

The script:
* Installs GNU Make and makes it accessible from msysGit
* Installs nano and makes it accessible from msysGit
* Installs SQLite and makes it accessible from msysGit
* Creates a ~/nano.rc with links to syntax highlighting configs
* Provides standard nosetests behavior for msysGit
* Adds R's bin directory to the path (if we can find it)

To use:

1. Install Python, IPython, and Nose.  An easy way to do this is with
   the Anaconda Python distribution
   http://continuum.io/downloads
2. Install msysGit
   https://github.com/msysgit/msysgit/releases
3. Install R (if your workshop uses R)
   http://cran.r-project.org/bin/windows/base/rw-FAQ.html#Installation-and-Usage
4. Run swc-windows-installer.py.
   You should be able to simply double click the file in Windows

"""

import glob
import hashlib
try:  # Python 3
    from io import BytesIO as _BytesIO
except ImportError:  # Python 2
    from StringIO import StringIO as _BytesIO
import logging
import os
import re
import sys
import tarfile
try:  # Python 3
    from urllib.request import urlopen as _urlopen
except ImportError:  # Python 2
    from urllib2 import urlopen as _urlopen
import zipfile


__version__ = '0.1'

LOG = logging.getLogger('swc-windows-installer')
LOG.addHandler(logging.StreamHandler())
LOG.setLevel(logging.INFO)


if sys.version_info >= (3, 0):  # Python 3
    open3 = open
else:
    def open3(file, mode='r', newline=None):
        if newline:
            if newline != '\n':
                raise NotImplementedError(newline)
            f = open(file, mode + 'b')
        else:
            f = open(file, mode)
        return f


def download(url, sha1):
    """Download a file and verify its hash"""
    LOG.debug('download {}'.format(url))
    r = _urlopen(url)
    byte_content = r.read()<commit_msg>swc-windows-installer.py: Bump version to 0.2<commit_after>#!/usr/bin/env python

"""Software Carpentry Windows Installer

Helps mimic a *nix environment on Windows with as little work as possible.

The script:
* Installs GNU Make and makes it accessible from msysGit
* Installs nano and makes it accessible from msysGit
* Installs SQLite and makes it accessible from msysGit
* Creates a ~/nano.rc with links to syntax highlighting configs
* Provides standard nosetests behavior for msysGit
* Adds R's bin directory to the path (if we can find it)

To use:

1. Install Python, IPython, and Nose.  An easy way to do this is with
   the Anaconda Python distribution
   http://continuum.io/downloads
2. Install msysGit
   https://github.com/msysgit/msysgit/releases
3. Install R (if your workshop uses R)
   http://cran.r-project.org/bin/windows/base/rw-FAQ.html#Installation-and-Usage
4. Run swc-windows-installer.py.
   You should be able to simply double click the file in Windows

"""

import glob
import hashlib
try:  # Python 3
    from io import BytesIO as _BytesIO
except ImportError:  # Python 2
    from StringIO import StringIO as _BytesIO
import logging
import os
import re
import sys
import tarfile
try:  # Python 3
    from urllib.request import urlopen as _urlopen
except ImportError:  # Python 2
    from urllib2 import urlopen as _urlopen
import zipfile


__version__ = '0.2'

LOG = logging.getLogger('swc-windows-installer')
LOG.addHandler(logging.StreamHandler())
LOG.setLevel(logging.INFO)


if sys.version_info >= (3, 0):  # Python 3
    open3 = open
else:
    def open3(file, mode='r', newline=None):
        if newline:
            if newline != '\n':
                raise NotImplementedError(newline)
            f = open(file, mode + 'b')
        else:
            f = open(file, mode)
        return f


def download(url, sha1):
    """Download a file and verify its hash"""
    LOG.debug('download {}'.format(url))
    r = _urlopen(url)
    byte_content = r.read()
{'fraction_non_alphanumeric': 0.08641662442980233, 'fraction_numerical': 0.00912316269640142, 'mean_word_length': 3.3231106243154436, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6013434', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1218, 'n_words': 459}