<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division, absolute_import, unicode_literals,
                        print_function)

import os
import platform
import subprocess
import sys

from setuptools import find_packages, setup


# Check if perl is installed.
try:
    out = subprocess.check_output(['perl', '-v'])
except (OSError, subprocess.CalledProcessError):
    print('`perl` (https://www.perl.org/) needs to be installed and needs '
          'to be made available in your PATH. '
          'On Ubuntu, you can do `sudo apt-get install perl`.')
    sys.exit(1)

# Check if java is installed.
try:
    out = subprocess.check_output(['java', '-version'])
except (OSError, subprocess.CalledProcessError):
    print('`java` (https://java.com/) needs to be installed and needs to '
          'be made available in your PATH. If using Ubuntu, you can do '
          '`sudo apt-get install openjdk-7-jre`')
    sys.exit(1)

# Check if avprobe of ffprobe is installed if system is not Linux.
# If it's linux, we use static builds from http://johnvansickle.com/libav/
if platform.system() != 'Linux':
    try:
        out = subprocess.check_output(['ffprobe', '-version'])
    except (OSError, subprocess.CalledProcessError):
        try:
            out = subprocess.check_output(['avprobe', '-version'])
        except (OSError, subprocess.CalledProcessError):
            print('Neither `ffprobe` (https://ffmpeg.org/ffprobe.html) nor '
                  '`avprobe` (https://libav.org/documentation/avprobe.html) '
                  'were found. Either one of them needs to be installed and '
                  'made available in your PATH. If using Ubuntu, you can do '
                  '`sudo apt-get install libav-tools` to install avprobe.')
        sys.exit(1)


required = open('requirements.txt').read().strip().splitlines()
test_required = open('test-requirements.txt').read().strip().splitlines()
VERSION = open(os.path.join('file_metadata', 'VERSION')).read().strip()

if sys.version_info >= (3,):
    # mock is not required for python 3
    test_required.remove('mock')<commit_msg>setup: Filter comments to fix bdist_wheel<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division, absolute_import, unicode_literals,
                        print_function)

import os
import platform
import subprocess
import sys

from setuptools import find_packages, setup


# Check if perl is installed.
try:
    out = subprocess.check_output(['perl', '-v'])
except (OSError, subprocess.CalledProcessError):
    print('`perl` (https://www.perl.org/) needs to be installed and needs '
          'to be made available in your PATH. '
          'On Ubuntu, you can do `sudo apt-get install perl`.')
    sys.exit(1)

# Check if java is installed.
try:
    out = subprocess.check_output(['java', '-version'])
except (OSError, subprocess.CalledProcessError):
    print('`java` (https://java.com/) needs to be installed and needs to '
          'be made available in your PATH. If using Ubuntu, you can do '
          '`sudo apt-get install openjdk-7-jre`')
    sys.exit(1)

# Check if avprobe of ffprobe is installed if system is not Linux.
# If it's linux, we use static builds from http://johnvansickle.com/libav/
if platform.system() != 'Linux':
    try:
        out = subprocess.check_output(['ffprobe', '-version'])
    except (OSError, subprocess.CalledProcessError):
        try:
            out = subprocess.check_output(['avprobe', '-version'])
        except (OSError, subprocess.CalledProcessError):
            print('Neither `ffprobe` (https://ffmpeg.org/ffprobe.html) nor '
                  '`avprobe` (https://libav.org/documentation/avprobe.html) '
                  'were found. Either one of them needs to be installed and '
                  'made available in your PATH. If using Ubuntu, you can do '
                  '`sudo apt-get install libav-tools` to install avprobe.')
        sys.exit(1)


def read_reqs(reqs_filename):
    reqs = open(reqs_filename).read().strip().splitlines()
    return list(i for i in reqs if not (i.startswith('#') or len(i) == 0))

required = read_reqs('requirements.txt')
test_required = read_reqs('test-requirements.txt')
VERSION = open(os.path.join('file_metadata', 'VERSION')).read().strip()

if sys.version_info >= (3,):
    # mock is not required for python 3
    test_required.remove('mock')
{'fraction_non_alphanumeric': 0.12841962468912502, 'fraction_numerical': 0.003391363328057879, 'mean_word_length': 3.0108794197642794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1604992', 'n_tokens_mistral': 1380, 'n_tokens_neox': 1302, 'n_words': 402}