<commit_before>#!/usr/bin/env python3

import re
import os
import sys
import argparse
import collections
import subprocess

from itertools import islice
from operator import itemgetter, attrgetter
from urllib.parse import urlparse
from pprint import pprint, pformat

if sys.version_info >= (3,5):
    _collections_Generator = collections.Generator
else:
    import _compatibility.Generator
    _collections_Generator = _compatibility.Generator

class Fields(list):

    def __getitem__(self, i):
        if isinstance(i, int):
            return super().__getitem__(i) if len(self) > i else ''
        else:
            return super().__getitem__(i)


def url(url):
    if not re.match('([a-z]+:)?//', url):
        url = '//' + url
    return urlparse(url)


def counter(l, n=10):
    return collections.Counter(l).most_common(n)


def sh(c, F=None):
    lines = subprocess.check_output(c).decode().splitlines()
    return [Fields(line.strip().split(F)) for line in lines]


def get(l, i, d=None):
    if isinstance(l, collections.Generator):
        for j, v in enumerate(l):
            if i == j:
                return v
        return d
    else:
        return l[i] if len(l) > i else d


def bytesize(x):
    units = ['P', 'T', 'G', 'M', 'K', 'B']
    for i in range(len(units)):
        if x == 0:
            return '{:6.2f} {}'.format(0,units[-1])
        if x // 1024**(len(units) - i - 1) > 0:
            return '{:6.2f} {}'.format(x / (1024**(len(units) - i - 1)), units[i])


def _len(value):
    if isinstance(value, collections.Generator):
        return sum(1 for x in value)
    else:
        return len(value)


def _stdin(args):
    for line in sys.stdin:
        if args.separator is not None:
            yield Fields(line.strip().split(args.separator))
        elif args.split:
            yield Fields(line.strip().split())
        else:
            yield line.strip()


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('expression', help="python expression")
    parser.add_argument('-F', '--separator', default=None, help="split each line by SEPARATOR")
    parser.add_argument('-s', '--split', const=True, default=False, action='store_const', help="split each line")
    parser.add_argument('-q', '--quiet', const=True, default=False, action='store_const',
                        help="don't implicitly print results")
    args = parser.parse_args()

    result = eval('(%s)' % args.expression, globals(), {
        '_': _stdin(args),
        'len': _len,
    })

    if not args.quiet:
        if isinstance(result, (list, collections.Generator)):
            for line in result:<commit_msg>Add package _compatibility to setup.py<commit_after>from __future__ import print_function

import re
import os
import sys
import argparse
import collections
import subprocess

from itertools import islice
from operator import itemgetter, attrgetter
try:
    from urllib.parse import urlparse
except:
    from urlparse import urlparse

from pprint import pprint, pformat

if sys.version_info >= (3,5):
    _collections_Generator = collections.Generator
else:
    from _compatibility import Generator
    _collections_Generator = Generator

class Fields(list):

    def __getitem__(self, i):
        if isinstance(i, int):
            return super().__getitem__(i) if len(self) > i else ''
        else:
            return super().__getitem__(i)


def url(url):
    if not re.match('([a-z]+:)?//', url):
        url = '//' + url
    return urlparse(url)


def counter(l, n=10):
    return collections.Counter(l).most_common(n)


def sh(c, F=None):
    lines = subprocess.check_output(c).decode().splitlines()
    return [Fields(line.strip().split(F)) for line in lines]


def get(l, i, d=None):
    if isinstance(l, _collections_Generator):
        for j, v in enumerate(l):
            if i == j:
                return v
        return d
    else:
        return l[i] if len(l) > i else d


def bytesize(x):
    units = ['P', 'T', 'G', 'M', 'K', 'B']
    for i in range(len(units)):
        if x == 0:
            return '{:6.2f} {}'.format(0,units[-1])
        if x // 1024**(len(units) - i - 1) > 0:
            return '{:6.2f} {}'.format(x / (1024**(len(units) - i - 1)), units[i])


def _len(value):
    if isinstance(value, _collections_Generator):
        return sum(1 for x in value)
    else:
        return len(value)


def _stdin(args):
    for line in sys.stdin:
        if args.separator is not None:
            yield Fields(line.strip().split(args.separator))
        elif args.split:
            yield Fields(line.strip().split())
        else:
            yield line.strip()


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('expression', help="python expression")
    parser.add_argument('-F', '--separator', default=None, help="split each line by SEPARATOR")
    parser.add_argument('-s', '--split', const=True, default=False, action='store_const', help="split each line")
    parser.add_argument('-q', '--quiet', const=True, default=False, action='store_const',
                        help="don't implicitly print results")
    args = parser.parse_args()

    result = eval('(%s)' % args.expression, globals(), {
        '_': _stdin(args),
        'len': _len,
    })

    if not args.quiet:
        if isinstance(result, (list, _collections_Generator)):
            for line in result:
{'fraction_non_alphanumeric': 0.14216688347890727, 'fraction_numerical': 0.00873443597844267, 'mean_word_length': 2.6989690721649486, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5748032', 'n_tokens_mistral': 1745, 'n_tokens_neox': 1669, 'n_words': 473}