<commit_before># flowtools_wrapper.py
# Copyright 2014 <NAME> (<EMAIL>)
# See http://github.com/bbayles/py3flowtools for documentation and license

from __future__ import division, print_function, unicode_literals

import io
import os
import sys

from .flow_line import FlowLine

if sys.version_info.major < 3:
    import subprocess32 as subprocess
else:
    import subprocess

FLOWD_READER_ARGS = [
    'flowd-reader',
    '-v',  # Verbose output
    '-c',  # CSV formatting
    '-U',  # UTC timestamps
    '{file_path:}'
]


def FlowLog(file_path):
    args = FLOWD_READER_ARGS[:]
    args[-1] = file_path
    with io.open(os.devnull, mode='wb') as DEVNULL:
        with subprocess.Popen(
                args,
                stdout=subprocess.PIPE,
                stderr=DEVNULL
        ) as proc:
            iterator = iter(proc.stdout.readline, b'')
            try:
                # Skip the headers
                next(iterator)
                next(iterator)
            except StopIteration:
                msg = 'Could not extract data from {}'.format(file_path)
                raise IOError(msg)
            for line in iterator:
                parsed_line = FlowLine(line)
                yield parsed_line
<commit_msg>Fix up error handling for flowd<commit_after># flowtools_wrapper.py
# Copyright 2014 <NAME> (<EMAIL>)
# See http://github.com/bbayles/py3flowtools for documentation and license

from __future__ import division, print_function, unicode_literals

import io
import os
import sys

from .flow_line import FlowLine

if sys.version_info.major < 3:
    import subprocess32 as subprocess
else:
    import subprocess

ERR_MSG = 'Could not extract data from {}'

FLOWD_READER_ARGS = [
    'flowd-reader',
    '-v',  # Verbose output
    '-c',  # CSV formatting
    '-U',  # UTC timestamps
    '{file_path:}'
]


def FlowLog(file_path):
    args = FLOWD_READER_ARGS[:]
    args[-1] = file_path
    with io.open(os.devnull, mode='wb') as DEVNULL:
        with subprocess.Popen(
                args,
                stdout=subprocess.PIPE,
                stderr=DEVNULL
        ) as proc:
            iterator = iter(proc.stdout.readline, b'')
            try:
                # Skip the headers
                next(iterator)
                next(iterator)
            except StopIteration:
                raise IOError(ERR_MSG.format(file_path))
            line = None
            for line in iterator:
                parsed_line = FlowLine(line)
                yield parsed_line
            else:
                if line is None:
                    raise IOError(ERR_MSG.format(file_path))

{'fraction_non_alphanumeric': 0.08838095238095238, 'fraction_numerical': 0.006857142857142857, 'mean_word_length': 1.9977168949771689, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1765189', 'n_tokens_mistral': 812, 'n_tokens_neox': 766, 'n_words': 205}