<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import pdb

import os as mod_os
import os.path as mod_path
import sys as mod_sys

def assert_in_git_repository():
    success, lines = execute_git('status', output=False)
    if not success:
        print 'Not a git repository!!!'
        mod_sys.exit(1)

def execute_command(command, output=True, prefix='', grep=None):
    result = ''
    p = mod_os.popen(command)
    for line in p:
        output_line = prefix + ('%s' % line).rstrip() + '\n'
        if not grep or grep in output_line:
            if output and output_line:
                print output_line.rstrip()
                mod_sys.stdout.flush()
            result += output_line

    return (not p.close(), result)

def execute_git(command, output=True, prefix='', grep=None):
    return execute_command('git %s' % command, output, prefix, grep)

def get_branches(remote=False, all=False):
    git_command = 'branch'
    if remote:
        git_command += ' -r'
    if all:
        git_command += ' -a'<commit_msg>rework execute_command to use subprocess.Popen instead of os.popen<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import pdb

import os as mod_os
import os.path as mod_path
import sys as mod_sys
import subprocess

def assert_in_git_repository():
    success, lines = execute_git('status', output=False)
    if not success:
        print 'Not a git repository!!!'
        mod_sys.exit(1)

def execute_command(command, output=True, prefix='', grep=None):
    result = ''

    if type(command) is not list:
        command = command.split(' ')

    p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1)
    (cmdout, cmderr) = p.communicate()
    if cmdout is None:
        return (0, "")

    for line in cmdout.split('\n'):
        output_line = prefix + ('%s' % line).rstrip() + '\n'
        if not grep or grep in output_line:
            if output and output_line:
                print output_line.rstrip()
                mod_sys.stdout.flush()
            result += output_line

    return (not p.returncode, result)

def execute_git(command, output=True, prefix='', grep=None):
    return execute_command('git %s' % command, output, prefix, grep)

def get_branches(remote=False, all=False):
    git_command = 'branch'
    if remote:
        git_command += ' -r'
    if all:
        git_command += ' -a'
{'fraction_non_alphanumeric': 0.12416387959866221, 'fraction_numerical': 0.002508361204013378, 'mean_word_length': 2.768503937007874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6662757', 'n_tokens_mistral': 776, 'n_tokens_neox': 754, 'n_words': 215}