<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

from __future__ import print_function

import os
import os.path as path
from os.path import join as pjoin
import sys
from sys import version_info
import re
import subprocess
    
# partial import
import Corrfunc
from Corrfunc import read_text_file, write_text_file

# Make sure we are running on posix (Linux, Unix, MAC OSX)
if os.name != 'posix':
    sys.exit("Sorry, Windows is not supported")


def strip_line(line, sep=os.linesep):
    """
    Removes occurrence of character (sep) from a line of text
    """
    
    try:
        return line.strip(sep)
    except TypeError:
        return line.decode('utf-8').strip(sep)


def run_command(command, **kwargs):
    proc = subprocess.Popen(command, shell=True,
                            **kwargs)
    stdout, stderr = proc.communicate(None)
    status = proc.wait()
    if status:
        msg = "command = {0} failed with stdout = {1} stderr = {2} "\
              "status {3:d}\n".format(command, stdout, stderr, status)
        raise Exception(msg)
    return stdout, stderr


def get_dict_from_buffer(buf, keys=['DISTNAME', 'MAJOR',
                                    'MINOR', 'PATCHLEVEL',
                                    'PYTHON',
                                    'MIN_PYTHON_MAJOR',
                                    'MIN_PYTHON_MINOR',
                                    'MIN_NUMPY_MAJOR',
                                    'MIN_NUMPY_MINOR']):
    """
    Parses a string buffer for key-val pairs for the supplied keys.

    Slightly modified from:
    "http://stackoverflow.com/questions/5323703/regex-how-to-"\
    "match-sequence-of-key-value-pairs-at-end-of-string

    """
    
    import re
    keys = [k.strip() for k in keys]
    regex = re.compile(r'''
    \n                 # all key-value pairs are on separate lines
    \s*                # there might be some leading spaces
    (                  # start group to return
    (?:{0}\s*)         # placeholder for tags to detect '\S+' == all
    \s*:*=\s*          # optional spaces, optional colon, = , optional spaces
    .*                 # the value
    )                  # end group to return
    '''.format('|'.join(keys)), re.VERBOSE)
    
    matches = regex.findall(buf)
    pairs = {k: [] for k in keys}
    for match in matches:
        key, val = match.split('=', 1)
        # remove colon and leading/trailing whitespace
        key = (strip_line(key, ':')).strip()
        # remove newline and leading/trailing whitespace
        val = (strip_line(val)).strip()
        if key not in keys:
            msg = "regex produced incorrect match. regex pattern = {0}"\
                  "claims key = [{1}] while original set of search keys = {2}"\
                  .format(regex.pattern, key, '|'.join(keys))
            raise RuntimeError(msg)
        pairs.setdefault(key, []).append(val)
            
    return pairs

    
base_url = "https://github.com/manodeep/Corrfunc"
common_mk_file = pjoin(path.dirname(path.abspath(__file__)),
                       "common.mk")
common = read_text_file(common_mk_file)
common_dict = get_dict_from_buffer(common)
name = common_dict['DISTNAME'][0]
major = common_dict['MAJOR'][0]
minor = common_dict['MINOR'][0]
patch = common_dict['PATCHLEVEL'][0]
if name is None or major is None or minor is None or patch is None:<commit_msg>Fixed the empty list initialized for python2.6 (possibly list comprehension issues)<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

from __future__ import print_function

import os
import os.path as path
from os.path import join as pjoin
import sys
from sys import version_info
import re
import subprocess
    
# partial import
import Corrfunc
from Corrfunc import read_text_file, write_text_file

# Make sure we are running on posix (Linux, Unix, MAC OSX)
if os.name != 'posix':
    sys.exit("Sorry, Windows is not supported")


def strip_line(line, sep=os.linesep):
    """
    Removes occurrence of character (sep) from a line of text
    """
    
    try:
        return line.strip(sep)
    except TypeError:
        return line.decode('utf-8').strip(sep)


def run_command(command, **kwargs):
    proc = subprocess.Popen(command, shell=True,
                            **kwargs)
    stdout, stderr = proc.communicate(None)
    status = proc.wait()
    if status:
        msg = "command = {0} failed with stdout = {1} stderr = {2} "\
              "status {3:d}\n".format(command, stdout, stderr, status)
        raise Exception(msg)
    return stdout, stderr


def get_dict_from_buffer(buf, keys=['DISTNAME', 'MAJOR',
                                    'MINOR', 'PATCHLEVEL',
                                    'PYTHON',
                                    'MIN_PYTHON_MAJOR',
                                    'MIN_PYTHON_MINOR',
                                    'MIN_NUMPY_MAJOR',
                                    'MIN_NUMPY_MINOR']):
    """
    Parses a string buffer for key-val pairs for the supplied keys.

    Slightly modified from:
    "http://stackoverflow.com/questions/5323703/regex-how-to-"\
    "match-sequence-of-key-value-pairs-at-end-of-string

    """
    
    import re
    keys = [k.strip() for k in keys]
    regex = re.compile(r'''
    \n                 # all key-value pairs are on separate lines
    \s*                # there might be some leading spaces
    (                  # start group to return
    (?:{0}\s*)         # placeholder for tags to detect '\S+' == all
    \s*:*=\s*          # optional spaces, optional colon, = , optional spaces
    .*                 # the value
    )                  # end group to return
    '''.format('|'.join(keys)), re.VERBOSE)
    
    matches = regex.findall(buf)
    pairs = dict()
    for k in keys:
        pairs[k] = []

    for match in matches:
        key, val = match.split('=', 1)
        # remove colon and leading/trailing whitespace
        key = (strip_line(key, ':')).strip()
        # remove newline and leading/trailing whitespace
        val = (strip_line(val)).strip()
        if key not in keys:
            msg = "regex produced incorrect match. regex pattern = {0}"\
                  "claims key = [{1}] while original set of search keys = {2}"\
                  .format(regex.pattern, key, '|'.join(keys))
            raise RuntimeError(msg)
        pairs.setdefault(key, []).append(val)
            
    return pairs

    
base_url = "https://github.com/manodeep/Corrfunc"
common_mk_file = pjoin(path.dirname(path.abspath(__file__)),
                       "common.mk")
common = read_text_file(common_mk_file)
common_dict = get_dict_from_buffer(common)
name = common_dict['DISTNAME'][0]
major = common_dict['MAJOR'][0]
minor = common_dict['MINOR'][0]
patch = common_dict['PATCHLEVEL'][0]
if name is None or major is None or minor is None or patch is None:
{'fraction_non_alphanumeric': 0.1158492213651579, 'fraction_numerical': 0.006694804249745306, 'mean_word_length': 2.0259797446059005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1575744', 'n_tokens_mistral': 2101, 'n_tokens_neox': 2039, 'n_words': 582}