<commit_before>import os
import re
import codecs
from optparse import OptionParser
import ConfigParser
from StringIO import StringIO


def main():
    parser = OptionParser("Usage: %prog [options] cfgpath cfgsect "
                          "catalog-path")
    (options, args) = parser.parse_args()
    if len(args) < 3:
        parser.print_usage()
        return

    cfg = ConfigParser.RawConfigParser(dict(seperator=' ',
                                            merge_charset='utf8',
                                            skip_head=0))
    cfgpath = os.path.expanduser(args[0])
    cfg.read(cfgpath)
    cfgsect = args[1]
    catalog = args[2]

    path_ptrn = re.compile(cfg.get(cfgsect, 'path_ptrn'))
    charset = cfg.get(cfgsect, 'merge_charset')
    seperator = cfg.get(cfgsect, 'seperator')
    line_head = cfg.get(cfgsect, 'merge_line_head')
    skip_head = int(cfg.get(cfgsect, 'skip_head'))

    with open(catalog, 'r') as cf:
        for cline in cf:
            afile = cline.rstrip().split('\t')[0]
            match = path_ptrn.search(afile)
            if match is None:
                continue

            ginfo = match.groupdict()
            lhead = seperator.join(line_head.format(**ginfo).split(','))
            buf = StringIO()
            with codecs.open(afile, 'r', charset, errors='ignore') as f:
                for i, line in enumerate(f.readlines()):
                    if i < skip_head:
                        continue
                    line = line.rstrip()
                    if len(line) > 0:
                        buf.write(u'{}{}{}\n'.format(lhead, seperator, line))
            print buf.getvalue().rstrip().encode('utf8')
            buf.close()


if __name__ == "__main__":
    main()<commit_msg>rename skip_head to merge_skip_head<commit_after>import os
import re
import codecs
from optparse import OptionParser
import ConfigParser
from StringIO import StringIO


def main():
    parser = OptionParser("Usage: %prog [options] cfgpath cfgsect "
                          "catalog-path")
    (options, args) = parser.parse_args()
    if len(args) < 3:
        parser.print_usage()
        return

    cfg = ConfigParser.RawConfigParser(dict(seperator=' ',
                                            merge_charset='utf8',
                                            merge_skip_head=0))
    cfgpath = os.path.expanduser(args[0])
    cfg.read(cfgpath)
    cfgsect = args[1]
    catalog = args[2]

    path_ptrn = re.compile(cfg.get(cfgsect, 'path_ptrn'))
    charset = cfg.get(cfgsect, 'merge_charset')
    seperator = cfg.get(cfgsect, 'seperator')
    line_head = cfg.get(cfgsect, 'merge_line_head')
    merge_skip_head = int(cfg.get(cfgsect, 'merge_skip_head'))

    with open(catalog, 'r') as cf:
        for cline in cf:
            afile = cline.rstrip().split('\t')[0]
            match = path_ptrn.search(afile)
            if match is None:
                continue

            ginfo = match.groupdict()
            lhead = seperator.join(line_head.format(**ginfo).split(','))
            buf = StringIO()
            with codecs.open(afile, 'r', charset, errors='ignore') as f:
                for i, line in enumerate(f.readlines()):
                    if i < merge_skip_head:
                        continue
                    line = line.rstrip()
                    if len(line) > 0:
                        buf.write(u'{}{}{}\n'.format(lhead, seperator, line))
            print buf.getvalue().rstrip().encode('utf8')
            buf.close()


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.12155317951603826, 'fraction_numerical': 0.005064715813168261, 'mean_word_length': 1.8531300160513644, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5582538', 'n_tokens_mistral': 1080, 'n_tokens_neox': 1040, 'n_words': 230}