<commit_before>"""
This script processes the output from the C preprocessor and extracts all
qstr. Each qstr is transformed into a qstr definition of the form 'Q(...)'.

This script works with Python 2.6, 2.7, 3.3 and 3.4.
"""

import sys
import re
import argparse
import os

# Blacklist of qstrings that are specially handled in further
# processing and should be ignored
QSTRING_BLACK_LIST = {'NULL', 'number_of', }


def debug(message):
    #sys.stderr.write("%s\n" % message)
    pass


def write_out(fname, output):
    if output:
        fname = fname.replace("/", "__").replace("..", "@@")
        with open(args.output_dir + "/" + fname + ".qstr", "w") as f:
            f.write("\n".join(output) + "\n")

def process_file(f):
    output = []
    last_fname = None
    outf = None
    for line in f:
        if line and line[0:2] == "# ":
            comp = line.split()
            fname = comp[2]
            assert fname[0] == '"' and fname[-1] == '"'
            fname = fname[1:-1]
            if fname[0] == "/" or not fname.endswith(".c"):
                continue
            if fname != last_fname:
                write_out(last_fname, output)
                output = []<commit_msg>py/makeqstrdefs.py: Remove unused function/variable/import.<commit_after>"""
This script processes the output from the C preprocessor and extracts all
qstr. Each qstr is transformed into a qstr definition of the form 'Q(...)'.

This script works with Python 2.6, 2.7, 3.3 and 3.4.
"""

import re
import argparse
import os

# Blacklist of qstrings that are specially handled in further
# processing and should be ignored
QSTRING_BLACK_LIST = {'NULL', 'number_of', }


def write_out(fname, output):
    if output:
        fname = fname.replace("/", "__").replace("..", "@@")
        with open(args.output_dir + "/" + fname + ".qstr", "w") as f:
            f.write("\n".join(output) + "\n")

def process_file(f):
    output = []
    last_fname = None
    for line in f:
        if line and line[0:2] == "# ":
            comp = line.split()
            fname = comp[2]
            assert fname[0] == '"' and fname[-1] == '"'
            fname = fname[1:-1]
            if fname[0] == "/" or not fname.endswith(".c"):
                continue
            if fname != last_fname:
                write_out(last_fname, output)
                output = []
{'fraction_non_alphanumeric': 0.14382978723404255, 'fraction_numerical': 0.013617021276595745, 'mean_word_length': 2.224965706447188, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3173372', 'n_tokens_mistral': 765, 'n_tokens_neox': 743, 'n_words': 234}