<commit_before>#!/usr/bin/python

import argparse
import os
import sys

# our modules
import opinfo
import query
import runtime

def main(args):
    if args.verbose:
        print("Verbose output turned on")
    if not os.path.exists(args.filename):
        print("File %s does not exist" % args.filename)
        sys.exit(0)

    # form the right query to run
    runtime_info = runtime.RunTimeInfo(verbose=args.verbose)
    filemap_process = runtime_info.filemap(args.filename)
    filemap_process.wait()
    
    if not os.path.exists("/tmp/filemap_output"):
        print("Could not get filemap?")
        sys.exit(0)

    writes = []
    # now process the filemap results
    f = open("/tmp/filemap_output", "r")
    for line in f.readlines():
        fields = line.split(" ")
        offset = int(fields[0])
        size = int(fields[1])
        group_id = int(fields[2])
        pid = int(fields[3])
        sysnum = int(fields[4])
        wi = opinfo.WriteInfo(group_id, pid, sysnum, offset, size)
        print("Starting Write info [%d, %d, %d, %d, %d]" % (group_id, pid, sysnum, offset, size))
        writes.append(wi)

    if not writes:
        print("File %s has no write information!")
        sys.exit(0)

    # write_info = opinfo.WriteInfo(4143, 3468, 42, 0, 5)
    q = query.Query(runtime_info, writes)
    q.run()
    q.draw_graph()<commit_msg>Read filemap info from stdout now instead of a file<commit_after>#!/usr/bin/python

import argparse
import os
import sys

# our modules
import opinfo
import query
import runtime

def main(args):
    if args.verbose:
        print("Verbose output turned on")
    if not os.path.exists(args.filename):
        print("File %s does not exist" % args.filename)
        sys.exit(0)

    # form the right query to run
    runtime_info = runtime.RunTimeInfo(verbose=args.verbose)
    filemap_process = runtime_info.filemap(args.filename)
    # filemap output is to stdout
    filemap_process.wait()
    filemap_output = filemap_process.communicate()[0]
    
    if not filemap_output:
        print("Could not get filemap info for %s" % args.filename)
        sys.exit(0)

    writes = []
    print(filemap_output.split("\n"))
    # now process the filemap results
    for line in filemap_output.split("\n"):
        if not line:
            continue
        fields = line.split(" ")
        offset = int(fields[0])
        size = int(fields[1])
        group_id = int(fields[2])
        pid = int(fields[3])
        sysnum = int(fields[4])
        wi = opinfo.WriteInfo(group_id, pid, sysnum, offset, size)
        print("Starting Write info [%d, %d, %d, %d, %d]" % (group_id, pid, sysnum, offset, size))
        writes.append(wi)

    if not writes:
        print("File %s has no write information!")
        sys.exit(0)

    # write_info = opinfo.WriteInfo(4143, 3468, 42, 0, 5)
    q = query.Query(runtime_info, writes)
    q.run()
    q.draw_graph()
    q.graph.output_graph()
{'fraction_non_alphanumeric': 0.12359550561797752, 'fraction_numerical': 0.013959822948586994, 'mean_word_length': 2.6587795765877957, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5172601', 'n_tokens_mistral': 987, 'n_tokens_neox': 958, 'n_words': 258}