<commit_before>#!/usr/bin/env python

import argparse
import numpy
import re
import sys

##############################################

def add2D (i, j, PE, PEMap):
  if not i in PEMap:
    PEMap[i] = {}
  PEMap[i][j] = PE

def printMap (PEMap):
  currentI = -1
  currentJ = -1

##############################################

parser = argparse.ArgumentParser()

parser.add_argument("FILE",
    help = "The output file to plot. A value of '-' means standard input.")

parser.add_argument("--debug",
    help = "Print debugging stuff",
    action = "store_true",
    default = False)

options = parser.parse_args()

if options.FILE == "-":
  fd = sys.stdin
else:
  fd = open(options.FILE)

inIteration = False
iteration = -1

inMap = False
currentMap = ""

PEMap = {}

for line in fd:
  if options.debug:
    print("read: ", line.rstrip())

  if not inIteration:
    result = re.compile("iteration ([0-9]+) on").search(line)
    if result:
      inIteration = True
      iteration = int(result.group(1))
      print("iteration {:d}".format(iteration))
      continue

  result = re.compile("] PEMap for (.*):").search(line)
  if result:
    mapName = result.group(1)
    if inMap and mapName != currentMap:
      raise(Exception("map {:s} already open for reading".format(mapName)))
    if not inMap:
      N = 0
      elementBuffer = []
    inMap = True
    currentMap = mapName
    if options.debug:
      print("opening map {:s}".format(currentMap))

  result = re.compile("PEMap\(([0-9]+),([0-9]+)\) = ([0-9]+)").search(line)
  if result:
    i = int(result.group(1))
    j = int(result.group(2))
    PE = int(result.group(3))
    if not inMap:
      raise(Exception("no map open for reading"))
    elementBuffer.append( (i, j, PE) )
    if i+1 > N:
      N = i+1
    if j+1 > N:
      N = j+1
    continue

  result = re.compile("PEMap\(([0-9]+),([0-9]+),([0-9]+)\) = ([0-9]+)").search(line)
  if result:
    i = int(result.group(1))
    j = int(result.group(2))
    k = int(result.group(3))
    PE = int(result.group(4))
    if not inMap:
      raise(Exception("no map open for reading"))
    elementBuffer.append( (i, j, k, PE) )
    if i+1 > N:
      N = i+1
    if j+1 > N:
      N = j+1
    if k+1 > N:
      N = k+1
    continue

  result = re.compile("end of PEMap").search(line)
  if result:
    if currentMap == "convolution":
      PEMap[currentMap] = numpy.empty([N, N, N])
      for (i, j, k, PE) in elementBuffer:
        PEMap[currentMap][i,j,k] = PE
    else:
      PEMap[currentMap] = numpy.empty([N, N])
      for (i, j, PE) in elementBuffer:
        PEMap[currentMap][i,j] = PE
    print(PEMap[currentMap])
    inMap = False
    if options.debug:
      print("closing map {:s}".format(currentMap))
    continue

fd.close()

for name in PEMap:
  printMap(PEMap[name])<commit_msg>Added default value for PEMap.<commit_after>#!/usr/bin/env python

import argparse
import numpy
import re
import sys

##############################################

def add2D (i, j, PE, PEMap):
  if not i in PEMap:
    PEMap[i] = {}
  PEMap[i][j] = PE

def printMap (PEMap):
  currentI = -1
  currentJ = -1

##############################################

parser = argparse.ArgumentParser()

parser.add_argument("FILE",
    help = "The output file to plot. A value of '-' means standard input.")

parser.add_argument("--debug",
    help = "Print debugging stuff",
    action = "store_true",
    default = False)

options = parser.parse_args()

if options.FILE == "-":
  fd = sys.stdin
else:
  fd = open(options.FILE)

inIteration = False
iteration = -1

inMap = False
currentMap = ""

PEMap = {}

for line in fd:
  if options.debug:
    print("read: ", line.rstrip())

  if not inIteration:
    result = re.compile("iteration ([0-9]+) on").search(line)
    if result:
      inIteration = True
      iteration = int(result.group(1))
      print("iteration {:d}".format(iteration))
      continue

  result = re.compile("] PEMap for (.*):").search(line)
  if result:
    mapName = result.group(1)
    if inMap and mapName != currentMap:
      raise(Exception("map {:s} already open for reading".format(mapName)))
    if not inMap:
      N = 0
      elementBuffer = []
    inMap = True
    currentMap = mapName
    if options.debug:
      print("opening map {:s}".format(currentMap))

  result = re.compile("PEMap\(([0-9]+),([0-9]+)\) = ([0-9]+)").search(line)
  if result:
    i = int(result.group(1))
    j = int(result.group(2))
    PE = int(result.group(3))
    if not inMap:
      raise(Exception("no map open for reading"))
    elementBuffer.append( (i, j, PE) )
    if i+1 > N:
      N = i+1
    if j+1 > N:
      N = j+1
    continue

  result = re.compile("PEMap\(([0-9]+),([0-9]+),([0-9]+)\) = ([0-9]+)").search(line)
  if result:
    i = int(result.group(1))
    j = int(result.group(2))
    k = int(result.group(3))
    PE = int(result.group(4))
    if not inMap:
      raise(Exception("no map open for reading"))
    elementBuffer.append( (i, j, k, PE) )
    if i+1 > N:
      N = i+1
    if j+1 > N:
      N = j+1
    if k+1 > N:
      N = k+1
    continue

  result = re.compile("end of PEMap").search(line)
  if result:
    if currentMap == "convolution":
      PEMap[currentMap] = numpy.empty([N, N, N])
      PEMap[currentMap].fill(-1)
      for (i, j, k, PE) in elementBuffer:
        PEMap[currentMap][i,j,k] = PE
    else:
      PEMap[currentMap] = numpy.empty([N, N])
      PEMap[currentMap].fill(-1)
      for (i, j, PE) in elementBuffer:
        PEMap[currentMap][i,j] = PE
    print(PEMap[currentMap])
    inMap = False
    if options.debug:
      print("closing map {:s}".format(currentMap))
    continue

fd.close()

for name in PEMap:
  printMap(PEMap[name])
{'fraction_non_alphanumeric': 0.19296394019349164, 'fraction_numerical': 0.014423922603342128, 'mean_word_length': 2.836707152496626, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '370853', 'n_tokens_mistral': 2070, 'n_tokens_neox': 2066, 'n_words': 517}