<commit_before>import cPickle as pickle
import numpy
import glob, os, stat, time, datetime, sys
import re
import tspl_utils
import math
import logging, multiprocessing

class TSPLException(Exception):
  def __init__(self,arg):
    self.value=arg
    print self.value

class TSPLBase:
  def __init__(self,file,k1,k2):
    self.f=open(file)
    try:
      self.j=pickle.load(self.f)
    except EOFError as e:
      raise TSPLException('End of file found for: ' + file)
    self.f.close()

    try:
      self.wayness=int(re.findall('\d+',self.j.acct['granted_pe'])[0])
    except KeyError:
      try:
        self.wayness=self.j.acct['cores']/self.j.acct['nodes']
      except ZeroDivisionError:
        print "Read zero nodes, assuming 16 way for job " + str(self.j.id)
        self.wayness=16
      except TypeError:
        raise TSPLException('Something is funny with job ' +str(self.j.id) +
                            ' ' + str(self.j.acct['cores']) + ' ' +
                            str(self.j.acct['nodes']))

    try:
      self.queue=self.j.acct['queue']
    except KeyError:
      print 'No queue found'
      self.queue = None

    try:
      self.owner=self.j.acct['owner']
    except KeyError:
      try:
        import pwd
        self.owner=pwd.getpwuid(int(self.j.acct['uid']))[0]
      except Exception as e:
        self.owner=self.j.acct['uid']
      
    self.numhosts=len(self.j.hosts.keys())
<commit_msg>added accounting job exit status as a class member data item<commit_after>import cPickle as pickle
import numpy
import glob, os, stat, time, datetime, sys
import re
import tspl_utils
import math
import logging, multiprocessing

class TSPLException(Exception):
  def __init__(self,arg):
    self.value=arg
    print self.value

class TSPLBase:
  def __init__(self,file,k1,k2):
    self.f=open(file)
    try:
      self.j=pickle.load(self.f)
    except EOFError as e:
      raise TSPLException('End of file found for: ' + file)
    self.f.close()

    try:
      self.wayness=int(re.findall('\d+',self.j.acct['granted_pe'])[0])
    except KeyError:
      try:
        self.wayness=self.j.acct['cores']/self.j.acct['nodes']
      except ZeroDivisionError:
        print "Read zero nodes, assuming 16 way for job " + str(self.j.id)
        self.wayness=16
      except TypeError:
        raise TSPLException('Something is funny with job ' +str(self.j.id) +
                            ' ' + str(self.j.acct['cores']) + ' ' +
                            str(self.j.acct['nodes']))

    try:
      self.queue=self.j.acct['queue']
    except KeyError:
      print 'No queue found'
      self.queue = None

    self.status='Unknown'
    try:
      self.status=self.j.acct['status']
    except KeyError:
      try:
        self.status=self.j.acct['exit_stats']
      except KeyError as e:
        pass
      pass

    try:
      self.owner=self.j.acct['owner']
    except KeyError:
      try:
        import pwd
        self.owner=pwd.getpwuid(int(self.j.acct['uid']))[0]
      except Exception as e:
        self.owner=self.j.acct['uid']
      
    self.numhosts=len(self.j.hosts.keys())

{'fraction_non_alphanumeric': 0.12938525909237206, 'fraction_numerical': 0.005149662053427744, 'mean_word_length': 2.4078947368421053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4715305', 'n_tokens_mistral': 1084, 'n_tokens_neox': 1054, 'n_words': 227}