<commit_before>import numpy as np, pandas as pd
import time, os
from synthicity.utils import misc
import warnings

warnings.filterwarnings('ignore',category=pd.io.pytables.PerformanceWarning)

# this is the central location to do all the little data format issues that will be needed by all models

class Dataset(object):

  def __init__(self,filename):
      self.store = pd.HDFStore(filename)
      self.d = {} # keep track of in memory pandas data frames so as not to load multiple times form disk
      self.coeffs = pd.DataFrame() # keep all coefficients in memory
      self.attrs = {} # keep all computed outputs in memory
      self.savetbls = [] # names of tables to write to output hdf5

  def close(self):
    self.store.close()

  def save_toinputfile(self,name,df):
    self.store[name] = df

  def save_tmptbl(self,name,tbl):
    # used to pass a different table instead of the one that comes from the database
    self.d[name] = tbl

  def save_table(self,name):
    if name not in self.savetbls: self.savetbls.append(name)

  def save_output(self,filename):
    outstore = pd.HDFStore(filename)
    for key in self.attrs.keys():
      df = self.attrs[key]
      df = misc.df64bitto32bit(df)
      print key+"\n", df.describe()
      outstore[key] = df
    
    for key in self.savetbls:
      df = self.fetch(key)
      df = misc.df64bitto32bit(df)
      print key+"\n", df.describe()
      outstore[key] = df

    outstore.close()

  def copy_coeffs(self,filename):
    coeffstore = pd.HDFStore(filename)
    self.coeffs = coeffstore['coefficients'] 
    coeffstore.close()

  def save_coeffs(self,filename):
    outstore = pd.HDFStore(filename)
    print self.coeffs
    outstore['coefficients'] = self.coeffs 
    outstore.close()

  def fetch(self,name,**kwargs):
    if name in self.d: return self.d[name]
   
    print "Fetching %s" % name
    f = "fetch_%s"%name
    if hasattr(self,f): tbl = getattr(self,f)(**kwargs)
    else: tbl = self.store[name]
    
    if hasattr(self,"modify_table"): tbl = self.modify_table(name,tbl,**kwargs) 
  
    self.d[name] = tbl
    
    return tbl
    
  def __getattr__(self,name):
    # this is a little hackish - basically if you called fetch we don't want to call fetch again
    if "fetch_" in name: raise Exception()
    return self.fetch(name)

  def load_attr(self,name,year):
    return self.attrs[name][year]

  def store_attr(self,name,year,value):
    value = misc.series64bitto32bit(value) 
    if name in self.attrs and year in self.attrs[name]: del self.attrs[name][year]
    df = self.attrs.get(name,pd.DataFrame(index=value.index))
    df[year] = value
    self.attrs[name] = df

  def load_coeff(self,name):
    return self.coeffs[(name,'coeffs')].dropna()
  
  def store_coeff(self,name,value,fnames=None):
    colname1 = (name,'coeffs')
    colname2 = (name,'fnames')
    if colname1 in self.coeffs: del self.coeffs[colname1]
    if colname2 in self.coeffs: del self.coeffs[colname2]

    d = {colname1:value}
    if fnames is not None: d[colname2] = fnames
    self.coeffs = pd.concat([self.coeffs,pd.DataFrame(d)],axis=1)
  
  # this is a shortcut function to join the table with dataset.fetch(tblname) 
  # using the foreign_key in order to add fieldname to the source table<commit_msg>changed spacing on dataset<commit_after>import numpy as np, pandas as pd
import time, os
from synthicity.utils import misc
import warnings

warnings.filterwarnings('ignore',category=pd.io.pytables.PerformanceWarning)

# this is the central location to do all the little data format issues that will be needed by all models

class Dataset(object):

  def __init__(self,filename):
      self.store = pd.HDFStore(filename)
      self.d = {} # keep track of in memory pandas data frames so as not to load multiple times form disk
      self.coeffs = pd.DataFrame() # keep all coefficients in memory
      self.attrs = {} # keep all computed outputs in memory
      self.savetbls = [] # names of tables to write to output hdf5

  def close(self):
    self.store.close()

  def save_toinputfile(self,name,df):
    self.store[name] = df

  def save_tmptbl(self,name,tbl):
    # used to pass a different table instead of the one that comes from the database
    self.d[name] = tbl

  def save_table(self,name):
    if name not in self.savetbls: self.savetbls.append(name)

  def save_output(self,filename):
    outstore = pd.HDFStore(filename)
    for key in self.attrs.keys():
      df = self.attrs[key]
      df = misc.df64bitto32bit(df)
      print key+"\n", df.describe()
      outstore[key] = df
    
    for key in self.savetbls:
      df = self.fetch(key)
      df = misc.df64bitto32bit(df)
      print key+"\n", df.describe()
      outstore[key] = df

    outstore.close()

  def copy_coeffs(self,filename):
    coeffstore = pd.HDFStore(filename)
    self.coeffs = coeffstore['coefficients'] 
    coeffstore.close()

  def save_coeffs(self,filename):
    outstore = pd.HDFStore(filename)
    print self.coeffs
    outstore['coefficients'] = self.coeffs 
    outstore.close()

  def fetch(self,name,**kwargs):
    if name in self.d: return self.d[name]
   
    print "Fetching %s" % name
    f = "fetch_%s"%name
    if hasattr(self,f): tbl = getattr(self,f)(**kwargs)
    else: tbl = self.store[name]
    
    if hasattr(self,"modify_table"): tbl = self.modify_table(name,tbl,**kwargs) 
  
    self.d[name] = tbl
    
    return tbl
    
  def __getattr__(self,name):
    # this is a little hackish - basically if you called fetch we don't want to call fetch again
    if "fetch_" in name: raise Exception()
    return self.fetch(name)

  def load_attr(self,name,year):
    return self.attrs[name][year]

  def store_attr(self,name,year,value):
    value = misc.series64bitto32bit(value)
    if name in self.attrs and year in self.attrs[name]: del self.attrs[name][year]
    df = self.attrs.get(name,pd.DataFrame(index=value.index))
    df[year] = value
    self.attrs[name] = df

  def load_coeff(self,name):
    return self.coeffs[(name,'coeffs')].dropna()
  
  def store_coeff(self,name,value,fnames=None):
    colname1 = (name,'coeffs')
    colname2 = (name,'fnames')
    if colname1 in self.coeffs: del self.coeffs[colname1]
    if colname2 in self.coeffs: del self.coeffs[colname2]

    d = {colname1:value}
    if fnames is not None: d[colname2] = fnames
    self.coeffs = pd.concat([self.coeffs,pd.DataFrame(d)],axis=1)
  
  # this is a shortcut function to join the table with dataset.fetch(tblname) 
  # using the foreign_key in order to add fieldname to the source table
{'fraction_non_alphanumeric': 0.11209529627367135, 'fraction_numerical': 0.006719609040928528, 'mean_word_length': 3.4041694687289845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6935663', 'n_tokens_mistral': 2292, 'n_tokens_neox': 2199, 'n_words': 604}