# Path: tensorlog/matrixdb.py
# NULL_ENTITY_NAME = dbschema.NULL_ENTITY_NAME
# THING = dbschema.THING
# OOV_ENTITY_NAME = dbschema.OOV_ENTITY_NAME
# TRAINABLE_DECLARATION_FUNCTOR = 'trainable'
# class MatrixDB(object):
#   def __init__(self,initSchema=None):
#   def checkTyping(self):
#   def isTypeless(self):
#   def _fillDefault(self,typeName):
#   def dim(self,typeName=None):
#   def onehot(self,s,typeName=None,outOfVocabularySymbolsAllowed=False):
#   def zeros(self,numRows=1,typeName=None):
#   def ones(self,typeName=None):
#   def nullMatrix(self,numRows=1,typeName=None,numCols=0):
#   def transposeNeeded(mode,transpose=False):
#   def matrix(self,mode,transpose=False):
#   def vector(self,mode):
#   def matrixPreimage(self,mode):
#   def matrixPreimageMat(self,mode):
#   def matrixPreimageOnesType(self,mode):
#   def matrixPreimageOnes(self,mode):
#   def isParameter(self,mode):
#   def markAsParam(self,functor,arity):
#   def markAsParameter(self,functor,arity):
#   def clearParameterMarkings(self):
#   def getParameter(self,functor,arity):
#   def parameterIsInitialized(self,functor,arity):
#   def setParameter(self,functor,arity,replacement):
#   def asSymbol(self,symbolId,typeName=None):
#   def asSymbolId(self,symbol,typeName=None):
#   def rowAsSymbolDict(self,row,typeName=None):
#   def arrayAsSymbolDict(self,arr,typeName=None):
#   def matrixAsSymbolDict(self,m,typeName=None):
#   def matrixAsPredicateFacts(self,functor,arity,m):
#   def inDB(self,functor,arity):
#   def summary(self,functor,arity):
#   def listing(self):
#   def numMatrices(self):
#   def size(self):
#   def parameterSize(self):
#   def createPartner(self):
#   def serialize(self,direc):
#   def serializeDataTo(self,fileLike,filter=None):
#   def importSerializedDataFrom(self,fileLike):
#   def deserializeDataFrom(fileLike):
#   def _saveMatDictWithScipy(fileLike,d):
#   def _restoreMatDictWithScipy(fileLike):
#   def deserialize(direc):
#   def uncache(dbFile,factFile,initSchema=None):
#   def addLines(self,lines):
#   def loadFile(filenames,initSchema=None):
#   def startBuffers(self):
#   def bufferFile(self,filename):
#   def flushBuffers(self):
#   def _flushBuffer(self,functor,arity):
#   def _bufferTriplet(self,functor,arity,a1,a2,w,filename,k):
#   def _bufferLine(self,line,filename,k):
#     def _atof(s):
# 
# Path: tensorlog/mutil.py
# NONETYPE=type(None) 
#         D = SS.csr_matrix((mat.data,mat.indices-loIndex,mat.indptr),shape=newShape,dtype='float32').todense()
#     D1 = SS.csr_matrix((m1.data,m1.indices-loIndex,m1.indptr),shape=newShape1,dtype='float32').todense()
#     D2 = SS.csr_matrix((m2.data,m2.indices-loIndex,m2.indptr),shape=newShape2,dtype='float32').todense()
# def summary(mat):
# def pprintSummary(mat):
# def checkCSR(mat,context='unknown'):
# def checkNoNANs(mat,context='unknown'):
# def maxValue(mat):
# def densify(mat,maxExpandFactor=-1,maxExpandIntercept=-1):
# def denseSize(m,loIndex,hiIndex):
# def sparseSize(m,loIndex,hiIndex):
# def codensify(m1,m2,maxExpandFactor=-1,maxExpandIntercept=-1):
# def undensify(denseMat, info):
# def mean(mat):
# def rowsum(mat):
# def mapData(dataFun,mat):
# def stack(mats):
# def numRows(m):
# def numCols(m):
# def nzCols(m,i):
# def repeat(row,n):
# def alterMatrixRows(mat,alterationFun):
# def softmax(db,mat):
#         def softMaxAlteration(data,lo,hi,unused):
# def denseSoftmax(m):
# def broadcastAndComponentwiseMultiply(m1,m2):
# def multiplyByBroadcastRowVec(m,v):
# def broadcastAndWeightByRowSum(m1,m2):
# def shuffleRows(m,shuffledRowNums=None):
# def selectRows(m,lo,hi):


import math
import numpy as np
import scipy.sparse as sp
import tensorflow as tf
from tensorlog import matrixdb,mutil
  from tensorflow.python.framework import ops
# random test/experimental code that william unaccountably wanted to
# check into git




def go():
  db = matrixdb.MatrixDB.loadFile("test-data/textcattoy_corpus.cfacts")
  m = db.matEncoding[('hasWord',2)]
  print('m',m.shape,m.nnz)
  def tfidf_transform(tf_matrix):
    # implements Salton's TFIDF transformation, ie l2-normalized
    # vector after scaling by: log(tf+1.0) * log(#docs/idf)
    df = tf_matrix.sum(axis=0)
    # count docs by summing word counts w/in each row of m,
    # clipping the sum down to 1, and and then adding up
    ndoc = np.clip(tf_matrix.sum(axis=1),0.0,1.0).sum()
    # this ensures idfs will be zero for any terms with df==0, after I
    # take the log
    df[df==0] = ndoc
    idf = np.log( np.reciprocal(df) * ndoc )
    # first compute log(tf+1.0)
    scaled_tf_matrix = mutil.mapData(lambda v:np.log(v+1.0),tf_matrix)
    # now multiply by idf factor
    unnormalized_tfidf_matrix = mutil.broadcastAndComponentwiseMultiply(scaled_tf_matrix, sp.csr_matrix(idf))
    # compute normalizer needed for l2 normalization

# Path: tensorlog/matrixdb.py
# NULL_ENTITY_NAME = dbschema.NULL_ENTITY_NAME
# THING = dbschema.THING
# OOV_ENTITY_NAME = dbschema.OOV_ENTITY_NAME
# TRAINABLE_DECLARATION_FUNCTOR = 'trainable'
# class MatrixDB(object):
#   def __init__(self,initSchema=None):
#   def checkTyping(self):
#   def isTypeless(self):
#   def _fillDefault(self,typeName):
#   def dim(self,typeName=None):
#   def onehot(self,s,typeName=None,outOfVocabularySymbolsAllowed=False):
#   def zeros(self,numRows=1,typeName=None):
#   def ones(self,typeName=None):
#   def nullMatrix(self,numRows=1,typeName=None,numCols=0):
#   def transposeNeeded(mode,transpose=False):
#   def matrix(self,mode,transpose=False):
#   def vector(self,mode):
#   def matrixPreimage(self,mode):
#   def matrixPreimageMat(self,mode):
#   def matrixPreimageOnesType(self,mode):
#   def matrixPreimageOnes(self,mode):
#   def isParameter(self,mode):
#   def markAsParam(self,functor,arity):
#   def markAsParameter(self,functor,arity):
#   def clearParameterMarkings(self):
#   def getParameter(self,functor,arity):
#   def parameterIsInitialized(self,functor,arity):
#   def setParameter(self,functor,arity,replacement):
#   def asSymbol(self,symbolId,typeName=None):
#   def asSymbolId(self,symbol,typeName=None):
#   def rowAsSymbolDict(self,row,typeName=None):
#   def arrayAsSymbolDict(self,arr,typeName=None):
#   def matrixAsSymbolDict(self,m,typeName=None):
#   def matrixAsPredicateFacts(self,functor,arity,m):
#   def inDB(self,functor,arity):
#   def summary(self,functor,arity):
#   def listing(self):
#   def numMatrices(self):
#   def size(self):
#   def parameterSize(self):
#   def createPartner(self):
#   def serialize(self,direc):
#   def serializeDataTo(self,fileLike,filter=None):
#   def importSerializedDataFrom(self,fileLike):
#   def deserializeDataFrom(fileLike):
#   def _saveMatDictWithScipy(fileLike,d):
#   def _restoreMatDictWithScipy(fileLike):
#   def deserialize(direc):
#   def uncache(dbFile,factFile,initSchema=None):
#   def addLines(self,lines):
#   def loadFile(filenames,initSchema=None):
#   def startBuffers(self):
#   def bufferFile(self,filename):
#   def flushBuffers(self):
#   def _flushBuffer(self,functor,arity):
#   def _bufferTriplet(self,functor,arity,a1,a2,w,filename,k):
#   def _bufferLine(self,line,filename,k):
#     def _atof(s):
# 
# Path: tensorlog/mutil.py
# NONETYPE=type(None) 
#         D = SS.csr_matrix((mat.data,mat.indices-loIndex,mat.indptr),shape=newShape,dtype='float32').todense()
#     D1 = SS.csr_matrix((m1.data,m1.indices-loIndex,m1.indptr),shape=newShape1,dtype='float32').todense()
#     D2 = SS.csr_matrix((m2.data,m2.indices-loIndex,m2.indptr),shape=newShape2,dtype='float32').todense()
# def summary(mat):
# def pprintSummary(mat):
# def checkCSR(mat,context='unknown'):
# def checkNoNANs(mat,context='unknown'):
# def maxValue(mat):
# def densify(mat,maxExpandFactor=-1,maxExpandIntercept=-1):
# def denseSize(m,loIndex,hiIndex):
# def sparseSize(m,loIndex,hiIndex):
# def codensify(m1,m2,maxExpandFactor=-1,maxExpandIntercept=-1):
# def undensify(denseMat, info):
# def mean(mat):
# def rowsum(mat):
# def mapData(dataFun,mat):
# def stack(mats):
# def numRows(m):
# def numCols(m):
# def nzCols(m,i):
# def repeat(row,n):
# def alterMatrixRows(mat,alterationFun):
# def softmax(db,mat):
#         def softMaxAlteration(data,lo,hi,unused):
# def denseSoftmax(m):
# def broadcastAndComponentwiseMultiply(m1,m2):
# def multiplyByBroadcastRowVec(m,v):
# def broadcastAndWeightByRowSum(m1,m2):
# def shuffleRows(m,shuffledRowNums=None):
# def selectRows(m,lo,hi):


# Path: tensorlog/try.py
import math
import numpy as np
import scipy.sparse as sp
import tensorflow as tf
from tensorlog import matrixdb,mutil
  from tensorflow.python.framework import ops

# random test/experimental code that william unaccountably wanted to
# check into git




def go():
  db = matrixdb.MatrixDB.loadFile("test-data/textcattoy_corpus.cfacts")
  m = db.matEncoding[('hasWord',2)]
  print('m',m.shape,m.nnz)
  def tfidf_transform(tf_matrix):
    # implements Salton's TFIDF transformation, ie l2-normalized
    # vector after scaling by: log(tf+1.0) * log(#docs/idf)
    df = tf_matrix.sum(axis=0)
    # count docs by summing word counts w/in each row of m,
    # clipping the sum down to 1, and and then adding up
    ndoc = np.clip(tf_matrix.sum(axis=1),0.0,1.0).sum()
    # this ensures idfs will be zero for any terms with df==0, after I
    # take the log
    df[df==0] = ndoc
    idf = np.log( np.reciprocal(df) * ndoc )
    # first compute log(tf+1.0)
    scaled_tf_matrix = mutil.mapData(lambda v:np.log(v+1.0),tf_matrix)
    # now multiply by idf factor
    unnormalized_tfidf_matrix = mutil.broadcastAndComponentwiseMultiply(scaled_tf_matrix, sp.csr_matrix(idf))
    # compute normalizer needed for l2 normalization

    normalizer = mutil.mapData(lambda v:v*v, unnormalized_tfidf_matrix).sum(axis=1)