<commit_before>import numpy.linalg
import numpy.random
import scipy.stats
import scipy.io
import numpy
import math
import sys
import os

import matplotlib.mlab as mlab
import matplotlib.pyplot as plt

###############################################################################

def mat_to_stdout (mat):

    for i in range(mat.shape[0]):
       for j in range(mat.shape[1]):
           sys.stdout.write ("%f "%mat[i][j])
       sys.stdout.write ("\n")

###############################################################################

def mat_to_file (bpm, oufilename):

    if os.path.exists(oufilename):
        os.remove(oufilename)
    
    outf = open(oufilename, "w")

    for i in range(bpm.shape[0]):
        for j in range(bpm.shape[1]):
            outf.write(" %f "%(bpm[i][j]))
        outf.write("\n")
        
    outf.close()

###############################################################################

def histo_to_file (xh, h, oufilename):

    if os.path.exists(oufilename):
        os.remove(oufilename)
    
    outf = open(oufilename, "w")

    for i in range(len(h)):
        outf.write("%f %f\n"%((xh[i]+xh[i+1])/2.0, h[i]))
        
    outf.close()

###############################################################################

def progress_bar (count, total, status=''):

    bar_len = 60
    filled_len = int(round(bar_len * count / float(total)))

    percents = round(100.0 * count / float(total), 1)
    bar = '=' * filled_len + '-' * (bar_len - filled_len)

    sys.stdout.write('[%s] %s%s ...%s\r' % (bar, percents, '%', status))
    sys.stdout.flush() 

###############################################################################

def get_histo (v, step):
    
    minh = min(v)
    maxh = max(v)
    nbins = int ((maxh - minh) / step)
    h, xh = numpy.histogram(v, bins=nbins, range=(minh, maxh))
    p = h/float(sum(h))

    t = sum((p*(math.log(len(p)))*p))

    bins = []
    for i in range(len(h)):
        bins.append((xh[i]+xh[i+1])/2.0) 

    return p, t, h, xh, nbins

###############################################################################

filename1 = "ms.mat"
filename2 = "bp.mat"
step = 0.25 
run = 100000
tprev = 37 # mesi previsione
namems = 'ms'
namebp = 'i_r'
#timeinf = False
timeinf = True

if len(sys.argv) != 6 and len(sys.argv) != 7:
    print "usage: ", sys.argv[0], \
            " msmatfilename bpmatfilename step tprev run [matname]" 
    exit(1)
else:
    if len(sys.argv) == 6:
      filename1 = sys.argv[1] 
      filename2 = sys.argv[2]
      step = float(sys.argv[3])
      tprev = int(sys.argv[4])
      run = int(sys.argv[5])
    elif len(sys.argv) == 7:
      filename1 = sys.argv[1] 
      filename2 = sys.argv[2]
      step = float(sys.argv[3])
      tprev = int(sys.argv[4])
      run = int(sys.argv[5])
      namems = sys.argv[6]

#numpy.random.seed(9001)

msd = scipy.io.loadmat(filename1)
bpd = scipy.io.loadmat(filename2)

if not(namems in msd.keys()):
    print "Cannot find ", namems, " in ", filename1
    print msd.keys()
    exit(1)

if not(namebp in bpd.keys()):
    print "Cannot find ", namebp, " in ", filename2
    print bpd.keys()
    exit(1)

if msd[namems].shape[0] != bpd[namebp].shape[0]:
    print "wrong dim of the input matrix"
    exit(1)

countries = msd[namems].shape[0]
rating = numpy.max(msd[namems])

if (rating <= 0) or (rating > 8):
    print "rating ", rating, " is not a valid value"
    exit(1)

ms = msd[namems]
i_r = bpd[namebp]
time = len(ms[1,:])

Nk = numpy.zeros((rating,rating,countries), dtype='int64')
Num = numpy.zeros((rating,rating), dtype='int64')
Den = numpy.zeros(rating, dtype='int64')
Pr = numpy.zeros((rating,rating), dtype='float64')

verbose = False

for k in range(countries):
    for t in range(time-1):
        for i in range(rating):
            for j in range(rating):
                if (ms[k][t] == (i+1)) and (ms[k][t+1] == (j+1)):
                    Nk[i][j][k] = Nk[i][j][k] + 1

                Num[i][j] = sum(Nk[i][j])

            Den[i] = sum(Num[i])

    progress_bar(k+1, countries)

#print Num 
#print Den

for i in range(rating):
    for j in range(rating):
        if Den[i] != 0:
            Pr[i][j] = float(Num[i][j])/float(Den[i])
        else: 
            Pr[i][j] = 0.0
print ""
if timeinf:
    ai = numpy.identity(rating, dtype='float64') - numpy.matrix.transpose(Pr)
    a = numpy.zeros((rating+1,rating), dtype='float64')
    for i in range(rating):
        for j in range(rating):
            a[i][j] = ai[i][j]
    for i in range(rating):
        a[rating][i] = 1.0 

    b = numpy.zeros(rating+1, dtype='float64')
    b[rating] = 1.0
    #x = numpy.linalg.solve(a, b)
    x = numpy.linalg.lstsq(a, b)
    print x[0]

print numpy.linalg.matrix_power(Pr, 20000)

exit()

newPr = Pr - numpy.identity(rating, dtype='float64')

#print newPr

s, v, d = numpy.linalg.svd(newPr)
print numpy.mean(v)

for i in range(len(i_r)):
    for j in range(len(i_r[0])):
        if math.isnan(i_r[i][j]):
           i_r[i][j] = float('inf')

benchmark = numpy.amin(i_r, 0)

r = numpy.zeros((countries,time), dtype='float64') 

for k in range(countries):
    for Time in range(time):
        r[k][Time] = i_r[k][Time] - benchmark[Time]

for i in range(len(r)):
    for j in range(len(r[0])):
        if (r[i][j] == float('Inf')):
           r[i][j] = float('nan')

ist = numpy.zeros((rating,time*countries), dtype='float64')
Nn = numpy.zeros((rating), dtype='int')

for i in range(rating):
    for k in range(countries):
        for Time in range(time):
            if ms[k][Time] == i+1: 
                Nn[i] = Nn[i] + 1 <commit_msg>using by defaulkt infinity time<commit_after>import numpy.linalg
import numpy.random
import scipy.stats
import scipy.io
import numpy
import math
import sys
import os

import matplotlib.mlab as mlab
import matplotlib.pyplot as plt

###############################################################################

def mat_to_stdout (mat):

    for i in range(mat.shape[0]):
       for j in range(mat.shape[1]):
           sys.stdout.write ("%f "%mat[i][j])
       sys.stdout.write ("\n")

###############################################################################

def mat_to_file (bpm, oufilename):

    if os.path.exists(oufilename):
        os.remove(oufilename)
    
    outf = open(oufilename, "w")

    for i in range(bpm.shape[0]):
        for j in range(bpm.shape[1]):
            outf.write(" %f "%(bpm[i][j]))
        outf.write("\n")
        
    outf.close()

###############################################################################

def histo_to_file (xh, h, oufilename):

    if os.path.exists(oufilename):
        os.remove(oufilename)
    
    outf = open(oufilename, "w")

    for i in range(len(h)):
        outf.write("%f %f\n"%((xh[i]+xh[i+1])/2.0, h[i]))
        
    outf.close()

###############################################################################

def progress_bar (count, total, status=''):

    bar_len = 60
    filled_len = int(round(bar_len * count / float(total)))

    percents = round(100.0 * count / float(total), 1)
    bar = '=' * filled_len + '-' * (bar_len - filled_len)

    sys.stdout.write('[%s] %s%s ...%s\r' % (bar, percents, '%', status))
    sys.stdout.flush() 

###############################################################################

def get_histo (v, step):
    
    minh = min(v)
    maxh = max(v)
    nbins = int ((maxh - minh) / step)
    h, xh = numpy.histogram(v, bins=nbins, range=(minh, maxh))
    p = h/float(sum(h))

    t = sum((p*(math.log(len(p)))*p))

    bins = []
    for i in range(len(h)):
        bins.append((xh[i]+xh[i+1])/2.0) 

    return p, t, h, xh, nbins

###############################################################################

filename1 = "ms.mat"
filename2 = "bp.mat"
step = 0.25 
run = 100000
tprev = 37 # mesi previsione
namems = 'ms'
namebp = 'i_r'
#timeinf = False
timeinf = True

if len(sys.argv) != 6 and len(sys.argv) != 7:
    print "usage: ", sys.argv[0], \
            " msmatfilename bpmatfilename step tprev run [matname]" 
    exit(1)
else:
    if len(sys.argv) == 6:
      filename1 = sys.argv[1] 
      filename2 = sys.argv[2]
      step = float(sys.argv[3])
      tprev = int(sys.argv[4])
      run = int(sys.argv[5])
    elif len(sys.argv) == 7:
      filename1 = sys.argv[1] 
      filename2 = sys.argv[2]
      step = float(sys.argv[3])
      tprev = int(sys.argv[4])
      run = int(sys.argv[5])
      namems = sys.argv[6]

#numpy.random.seed(9001)

msd = scipy.io.loadmat(filename1)
bpd = scipy.io.loadmat(filename2)

if not(namems in msd.keys()):
    print "Cannot find ", namems, " in ", filename1
    print msd.keys()
    exit(1)

if not(namebp in bpd.keys()):
    print "Cannot find ", namebp, " in ", filename2
    print bpd.keys()
    exit(1)

if msd[namems].shape[0] != bpd[namebp].shape[0]:
    print "wrong dim of the input matrix"
    exit(1)

countries = msd[namems].shape[0]
rating = numpy.max(msd[namems])

if (rating <= 0) or (rating > 8):
    print "rating ", rating, " is not a valid value"
    exit(1)

ms = msd[namems]
i_r = bpd[namebp]
time = len(ms[1,:])

Nk = numpy.zeros((rating,rating,countries), dtype='int64')
Num = numpy.zeros((rating,rating), dtype='int64')
Den = numpy.zeros(rating, dtype='int64')
Pr = numpy.zeros((rating,rating), dtype='float64')

verbose = False

for k in range(countries):
    for t in range(time-1):
        for i in range(rating):
            for j in range(rating):
                if (ms[k][t] == (i+1)) and (ms[k][t+1] == (j+1)):
                    Nk[i][j][k] = Nk[i][j][k] + 1

                Num[i][j] = sum(Nk[i][j])

            Den[i] = sum(Num[i])

    progress_bar(k+1, countries)

#print Num 
#print Den

for i in range(rating):
    for j in range(rating):
        if Den[i] != 0:
            Pr[i][j] = float(Num[i][j])/float(Den[i])
        else: 
            Pr[i][j] = 0.0

print ""
if timeinf:
    ai = numpy.identity(rating, dtype='float64') - numpy.matrix.transpose(Pr)
    a = numpy.zeros((rating+1,rating), dtype='float64')
    for i in range(rating):
        for j in range(rating):
            a[i][j] = ai[i][j]
    for i in range(rating):
        a[rating][i] = 1.0 

    b = numpy.zeros(rating+1, dtype='float64')
    b[rating] = 1.0
    #x = numpy.linalg.solve(a, b)
    x = numpy.linalg.lstsq(a, b)
    print x[0]
    #print numpy.linalg.matrix_power(Pr, 20000)
    for j in range(rating):
        for i in range(rating):
            Pr[i][j] = x[0][j] 

print Pr

newPr = Pr - numpy.identity(rating, dtype='float64')

s, v, d = numpy.linalg.svd(newPr)
print numpy.mean(v)

for i in range(len(i_r)):
    for j in range(len(i_r[0])):
        if math.isnan(i_r[i][j]):
           i_r[i][j] = float('inf')

benchmark = numpy.amin(i_r, 0)

r = numpy.zeros((countries,time), dtype='float64') 

for k in range(countries):
    for Time in range(time):
        r[k][Time] = i_r[k][Time] - benchmark[Time]

for i in range(len(r)):
    for j in range(len(r[0])):
        if (r[i][j] == float('Inf')):
           r[i][j] = float('nan')

ist = numpy.zeros((rating,time*countries), dtype='float64')
Nn = numpy.zeros((rating), dtype='int')

for i in range(rating):
    for k in range(countries):
        for Time in range(time):
            if ms[k][Time] == i+1: 
                Nn[i] = Nn[i] + 1 
{'fraction_non_alphanumeric': 0.24727755644090305, 'fraction_numerical': 0.020451527224435592, 'mean_word_length': 2.9482698357217756, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5466307', 'n_tokens_mistral': 4349, 'n_tokens_neox': 4160, 'n_words': 941}