<jupyter_start><jupyter_text>- - -script para converter a planilha de pesos em um dicionário javascript:<jupyter_code>df = pd.read_csv('data/weights3.csv', delimiter=';')
d = dict()

QUESTION = str()
INDEX    = int()

for i,r in df.iterrows():
    if str(r['question']) != 'nan':
        INDEX    = str(i).zfill(3)
        d[INDEX] = dict()
        d[INDEX]['question']   = r['question']
        d[INDEX]['answers'] = [ [r['answer'], [float(r['w0']),
                                                 float(r['w1']),
                                                 float(r['w2']),
                                                 float(r['w3']),
                                                 float(r['w4'])] 
                                   ] 
                                 ]
        d[INDEX]['button'] = r['button']
        d[INDEX]['image']  = r['img']
    else:
        d[INDEX]['answers'].append( [r['answer'], [float(r['w0']),
                                                 float(r['w1']),
                                                 float(r['w2']),
                                                 float(r['w3']),
                                                 float(r['w4'])] 
                                   ]  )
with open('quizquestions.js', 'w') as qq:
    for k in sorted(d):
        qq.write(
"""
{{
      question: "{}",
      answers: {{
""".format(d[k]['question']))

                
        for i,v in enumerate(d[k]['answers']):
            if i < len(d[k]['answers'])-1:
                qq.write(
"""
        {}:"{}",
""".format(str(i), v[0]))
            else:
                qq.write(
"""
        {}:"{}" }},
""".format(str(i), v[0]))
        
        qq.write(
"""
      weights: [
""")
                
        for i,v in enumerate(d[k]['answers']):
            if i < len(d[k]['answers'])-1:
                qq.write(
"""
                    {},
""".format(repr(v[1])))
            else:
                qq.write(
"""
                    {} ],
      img: "{}",
      button: "{}"
    }},
""".format(repr(v[1]), d[k]['image'], d[k]['button']))<jupyter_output><empty_output><jupyter_text>- - -transformando os pesos em tensor:<jupyter_code>max_answer_shape = 0
# get max_answer_shape
for k in sorted(d):
    q = d[k]
    if len(q['answers']) > max_answer_shape:
        max_answer_shape = len(q['answers'])

T = np.multiply.outer(np.multiply.outer(np.zeros(len(d)), np.zeros(max_answer_shape)), np.zeros(5))
T.shape
index = 0
for k in sorted(d):
    q = d[k]
    for i,v in enumerate(q['answers']):
        T[index][i] = v[1]
    
    index += 1
pkl.dump( T, open( "B.p", "wb" ) )<jupyter_output><empty_output><jupyter_text>- - -convertendo questionário em dataframe<jupyter_code>df = pd.read_csv('data/weights_calibration.csv')
DATA     = dict()
PROFILES = 4
max_answer_shape = 5

for i in range(PROFILES):
    COL = 'answers_{}'.format(str(i))
    
    d         = dict()
    DATA[COL] = dict()
    QUESTION = str()
    INDEX    = int()
    ANS_ID   = 0
    
    for i,r in df.iterrows():
        if str(r['question']) != 'nan':
            INDEX    = str(i).zfill(3)
            ANS_ID   = 0
            
            if str(r['question']) != 'TARGET':            
                d[INDEX] = dict()
                d[INDEX]['question']   = r['question']
                d[INDEX]['answers']    = np.zeros(max_answer_shape)
                if str(r[COL]) != 'nan':
                    d[INDEX]['answers'][ANS_ID] = 1

                ANS_ID += 1
            else:
                d['TARGET'] = np.zeros(max_answer_shape)
                d['TARGET'][int(r[COL])] = 1
        else:
            if str(r[COL]) != 'nan':
                d[INDEX]['answers'][ANS_ID] = 1
            
            ANS_ID += 1

    for i,k in enumerate(sorted(d)):
        if k != "TARGET":
            DATA[COL]['ans_{}'.format(str(i).zfill(2))] = d[k]['answers']
        else:
            DATA[COL]['TARGET'] = d[k]
data = pd.DataFrame.from_dict(DATA, orient='index')
df = data[
['ans_00',
'ans_01',
'ans_02',
'ans_03',
'ans_04',
'ans_05',
'ans_06',
'ans_07',
'ans_08',
'ans_09',
'ans_10',
'ans_11',
'ans_12',
'TARGET']]
df<jupyter_output><empty_output><jupyter_text>- - -backpropagation de pobre:<jupyter_code>max_answer_shape = 5
Q                = 14
WEIGHTS          = pkl.load( open( "B.p", "rb" ) )
ANS              = ['ans_00',
                    'ans_01',
                    'ans_02',
                    'ans_03',
                    'ans_04',
                    'ans_05',
                    'ans_06',
                    'ans_07',
                    'ans_08',
                    'ans_09',
                    'ans_10',
                    'ans_11',
                    'ans_12']
eta = 0.015

def back(_df):
    
    A = _df[ ANS ].T.values
    y = _df[ 'TARGET' ]
    
    A = np.stack(A, axis=0)
    B = np.copy(WEIGHTS)
    C = np.zeros( (len(A),5) )
    
    for i in range(len(A)):
        C[i] = np.sum(B[i]*A[i].T[:, np.newaxis], axis=0)
    
    D = np.sum(C, axis=0)
    
    #
    
    W = np.copy(WEIGHTS)
    
    for ITERATIONS in range(250):
        B = np.copy(W)
        for i in range(len(A)):
            C[i] = np.sum(B[i]*A[i].T[:, np.newaxis], axis=0)
        D = np.sum(C, axis=0)
        
        dW = 0*np.copy(WEIGHTS)
        
        for I in range(len(dW)):
            for J in range(len(dW[0])):
                for K in range(len(dW[0][0])):
                    OMEGA = (D[K] - y[K] )
                    dL    = 2 * OMEGA * A[I][J]

                    dW[I][J][K] = dL
        W = np.copy(W -eta*dW)
    
    return W
avg_W = [ back(df.iloc[i]) for i in range(df.shape[0]) ]
avg_W = avg_W + WEIGHTS
Wstar = sum(avg_W)/len(avg_W)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17984983411908503, 'fraction_numerical': 0.01885804085908853, 'mean_word_length': 1.4097602019352125, 'pattern_counts': {'":': 1, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '339259', 'license': 'MIT', 'n_tokens_mistral': 2105, 'n_tokens_neox': 1974, 'n_words': 355, 'path': 'tmp/semana07_dataprep.ipynb', 'repo_name': 'thiagodsd/teafinbhv', 'size': '5727'}