<jupyter_start><jupyter_text>```0. supportLevel == ? % (ie. 20%), confidence == ? % (20%)1. read in transactionsk = 1  (number of items combined) 2. do    - generate support levels for level k    - calculate support and confidence level for each "item set"     - eliminate ``` Candidate ItemSets come from frequent items _> get support level - then create L2C0 -> L1 -> C1 -> L2 -> C2 -> L3 -> C3  (until only 1 itemset is left.Calculate all support ItemSets starting from k=1, to k=n, after all done, go through each and calculate the confidence level.The support levels from ItemSets stays the same, but then calc each Confidence level using each combination of item within each ItemSet for each overall ItemSets for k=N.<jupyter_code>ta = ('i2', 'i3')
tb = ('i4',)

tc = tb + ta
set(tb).issubset(tc)
tuple1 = (1,2,3)
tuple2 = (4,5,6)
tuple3 = (3,2,1)
sorted(tuple3) == sorted(tuple2)
sorted(tuple3)
# load some helpers
import os, sys
from typing import List, Tuple
from typing import List, Tuple
from itertools import combinations, product, permutations
from abc import ABC, abstractmethod, abstractstaticmethod, abstractclassmethod
import pandas as pd
import logging


def source_file(file_name):
    f = os.path.join(os.getcwd(), file_name)
    exec(open(f).read())

    
source_file('apriori.py')
source_file('assoc.py')
from apriori import CollapsedCsvFileReader, Apriori
from assoc import generate_associations

fr = CollapsedCsvFileReader('./data/test-dm-bookch6.csv')

t2 = fr.read()

#logger.info("tran count: {0}".format(t2))
g = Apriori(t2)
#output = g.generate_levels(support_level=0.60, drop_below_support=False)
output = g.generate_levels(support_level=0.22, drop_below_support=True)

nnn = generate_associations(output)
[ item for item in nnn if not set(item[][2]).issubset(item[][0]) ]
nnn
nnn[[0]]
n2 = pd.DataFrame(nnn, columns= ['antecedent', 'support1', 'result', 'support2'])
n2
pd.set_option('display.width', 200)

possible_combos = [ ( i[1]['antecedent'] + i[1]['result'], i[1]['antecedent'], i[1]['support1'], i[1]['result'], i[1]['support2'] ) 
   for i in n2.iterrows()
   #elimiate where item on boths sides 
   if not bool(set(i[1]['result']) & set(i[1]['antecedent'])) ]
   #and set(i[1]['antecedent'] + i[1]['result']) in t2]
   #if not set(i[1]['result']).issubset(i[1]['antecedent']) ]
    

pd.set_option('max_rows', 100)
n3 = pd.DataFrame(possible_combos, columns= ['fullkey', 'antecedent', 'support1', 'result', 'support2'])
n3
def get_support_for_key(trans_list: pd.DataFrame, antecedent_key: Tuple):
    rv = 0
    srted_key = tuple(sorted(antecedent_key))
    matches = trans_list[trans_list['antecedent'] == srted_key].head(1)
    if len(matches) ==  1:
        rv =  matches['support1'].array[0]
    
    return rv
from collections import namedtuple

myRecord =  namedtuple('myRecord', ['full_key', 'antecedent', 'support1', 'result', 'support2', 'support_full_key', 'confidence'])

def calculate_confidence(data: pd.DataFrame, trans_list):
    print('starting')
    ctr = 0
    rv = list()
    for r in data.itertuples():
        full_key = r.fullkey
        ant = r.antecedent
        support1 = r.support1
        res = r.result
        support2 = r.support2
        support_full_key = get_support_for_key(data, full_key)
        if support1 != 0:
            confidence = support_full_key / support1
        else:
            confidence = -1
            
        item_rv = myRecord(full_key, ant, support1, res, support2, support_full_key, confidence )
        
        rv.append( item_rv )
        
    ctr += 1
    #print("ctr: {0}".format(ctr))
    #print(rv)
    return rv

#print(len(calculate_confidence(n3, t2)))
rrrr = calculate_confidence(n3, t2)
#len(rrrr)
ddddd = pd.DataFrame(rrrr)
ddddd[ddddd.confidence > 0]   #success!!!!
ttt = [ (sorted(i[1]+i[3]), i[1], i[3] ) for i in possible_combos ]

ttt[0][0]
[ (sorted(i[1]+i[3]), i[1], i[3] ) for i in possible_combos #]
   if sorted(i[]+i[3]) in t2 ]
t2
set(ttt[0][0]).issubset(t2
[ (ttt[0][0], i) for i in t2 if set(ttt[0][0]).issubset(i)]
ttt[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1511513963743263, 'fraction_numerical': 0.03111219990200882, 'mean_word_length': 4.17490494296578, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '469846', 'license': 'MIT', 'n_tokens_mistral': 1503, 'n_tokens_neox': 1457, 'n_words': 374, 'path': 'nb/Untitled.ipynb', 'repo_name': 'cicorias/njit-dm-scratch', 'size': '4082'}