<jupyter_start><jupyter_text>Imports<jupyter_code>import pandas as pd
import numpy as np
from itertools import combinations
from time import time
from scipy.stats import fisher_exact
from jupyterthemes import jtplot
jtplot.style(theme='onedork', context='talk', fscale=1.8, spines=False, gridlines='--', ticks=True, grid=False, figsize=(12, 8))
import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Load the data & build the product-transaction matrix<jupyter_code>def get_transaction_data():
    """Load groceries transaction data into DataFrame"""
    df = pd.read_csv('grocery_transactions.csv')
    df = df.stack().reset_index(-1, drop=True)
    df.index.names = ['tx_id']
    return pd.get_dummies(df, prefix='', prefix_sep='').groupby(level='tx_id').sum()
data = get_transaction_data()

item_id = pd.Series(dict(enumerate(data.columns)))
transactions = data.values
n_txn = transactions.shape[0]
min_support = 0.01

item_length = 1
candidates = list(zip(item_id.index))
itemsets = pd.DataFrame(columns=['support', 'length'])

new_rules = []
rule_data = ['itemset', 'antecedent', 'consequent',
             'support_rule', 'support_antecedent', 'support_consequent',
             'confidence', 'lift', 'pvalue']
rules = pd.DataFrame(columns=rule_data)
def prune_candidates(all_txn, candidates, candidate_size, min_support):
    """Return DataFrame with itemsets of candidate_size with min_support
        all_txn: numpy array of transaction-product matrix
        candidates: list of tuples containing product id
        candidate_size: length of item set
        min_support: support threshold
        """
    itemsets = {}
    for candidate in candidates:
        candidate_txn = all_txn[:, candidate].reshape(-1, candidate_size)
        relevant_txn = candidate_txn[(candidate_txn == 1).all(axis=1)]
        support = relevant_txn.shape[0] / all_txn.shape[0]
        if support >= min_support:
            itemsets[frozenset(candidate)] = support
    return pd.Series(itemsets).to_frame('support').assign(length=candidate_size)
def find_association_rules(itemsets, n_txn, n_items, min_confidence=0, min_lift=0, min_pvalue=0):
    """Find rules {antecedent} => {consequent} with min_confidence, min_lift and min_pvalue
    itemsets: DataFrame containing all itemsets and their support
    min_confidence, min_lift, min_pvalue: confidence & lift & pvalue thresholds
    """
    support = itemsets.loc[:, 'support'].to_dict()
    new_rules = []
    for itemset in itemsets.loc[itemsets.length == n_items].index:
        for n_antecedents in range(1, n_items):
            antecedents = [frozenset(a)
                           for a in combinations(itemset, r=n_antecedents)]
            for antecedent in antecedents:
                consequent = itemset.difference(antecedent)
                sAC = support[itemset]
                sA, sC = support[antecedent], support[consequent]
                confidence = sAC / sA
                lift = sAC / (sA * sC)
                contingency_table = n_txn * np.array([[sAC, sA - sAC],
                                        [sC - sAC, 1 - sA - sC + sAC]])
                _, p_value = fisher_exact(contingency_table,
                                          alternative='greater')

                if (confidence >= min_confidence) and (lift >= min_lift) and (p_value >= min_pvalue):
                    new_rule = [itemset, antecedent, consequent,
                                support[itemset], support[antecedent], support[consequent],
                                confidence, lift, p_value]
                    new_rules.append(new_rule)
    return new_rules
while candidates:
    new_items = prune_candidates(transactions, candidates, item_length, min_support)
    itemsets = itemsets.append(new_items)

    if item_length > 1:
        new_rules = find_association_rules(itemsets, n_txn, item_length)
        rules = pd.concat([rules, pd.DataFrame(new_rules, columns=rules.columns)], ignore_index=True)
        
    print('Itemset Length {}\tCandidates: {:>7,.0f}\tNew Items: {:>7,.0f}\tNew Rules: {:>7,.0f}'.format(
            item_length, len(candidates), len(new_items), len(new_rules)))
    
    item_length += 1
    remaining_items = np.unique([item for t in new_items.index for item in t])
    candidates = list(combinations(remaining_items, r=item_length))

rules = rules.apply(pd.to_numeric, errors='ignore')
rules.info()
rules.head()
with pd.HDFStore('rules.h5') as store:
    store.put('rules', rules)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11465669014084508, 'fraction_numerical': 0.006602112676056338, 'mean_word_length': 3.2200557103064065, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '643315', 'license': 'MIT', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1383, 'n_words': 312, 'path': 'Section 2/03_association_rules.ipynb', 'repo_name': 'PacktPublishing/Hands-on-Unsupervised-Learning-with-Python', 'size': '4544'}