<commit_before># -*- coding:utf-8 -*-
# Usage : spark-submit ~~.py
# Usage : spark-submit --properties-file spark.conf ~~.py
from pyspark.context import SparkContext
from pyspark.mllib.fpm import FPGrowth
from pyspark import StorageLevel
from pyspark import SparkConf
import sys
import os
import pickle
import collections
import pandas as pd
import numpy as np
from itertools import chain
from itertools import combinations
from itertools import compress
from itertools import product
from sklearn.metrics import accuracy_score
from multiprocessing import Pool
from multiprocessing import freeze_support

# Global Setting
DIR_UCI = '/mnt/data/uci'

# ------------------------------------------------------
# Rule Class
# ------------------------------------------------------
class Rule :
   def __init__(self):
       self.value = list()
       self.consequent = list()
       self.strength = float()
       self.support = list()
       self.support_v = float()
       self.conf = float()
   def setValue(self, values) :
       self.value = values
   def setConsequent(self, consequents) :
       self.consequent = consequents
   def setStrength(self, strength) :
       self.strength = strength
   def setSupport(self, supports) :
       self.support = supports
   def setSupportV(self, support_v):
       self.support_v = support_v
   def setConf(self, confidence) :
       self.conf = confidence
   def getValue(self) :
       return(self.value)
   def getConsequent(self) :
       return(self.consequent)
   def getStrength(self):
       return(self.strength)
   def getSupport(self) :
       return(self.support)
   def getSupportV(self) :
       return(self.support_v)
   def getSupportD(self) :
       return(self.support_v * len(self.value))<commit_msg>update fpgrowth<commit_after># -*- coding:utf-8 -*-
# Usage : spark-submit ~~.py
# Usage : spark-submit --properties-file spark.conf ~~.py
from pyspark.context import SparkContext
from pyspark.mllib.fpm import FPGrowth
from pyspark import StorageLevel
from pyspark import SparkConf
import sys
import os
import pickle
import collections
import pandas as pd
import numpy as np
from itertools import chain
from itertools import combinations
from itertools import compress
from itertools import product
from sklearn.metrics import accuracy_score
from multiprocessing import Pool
from multiprocessing import freeze_support

# Global Setting
DIR_UCI = '/mnt/data/uci'

# ------------------------------------------------------
# Rule Class
# ------------------------------------------------------
class Rule :
   def __init__(self):
       self.value = list()
       self.consequent = list()
       self.strength = float()
       self.support = list()
       self.support_v = float()
       self.conf = float()
   def setValue(self, values) :
       self.value = values
   def setConsequent(self, consequents) :
       self.consequent = consequents
   def setStrength(self, strength) :
       self.strength = strength
   def setSupport(self, supports) :
       self.support = supports
   def setSupportV(self, support_v):
       self.support_v = support_v
   def setConf(self, confidence) :
       self.conf = confidence
   def getValue(self) :
       return(self.value)
   def getConsequent(self) :
       return(self.consequent)
   def getStrength(self):
       return(self.strength)
   def getSupport(self) :
       return(self.support)
   def getSupportV(self) :
       return(self.support_v)
   def getSupportD(self) :
       return(self.strength * len(self.value))
{'fraction_non_alphanumeric': 0.14411347517730497, 'fraction_numerical': 0.0005673758865248227, 'mean_word_length': 2.5833333333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5739333', 'n_tokens_mistral': 1014, 'n_tokens_neox': 991, 'n_words': 282}