<commit_before>import collections, math, article, codingjob, toolkit, re

PRINTFEATURES = False

def tf(tf, d, df):
    return tf
def logtfidf(tf, d, df):
    s = math.log(tf + .5) * math.log(d / float(df))
    #print "log(tf).log(idf) = log(%i + .5) * log(%i / %i) = %1.3f" % (tf, d, df, s)
    return s
def tfidf(tf, d, df):
    s = tf * math.log(d / float(df))
    #print "tf.log(idf) = %i * log(%i / %i) = %1.3f" % (tf, d, df, s)
    return s

TRANS = "SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED\n "

VW_LEMMA = "wva.vw_lemmafreqs"
VW_WORD = "wva.vw_wordfreqs"
VW_LEMMASTRING = "wva.vw_lemmastringfreqs"
VW_WORDSTRING = "wva.vw_wordstringfreqs"

def getFeatures(db, corpusclause, view, fthres=1, dfthres=None):
    docfreqs = {}
    SQL = TRANS+"select id, sum(n), count(distinct articleid) from %s where articleid in (%s) and analysisid=3 group by id having sum(n) >= %i" % (view, corpusclause, fthres)
    for id, n, ndocs in db.doQuery(SQL, select=True):
        if 'raw' in view:
            if not re.match("[A-Za-z]+", id): continue
        docfreqs[id] = ndocs
    SQL = TRANS+"select count(distinct articleid) from %s where articleid in (%s) and analysisid=3 " % (view, corpusclause)
    ndocs = db.getValue(SQL)
    if dfthres:
        dels = set()
        dfthres = int(ndocs * dfthres)
        for id, df in docfreqs.iteritems():
            if df > dfthres:
                dels.add(id)
        for id in dels:
            del docfreqs[id]
    features = {}
    for id in docfreqs:
        features[id] = len(features)+1
    #return docfreqs, features, ndocs
    return features

class DBWordFeatureSet(object):
    def __init__(self, db, view, features):
        self.db = db
        self.view = view
        self.features =features

    def start(self, units):
        self.memo = collections.defaultdict(list)

        for subset in toolkit.splitlist(units, 1000):
            aidsel = toolkit.intselectionSQL("articleid", map(getAID, subset))
            idsel = ""
            #idsel = "AND (%s)" % toolkit.intselectionSQL("id", self.lid2fno.keys())<commit_msg>update refleciting toolkit changes<commit_after>import collections, math, article, codingjob, toolkit, re

PRINTFEATURES = False

def tf(tf, d, df):
    return tf
def logtfidf(tf, d, df):
    s = math.log(tf + .5) * math.log(d / float(df))
    #print "log(tf).log(idf) = log(%i + .5) * log(%i / %i) = %1.3f" % (tf, d, df, s)
    return s
def tfidf(tf, d, df):
    s = tf * math.log(d / float(df))
    #print "tf.log(idf) = %i * log(%i / %i) = %1.3f" % (tf, d, df, s)
    return s

TRANS = "SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED\n "

VW_LEMMA = "wva.vw_lemmafreqs"
VW_WORD = "wva.vw_wordfreqs"
VW_LEMMASTRING = "wva.vw_lemmastringfreqs"
VW_WORDSTRING = "wva.vw_wordstringfreqs"

def getFeatures(db, corpusclause, view, fthres=1, dfthres=None):
    docfreqs = {}
    SQL = TRANS+"select id, sum(n), count(distinct articleid) from %s where articleid in (%s) and analysisid=3 group by id having sum(n) >= %i" % (view, corpusclause, fthres)
    for id, n, ndocs in db.doQuery(SQL, select=True):
        if 'raw' in view:
            if not re.match("[A-Za-z]+", id): continue
        docfreqs[id] = ndocs
    SQL = TRANS+"select count(distinct articleid) from %s where articleid in (%s) and analysisid=3 " % (view, corpusclause)
    ndocs = db.getValue(SQL)
    if dfthres:
        dels = set()
        dfthres = int(ndocs * dfthres)
        for id, df in docfreqs.iteritems():
            if df > dfthres:
                dels.add(id)
        for id in dels:
            del docfreqs[id]
    features = {}
    for id in docfreqs:
        features[id] = len(features)+1
    #return docfreqs, features, ndocs
    return features

class DBWordFeatureSet(object):
    def __init__(self, db, view, features):
        self.db = db
        self.view = view
        self.features =features

    def start(self, units):
        self.memo = collections.defaultdict(list)

        for subset in toolkit.splitlist(units, 1000):
            aidsel = self.db.intSelectionSQL("articleid", map(getAID, subset))
            idsel = ""
            #idsel = "AND (%s)" % toolkit.intselectionSQL("id", self.lid2fno.keys())
{'fraction_non_alphanumeric': 0.13986180605194187, 'fraction_numerical': 0.007147962830593281, 'mean_word_length': 2.7582811101163833, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3470834', 'n_tokens_mistral': 1585, 'n_tokens_neox': 1555, 'n_words': 410}