<commit_before>import os
import sys
from collections import defaultdict
from . import helpers as util
import requests
import json

class Empath:
    def __init__(self, backend_url="http://localhost:8000"):
        self.cats = defaultdict(list)
        #self.invcats = defaultdict(list)
        self.staging = {}
        self.backend_url = backend_url
        self.base_dir = os.path.dirname(util.__file__)
        self.load(self.base_dir+"/data/categories.tsv")

    def load(self,file):
        with open(file,"r") as f:
            for line in f:
                cols = line.strip().split("\t")
                name = cols[0]
                terms = cols[1:]
                for t in terms:
                    self.cats[name].append(t)
                    #self.invcats[t].append(name)

    def analyze(self,doc,cats=None,tokenizer="default",normalize=True):
        if tokenizer == "default":
            tokenizer = util.default_tokenizer
        elif tokenizer == "bigrams":
            tokenizer = util.bigram_tokenizer
        if not hasattr(tokenizer,"__call__"):
            raise Exception("invalid tokenizer")
        if not cats:
            cats = self.cats.keys()
        invcats = defaultdict(list)
        for k in cats:
           for t in self.cats[k]: invcats[t].append(k)
        count = {}
        tokens = 0.0
        for cat in cats: count[cat] = 0.0
        for tk in tokenizer(doc):
            tokens += 1.0
            for cat in invcats[tk]:
                count[cat]+=1.0
        if normalize:
            for cat in count.keys():
                count[cat] = count[cat] / tokens
        return count

    def create_category(self,name,seeds,size=100):
        resp = requests.post(self.backend_url + "/create_category", json={"terms":seeds,"size":size})
        print(resp.text)
        results = json.loads(resp.text)
        self.staging[name] = results
        print("Save this category by calling .save_category('{}')".format(name))
        print(name, results)

    def save_category(self,name):
        if not name in staging:
            raise Exception("not category {} in staging".format(name))
        else: self.cats[name] = self.staging[name]
        del self.staging[name]
<commit_msg>files<commit_after>import os
import sys
from collections import defaultdict
from . import helpers as util
import requests
import json

class Empath:
    def __init__(self, backend_url="http://localhost:8000"):
        self.cats = defaultdict(list)
        #self.invcats = defaultdict(list)
        self.staging = {}
        self.backend_url = backend_url
        self.base_dir = os.path.dirname(util.__file__)
        self.load(self.base_dir+"/data/categories.tsv")
        for f in os.listdir(self.base_dir+"/data/user/"):
            print("loading ",f)
            self.load(self.base_dir+"/data/user/"+f)

    def load(self,file):
        with open(file,"r") as f:
            for line in f:
                cols = line.strip().split("\t")
                name = cols[0]
                terms = cols[1:]
                for t in terms:
                    self.cats[name].append(t)
                    #self.invcats[t].append(name)

    def analyze(self,doc,cats=None,tokenizer="default",normalize=True):
        if tokenizer == "default":
            tokenizer = util.default_tokenizer
        elif tokenizer == "bigrams":
            tokenizer = util.bigram_tokenizer
        if not hasattr(tokenizer,"__call__"):
            raise Exception("invalid tokenizer")
        if not cats:
            cats = self.cats.keys()
        invcats = defaultdict(list)
        for k in cats:
           for t in self.cats[k]: invcats[t].append(k)
        count = {}
        tokens = 0.0
        for cat in cats: count[cat] = 0.0
        for tk in tokenizer(doc):
            tokens += 1.0
            for cat in invcats[tk]:
                count[cat]+=1.0
        if normalize:
            for cat in count.keys():
                count[cat] = count[cat] / tokens
        return count

    def create_category(self,name,seeds,size=100):
        resp = requests.post(self.backend_url + "/create_category", json={"terms":seeds,"size":size})
        print(resp.text)
        results = json.loads(resp.text)
        self.cats[name] = results
        with open(self.base_dir+"/data/user/"+name) as f:
            f.write("\t".join([name]+results))

{'fraction_non_alphanumeric': 0.12136478131440348, 'fraction_numerical': 0.007785665216395695, 'mean_word_length': 1.9836065573770492, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1493160', 'n_tokens_mistral': 1344, 'n_tokens_neox': 1276, 'n_words': 297}