<commit_before>#!/usr/bin/python

import os, subprocess, codecs

class BackOffTMPipe:

    def __init__(self, pathtotmpipeexecutable, pathtolangmod):
        self.tmpipe = subprocess.Popen([pathtotmpipeexecutable, pathtolangmod], stdin=-1, stdout=-1)
        self.stdin = self.tmpipe.stdin
        self.stdin_byte_writer = codecs.getwriter('utf-8')(self.tmpipe.stdin)
        self.stdout = self.tmpipe.stdout

    def in_vocabulary(self, token):
        self.stdin_byte_writer.write("v " + token + '\n')
        return int(self.stdout.readline())

    def vocabulary_with_prefix(self, prefix):
        self.stdin_byte_writer.write("p " + prefix + '\n')
        return [ b.decode('utf-8') for b in self.stdout.readline().split() ]

    def unigram_probability(self, token):
        if self.in_vocabulary(token):
            self.stdin_byte_writer.write("u " + token + '\n')
            return float(self.stdout.readline())
        else:
            self.stdin_byte_writer.write("U\n") # Is this an unk model?
            try:
                return float(self.stdout.readline())
            except ValueError, exp:
                return None

    def trigram_probability(self, tokens):
        self.stdin_byte_writer.write("U\n") # Is this an unk model?
        if self.stdout.readline() == "None\n":
            if reduce(lambda x, y: x and y, [self.in_vocabulary(t) for t in tokens]):
                self.stdin_byte_writer.write("t " + " ".join(tokens) + '\n')
                return float(self.stdout.readline())
            else:
                return None
        else:
            self.stdin_byte_writer.write("t " + " ".join(tokens) + '\n')
            return float(self.stdout.readline())


    def shutdown(self):
        if hasattr(self, "tmpipe"):
            if hasattr(self.tmpipe, "pid") and self.tmpipe.pid:
                os.kill(self.tmpipe.pid, 15)
            self.tmpipe.wait()

    def __del__(self):
        self.shutdown()<commit_msg>Supplied missing import of os.<commit_after>#!/usr/bin/python

import os, subprocess, codecs

class BackOffTMPipe:

    def __init__(self, pathtotmpipeexecutable, pathtolangmod):
        self.tmpipe = subprocess.Popen([pathtotmpipeexecutable, pathtolangmod], stdin=-1, stdout=-1)
        self.stdin = self.tmpipe.stdin
        self.stdin_byte_writer = codecs.getwriter('utf-8')(self.tmpipe.stdin)
        self.stdout = self.tmpipe.stdout

    def in_vocabulary(self, token):
        self.stdin_byte_writer.write("v " + token + '\n')
        return int(self.stdout.readline())

    def vocabulary_with_prefix(self, prefix):
        self.stdin_byte_writer.write("p " + prefix + '\n')
        return [ b.decode('utf-8') for b in self.stdout.readline().split() ]

    def unigram_probability(self, token):
        if self.in_vocabulary(token):
            self.stdin_byte_writer.write("u " + token + '\n')
            return float(self.stdout.readline())
        else:
            self.stdin_byte_writer.write("U\n") # Is this an unk model?
            try:
                return float(self.stdout.readline())
            except ValueError, exp:
                return None

    def trigram_probability(self, tokens):
        self.stdin_byte_writer.write("U\n") # Is this an unk model?
        if self.stdout.readline() == "None\n":
            if reduce(lambda x, y: x and y, [self.in_vocabulary(t) for t in tokens]):
                self.stdin_byte_writer.write("t " + " ".join(tokens) + '\n')
                return float(self.stdout.readline())
            else:
                return None
        else:
            self.stdin_byte_writer.write("t " + " ".join(tokens) + '\n')
            return float(self.stdout.readline())


    def shutdown(self):
        if hasattr(self, "tmpipe"):
            if hasattr(self.tmpipe, "pid") and self.tmpipe.pid:
                import os
                os.kill(self.tmpipe.pid, 15)
            self.tmpipe.wait()

    def __del__(self):
        self.shutdown()
{'fraction_non_alphanumeric': 0.13096139288417866, 'fraction_numerical': 0.003028009084027252, 'mean_word_length': 2.228013029315961, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3831612', 'n_tokens_mistral': 1295, 'n_tokens_neox': 1257, 'n_words': 247}