<commit_before>#!/usr/bin/python

import sys
import math


def read_bo_model(infname):

    curr_n = -1
    ngrams = dict()
    ngramfile = open(infname)

    for line in ngramfile:
        line = line.strip()
        if line.startswith(r'\data'): continue
        if line.startswith(r'\interpolated'):
            print >>sys.stderr, "Model in interpolated format."
            exit()
        if line.startswith(r'\end'): continue
        if line.startswith('ngram'): continue
        if not line: continue
        if line.startswith('\\'):
            vals = line.split('-')
            curr_n  = int(vals[0][1:len(vals[0])])
            continue

        vals = line.partition(' ')
        logp  = float(vals[0])
        backoff = None
        ngram = vals[2]
        vals = ngram.split()
        if len(vals) > curr_n:
            backoff = vals[len(vals)-1]
            ngram = ngram.replace(backoff, "")
            backoff = float(backoff)
        ngram = ngram.strip()
        ngrams[ngram] = (logp, backoff)
<commit_msg>Better handling for ARPA files with different separators.<commit_after>#!/usr/bin/python

import sys
import math


def read_bo_model(infname):

    curr_n = -1
    ngrams = dict()
    ngramfile = open(infname)
    separator = ' '

    for line in ngramfile:
        line = line.strip()
        if line.startswith(r'\data'): continue
        if line.startswith(r'\interpolated'):
            print >>sys.stderr, "Model in interpolated format."
            exit()
        if line.startswith(r'\end'): continue
        if line.startswith('ngram'): continue
        if not line: continue
        if line.startswith('\\'):
            vals = line.split('-')
            curr_n  = int(vals[0][1:len(vals[0])])
            continue

        if '\t' in line: separator = '\t'
        vals = line.partition(separator)
        logp  = float(vals[0])
        backoff = None
        ngram = vals[2]
        vals = ngram.split()
        if len(vals) > curr_n:
            backoff = vals[len(vals)-1]
            ngram = ngram.replace(backoff, "")
            backoff = float(backoff)
        ngram = ngram.strip()
        ngrams[ngram] = (logp, backoff)

{'fraction_non_alphanumeric': 0.13087557603686636, 'fraction_numerical': 0.0064516129032258064, 'mean_word_length': 1.9219380888290714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1139773', 'n_tokens_mistral': 713, 'n_tokens_neox': 688, 'n_words': 158}