<jupyter_start><jupyter_text>Fitting large maximum entropy models with simulation - Berger machine translation example Copyright (c) 2003-2017 <NAME> Here is an example with simulation on a tiny problemIt demonstrates how to use simulation conceptually and the API of `maxentropy`.As in `example_berger.py`, this is the machine translation example-- English to French -- from the paper 'A maximum entropy approachto natural language processing' by Berger etal., 1996.Consider the translation of the English word 'in' into French.  Wenotice in a corpus of parallel texts the following facts:    (1)    p(dans) + p(en) + p(à) + p(au cours de) + p(pendant) = 1    (2)    p(dans) + p(en) = 3/10    (3)    p(dans) + p(à)  = 1/2This code finds the probability distribution with maximal entropysubject to these constraints **without enumerating the sample space**,using importance sampling instead.This is way overkill for this tiny problem (which can be solved analytically),but it demonstrates how to use simulation in principle to solve larger problemson a continuous or larger discrete sample space.<jupyter_code>from __future__ import print_function

import sys

import maxentropy
from maxentropy.maxentutils import dictsampler
import numpy as np
samplespace = ['dans', 'en', 'à', 'au cours de', 'pendant']
@np.vectorize
def f0(x):
    return x in samplespace

@np.vectorize
def f1(x):
    return x == 'dans' or x == 'en'

@np.vectorize
def f2(x):
    return x == 'dans' or x == 'à'

f = [f0, f1, f2]
f0('dans')
# Define a uniform instrumental distribution for sampling
samplefreq = {e: 1 for e in samplespace}
auxiliary_sampler = dictsampler(samplefreq, size=10**5, return_probs='logprob')
next(auxiliary_sampler)
model = maxentropy.BigModel(auxiliary_sampler)
# Default: model.algorithm = 'CG'
# Can choose from ['CG', 'BFGS', 'LBFGSB', 'Powell', 'Nelder-Mead']
# Now set the desired feature expectations
K = [1.0, 0.3, 0.5]
from maxentropy.maxentutils import importance_sampler, create_vectorized_feature_function
features = create_vectorized_feature_function(f, sparse=False)
xs, logprobs = next(auxiliary_sampler)
xs
features(xs)
model.samplegen = importance_sampler(features, auxiliary_sampler)
model.reset(len(f))
next(model.samplegen)
model.resample()
model.verbose = True

# Fit the model
# model.avegtol = 1e-5
model.fit(f, K)
# Output the true distribution
print("Fitted model parameters are:")
model.params
smallmodel = maxentropy.Model(samplespace)
smallmodel.setparams(model.params)
smallmodel.params
smallmodel.setfeatures(f)
smallmodel.F.todense()
F = smallmodel.F.todense().T
F
smallmodel.params
F.dot(smallmodel.params)
smallmodel.F.T.dot(smallmodel.params)
print("\nFitted distribution is:")
smallmodel.showdist()
# Now show how well the constraints are satisfied:
print()
print("Desired constraints:")
print("\tp['dans'] + p['en'] = 0.3")
print("\tp['dans'] + p['à']  = 0.5")
print()
print("Actual expectations under the fitted model:")
print("\tp['dans'] + p['en'] =", p[0] + p[1])
print("\tp['dans'] + p['à']  = " + str(p[0]+p[2]))

print("\nEstimated error in constraint satisfaction (should be close to 0):\n"
        + str(abs(model.expectations() - K)))
print("\nTrue error in constraint satisfaction (should be close to 0):\n" +
        str(abs(smallmodel.expectations() - K)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12440047961630696, 'fraction_numerical': 0.01498800959232614, 'mean_word_length': 5.332068311195446, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '601737', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1054, 'n_words': 350, 'path': 'notebooks/Fitting large maximum entropy models - simulated ... Berger machine translation example.ipynb', 'repo_name': 'bluerobe25/maxentropy', 'size': '3342'}