 Read in the book "The War of the Worlds", by H. G. Wells.
 Skip to the start of the book, proper.
 Remove extraneous punctuation, but keep at least sentence-ending punctuation characters . ! and ?
 Keep account of what words follow words and how many times it is seen, (treat sentence terminators as words too).
 Keep account of what words follow two words and how many times it is seen, (again treating sentence terminators as words too).
 Assume that a sentence starts with a not to be shown full-stop character then use a weighted random choice of the possible words that may follow a full-stop to add to the sentence.
 Then repeatedly add words to the sentence based on weighted random choices of what words my follow the last two words to extend the sentence.
 Stop after adding a sentence ending punctuation character.
 Tidy and then print the sentence.


Show examples of random sentences generated.

Related task
 Markov_chain_text_generator


-- demo/rosetta/RandomSentence.exw
include builtins\libcurl.e
constant url =  "http://www.gutenberg.org/files/36/36-0.txt",
         filename = "war_of_the_worlds.txt",
         fsent = "No one would have believed",
         lasts = "End of the Project Gutenberg EBook",
         unicodes = {utf32_to_utf8({#2019}),    -- rsquo
                     utf32_to_utf8({#2014})},   -- hyphen
         asciis = {"'","-"},
         aleph = tagset('Z','A')&tagset('z','a')&tagset('9','0')&",'.?! ",
         follow = new_dict(),   -- {word}      -> {words,counts}
         follow2 = new_dict()   -- {word,word} -> {words,counts}
 
if not file_exists(filename) then
    printf(1,"Downloading %s...\n",{filename})
    CURLcode res = curl_easy_get_file(url,"",filename)
    if res!=CURLE_OK then crash("cannot download") end if
end if
string text = get_text(filename)
text = text[match(fsent,text)..match(lasts,text)-1]
text = substitute_all(text,unicodes,asciis)
text = substitute_all(text,".?!-\n",{" ."," ? "," ! "," "," "})
text = filter(text,"in",aleph)
sequence words = split(text)

procedure account(sequence words)
    string next = words[$]
    words = words[1..$-1]
    for i=length(words) to 1 by -1 do
        integer d = {follow,follow2}[i]
        sequence t = getdd(words,{{},{}},d)
        integer tk = find(next,t[1])
        if tk=0 then
            t[1] = append(t[1],next)
            t[2] = append(t[2],1)
        else
            t[2][tk] += 1
        end if
        setd(words,t,d)
        words = words[2..$]
        if words!={"."} then exit end if -- (may as well quit)
    end for
end procedure

for i=2 to length(words) do
    if find(words[i],{".","?","!"})
    and i<length(words) then
        words[i+1] = lower(words[i+1])
    end if
    account(words[max(1,i-2)..i])
end for

function weighted_random_pick(sequence words, integer dict)
    sequence t = getd(words,dict)
    integer total = sum(t[2]),
            r = rand(total)
    for i=1 to length(t[2]) do
        r -= t[2][i]
        if r<=0 then
            return t[1][i]
        end if
    end for
end function

for i=1 to 5 do
    sequence sentence = {".",weighted_random_pick({"."},follow)}
    while true do
        string next = weighted_random_pick(sentence[-2..-1],follow2)
        sentence = append(sentence,next)
        if find(next,{".","?","!"}) then exit end if
    end while
    sentence[2][1] = upper(sentence[2][1])
    printf(1,"%s\n",{join(sentence[2..$-1])&sentence[$]})
end for
{} = wait_key()
