 Read in the book "The War of the Worlds", by H. G. Wells.
 Skip to the start of the book, proper.
 Remove extraneous punctuation, but keep at least sentence-ending punctuation characters . ! and ?
 Keep account of what words follow words and how many times it is seen, (treat sentence terminators as words too).
 Keep account of what words follow two words and how many times it is seen, (again treating sentence terminators as words too).
 Assume that a sentence starts with a not to be shown full-stop character then use a weighted random choice of the possible words that may follow a full-stop to add to the sentence.
 Then repeatedly add words to the sentence based on weighted random choices of what words my follow the last two words to extend the sentence.
 Stop after adding a sentence ending punctuation character.
 Tidy and then print the sentence.


Show examples of random sentences generated.

Related task
 Markov_chain_text_generator


import random, sequtils, strutils, tables
from unicode import utf8
 
const StopChars = [".", "?", "!"]
 
proc weightedChoice(choices: CountTable[string]; totalCount: int): string =
  ## Return a string from "choices" key using counts as weights.
  var n = rand(1..totalCount)
  for word, count in choices.pairs:
    dec n, count
    if n <= 0: return word
  assert false, "internal error"
 
proc finalFilter(words: seq[string]): seq[string] =
  ## Eliminate words of length one (except those of a given list)
  ## and words containing only uppercase letters (words from titles).
  for word in words:
    if word in [".", "?", "!", "I", "A", "a"]:
      result.add word
    elif word.len > 1 and any(word, isLowerAscii):
      result.add word
 
 
randomize()
 
var text = readFile("The War of the Worlds.txt")
 
# Extract the actual text from the book.
const
  StartText = "BOOK ONE\r\nTHE COMING OF THE MARTIANS"
  EndText = "End of the Project Gutenberg EBook"
let startIndex = text.find(StartText)
let endIndex = text.find(EndText)
text = text[startIndex..<endIndex]
 
# Clean the text by removing some characters and replacing others.
# As there are some non ASCII characters, we have to apply special rules.
var processedText: string
for uchar in text.utf8():
  if uchar.len == 1:
    # ASCII character.
    let ch = uchar[0]
    case ch
    of '0'..'9', 'a'..'z', 'A'..'Z', ' ': processedText.add ch  # Keep as is.
    of '\n': processedText.add ' '                              # Replace with a space.
    of '.', '?', '!': processedText.add ' ' & ch                # Make sure these characters are isolated.
    else: discard                                               # Remove others.
  else:
    # Some UTF-8 representation of a non ASCII character.
    case uchar
    of "—": processedText.add ' '                               # Replace EM DASH with space.
    of "ç", "æ", "’": processedText.add uchar                   # Keep these characters as they are parts of words.
    of "“", "”", "‘": discard                                   # Removed these ones.
    else: echo "encountered: ", uchar                           # Should not go here.
 
# Extract words and filter them.
let words = processedText.splitWhitespace().finalFilter()
 
# Build count tables.
var followCount, followCount2: Table[string, CountTable[string]]
for i in 1..words.high:
  followCount.mgetOrPut(words[i - 1], initCountTable[string]()).inc(words[i])
for i in 2..words.high:
  followCount2.mgetOrPut(words[i - 2] & ' ' & words[i - 1], initCountTable[string]()).inc words[i]
 
# Build sum tables.
var followSum, followSum2: CountTable[string]
for key in followCount.keys:
  for count in followCount[key].values:
    followSum.inc key, count
for key in followCount2.keys:
  for count in followCount2[key].values:
    followSum2.inc key, count
 
# Build table of starting words and compute the sum.
var
  startingWords: CountTable[string]
  startingSum: int
for stopChar in StopChars:
  for word, count in followCount[stopChar].pairs:
    startingWords.inc word, count
    inc startingSum, count
 
# Build a sentence.
let firstWord = weightedChoice(startingWords, startingSum)
var sentence = @[firstWord]
var lastWord = weightedChoice(followCount[firstWord], followSum[firstWord])
while lastWord notin StopChars:
  sentence.add lastWord
  let key = sentence[^2] & ' ' & lastWord
  lastWord = if key in followCount2:
               weightedChoice(followCount2[key], followSum2[key])
             else:
               weightedChoice(followCount[lastWord], followSum[lastWord])
echo sentence.join(" ") & lastWord