<commit_before>/**
 * While handwritten, this class is designed to mirror the potential results of TypeScript compilation of
 * model TS source files.
 */
(function(){
  var Model = /** @class */ (function() {
    function Model() { // implements Model
    }

    // A direct import/copy from english-1000.json.
    Model.wordlist = ["the", "of", "and", "to", "a", "in", "that", "is", "was", "he", "for", "it", "with", "as", "his", "on", "be", "at", "by", "i", "this", "had", "not", "are", "but", "from", "or", "have", "an", "they", "which", "one", "you", "were", "her", "all", "she", "there", "would", "their", "we", "him", "been", "has", "when", "who", "will", "more", "if", "no", "out", "so", "said", "what", "up", "its", "about", "into", "than", "them", "can", "only", "other", "new", "some", "could", "time", "these", "two", "may", "then", "do", "first", "any", "my", "now", "such", "like", "our", "over", "man", "me", "even", "most", "made", "after", "also", "did", "many", "before", "must", "af", "through", "back", "years", "much", "where", "your", "way", "well", "down", "should", "because", "each", "just", "those", "people", "too", "how", "little", "state", "good", "very", "make", "world", "still", "own", "see", "men", "work", "long", "here", "get", "between", "both", "life", "being", "under", "never", "day", "same", "another", "know", "while", "last", "us", "might", "great", "old", "year", "off", "come", "since", "against", "go", "came", "right", "used", "three", "take", "himself", "states", "few", "use", "house", "during", "without", "again", "place", "american", "around", "however", "home", "small", "found", "thought", "went", "say", "part", "once", "general", "high", "upon", "school", "every", "does", "got", "united", "left", "number", "course", "war", "until", "always", "away", "something", "fact", "water", "though", "public", "put", "less", "think", "almost", "hand", "enough", "took", "far", "head", "yet", "government", "system", "set", "better", "told", "nothing", "night", "end", "why", "called", "eyes", "find", "look", "going", "asked", "later", "point", "knew", "next", "program", "city", "business", "group", "give", "toward", "young", "let", "room", "days", "president", "side", "social", "given", "present", "several", "order", "national", "possible", "rather", "second", "face", "per", "among", "form", "often", "important", "things", "looked", "early", "white", "case", "john", "large", "four", "need", "big", "within", "become", "felt", "along", "children", "saw", "best", "church", "ever", "least", "power", "development", "thing", "seemed", "light", "family", "interest", "want", "mind", "members", "area", "country", "others", "although", "turned", "done", "open", "god", "service", "problem", "kind", "certain", "different", "thus", "began", "door", "sense", "help", "means", "whole", "matter", "perhaps", "itself", "york", "times", "human", "law", "line", "above", "name", "example", "action", "company", "hands", "local", "show", "whether", "history", "five", "gave", "either", "today", "act", "feet", "across", "quite", "taken", "past", "anything", "having", "seen", "death", "experience", "body", "really", "half", "week", "car", "field", "words", "word", "already", "themselves", "information", "tell", "shall", "together", "college", "money", "period", "keep", "held", "sure", "probably", "free", "seems", "behind", "real", "cannot", "political", "question", "air", "making", "office", "brought", "miss", "whose", "special", "problems", "major", "heard", "became", "moment", "study", "ago", "federal", "known", "available", "street", "result", "economic", "boy", "reason", "position", "south", "change", "board", "individual", "job", "society", "am", "areas", "west", "close", "turn", "love", "community", "true", "force", "full", "court", "seem", "cost", "wife", "age", "future", "wanted", "voice", "department", "center", "woman", "common", "control", "necessary", "policy", "front", "following", "sometimes", "girl", "six", "clear", "further", "land", "music", "feel", "mother", "able", "party", "provide", "university", "education", "level", "run", "students", "effect", "child", "stood", "town", "short", "military", "total", "morning", "outside", "rate", "figure", "art", "class", "century", "usually", "north", "washington", "leave", "therefore", "plan", "top", "evidence", "sound", "million", "black", "hard", "strong", "various", "tax", "believe", "type", "value", "says", "play", "surface", "mean", "soon", "modern", "near", "peace", "lines", "table", "book", "road", "red", "situation", "personal", "minutes", "process", "nor", "idea", "alone", "women", "english", "schools", "gone", "increase", "living", "america", "started", "longer", "cut", "finally", "private", "nature", "secretary", "third", "section", "months", "greater", "call", "fire", "needed", "expected", "view", "values", "kept", "ground", "everything", "pressure", "dark", "basis", "space", "father", "east", "spirit", "required", "union", "complete", "wrote", "except", "moved", "support", "return", "conditions", "recent", "particular", "attention", "late", "hope", "live", "else", "costs", "brown", "beyond", "stage", "taking", "material", "nations", "forces", "report", "inside", "dead", "read", "coming", "person", "hours", "heart", "instead", "low", "miles", "lost", "looking", "data", "added", "makes", "single", "followed", "amount", "pay", "feeling", "basic", "including", "simply", "move", "cold", "hundred", "research", "industry", "tried", "developed", "reached", "hold", "committee", "defense", "equipment", "island", "actually", "shown", "son", "central", "religious", "river", "getting", "beginning", "ten", "sort", "received", "rest", "terms", "doing", "trying", "indeed", "care", "friends", "medical", "picture", "especially", "administration", "subject", "fine", "difficult", "building", "higher", "simple", "wall", "walked", "meeting", "bring", "floor", "foreign", "passed", "similar", "paper", "range", "natural", "final", "property", "training", "police", "international", "cent", "county", "market", "growth", "england", "talk", "written", "start", "story", "suddenly", "hear", "issue", "hall", "answer", "needs", "congress", "working", "likely", "considered", "countries", "earth", "sat", "entire", "meet", "purpose", "labor", "happened", "results", "difference", "cases", "stand", "hair", "william", "production", "stock", "involved", "fall", "food", "whom", "earlier", "increased", "particularly", "thinking", "club", "letter", "knowledge", "below", "effort", "hour", "using", "paid", "sent", "christian", "yes", "points", "boys", "industrial", "bill", "deal", "ready", "ideas", "certainly", "square", "blue", "trade", "moral", "bad", "due", "methods", "girls", "method", "addition", "neither", "showed", "throughout", "directly", "nearly", "statement", "decided", "reading", "weeks", "according", "questions", "anyone", "try", "color", "kennedy", "programs", "nation", "services", "lay", "french", "remember", "physical", "size", "member", "understand", "western", "strength", "record", "comes", "southern", "normal", "population", "appeared", "merely", "concerned", "district", "volume", "temperature", "direction", "trial", "trouble", "summer", "aid", "ran", "friend", "sales", "literature", "list", "maybe", "continued", "evening", "association", "generally", "influence", "led", "provided", "met", "army", "science", "changes", "husband", "step", "chance", "student", "opened", "former", "average", "month", "series", "hot", "works", "cause", "lead", "direct", "systems", "planning", "stopped", "myself", "theory", "piece", "wrong", "effective", "george", "soviet", "freedom", "worked", "ask", "movement", "organization", "clearly", "ways", "treatment", "fear", "beautiful", "meaning", "consider", "bed", "spring", "somewhat", "press", "lot", "forms", "efforts", "note", "truth", "placed", "hotel", "carried", "numbers", "respect", "plant", "herself", "apparently", "groups", "degree", "wide", "easy", "manner", "reaction", "farm", "lower", "recently", "approach", "immediately", "game", "larger", "running", "daily", "charge", "couple", "eye", "oh", "performance", "feed", "de", "persons", "c", "understanding", "arms", "blood", "opportunity", "march", "progress", "additional", "described", "technical", "stop", "fiscal", "radio", "religion", "test", "reported", "main", "based", "steps", "image", "chief", "served", "window", "decision", "determined", "responsibility", "character", "middle", "british", "aj", "europe", "gun", "writing", "horse", "appear", "learned", "account", "serious", "ones", "types", "activity", "green", "length", "letters", "nuclear", "specific", "slowly", "forward", "returned", "lived", "activities", "corner", "obtained", "audience", "justice", "doubt", "latter", "hit", "plane", "gives", "moving", "obviously", "straight", "design", "quality", "saying", "function", "choice", "staff", "include", "pattern", "figures", "born", "parts", "stay", "seven", "poor", "plans", "operation", "shot", "whatever", "cars", "sun", "faith", "pool", "extent", "speak", "heavy", "completely", "lack", "waiting", "standard", "hospital", "ball", "wish", "corps", "visit", "language", "ahead", "deep", "income", "principle", "democratic", "firm", "importance", "growing", "analysis", "designed", "effects", "price", "none", "distance", "indicated", "established", "products", "expect", "division", "continue", "leaders", "existence", "determine", "serve", "stress", "attitude", "pretty", "elements", "easily", "cities", "negro", "remained", "factors", "hardly", "applied", "limited", "closed", "agreement", "scene", "write", "afternoon", "b", "suggested", "health", "professional", "attack", "reach", "drive", "season", "interested", "station", "rhode", "married", "despite", "covered", "role", "becomes", "eight", "current", "played", "spent", "reasons", "council", "unit", "built", "commission", "date", "mouth", "exactly", "original", "studies", "race", "machine"];

    return Model;
  }());

  // As a wordlist model, we can rely on our "model library" implementation, models.WordListModel.
  LMLayerWorker.loadModel(new models.WordListModel(Model.wordlist));
})();<commit_msg>Update english-1000 in integration test.<commit_after>/**
 * While handwritten, this class is designed to mirror the potential results of TypeScript compilation of
 * model TS source files.
 */
(function(){
  var Model = /** @class */ (function() {
    function Model() { // implements Model
    }

    // A direct import/copy from english-1000.json.
    Model.wordlist = [["the",1],["of",1],["and",1],["to",1],["a",1],["in",1],["that",1],["is",1],["was",1],["he",1],["for",1],["it",1],["with",1],["as",1],["his",1],["on",1],["be",1],["at",1],["by",1],["i",1],["this",1],["had",1],["not",1],["are",1],["but",1],["from",1],["or",1],["have",1],["an",1],["they",1],["which",1],["one",1],["you",1],["were",1],["her",1],["all",1],["she",1],["there",1],["would",1],["their",1],["we",1],["him",1],["been",1],["has",1],["when",1],["who",1],["will",1],["more",1],["if",1],["no",1],["out",1],["so",1],["said",1],["what",1],["up",1],["its",1],["about",1],["into",1],["than",1],["them",1],["can",1],["only",1],["other",1],["new",1],["some",1],["could",1],["time",1],["these",1],["two",1],["may",1],["then",1],["do",1],["first",1],["any",1],["my",1],["now",1],["such",1],["like",1],["our",1],["over",1],["man",1],["me",1],["even",1],["most",1],["made",1],["after",1],["also",1],["did",1],["many",1],["before",1],["must",1],["af",1],["through",1],["back",1],["years",1],["much",1],["where",1],["your",1],["way",1],["well",1],["down",1],["should",1],["because",1],["each",1],["just",1],["those",1],["people",1],["too",1],["how",1],["little",1],["state",1],["good",1],["very",1],["make",1],["world",1],["still",1],["own",1],["see",1],["men",1],["work",1],["long",1],["here",1],["get",1],["between",1],["both",1],["life",1],["being",1],["under",1],["never",1],["day",1],["same",1],["another",1],["know",1],["while",1],["last",1],["us",1],["might",1],["great",1],["old",1],["year",1],["off",1],["come",1],["since",1],["against",1],["go",1],["came",1],["right",1],["used",1],["three",1],["take",1],["himself",1],["states",1],["few",1],["use",1],["house",1],["during",1],["without",1],["again",1],["place",1],["american",1],["around",1],["however",1],["home",1],["small",1],["found",1],["thought",1],["went",1],["say",1],["part",1],["once",1],["general",1],["high",1],["upon",1],["school",1],["every",1],["does",1],["got",1],["united",1],["left",1],["number",1],["course",1],["war",1],["until",1],["always",1],["away",1],["something",1],["fact",1],["water",1],["though",1],["public",1],["put",1],["less",1],["think",1],["almost",1],["hand",1],["enough",1],["took",1],["far",1],["head",1],["yet",1],["government",1],["system",1],["set",1],["better",1],["told",1],["nothing",1],["night",1],["end",1],["why",1],["called",1],["eyes",1],["find",1],["look",1],["going",1],["asked",1],["later",1],["point",1],["knew",1],["next",1],["program",1],["city",1],["business",1],["group",1],["give",1],["toward",1],["young",1],["let",1],["room",1],["days",1],["president",1],["side",1],["social",1],["given",1],["present",1],["several",1],["order",1],["national",1],["possible",1],["rather",1],["second",1],["face",1],["per",1],["among",1],["form",1],["often",1],["important",1],["things",1],["looked",1],["early",1],["white",1],["case",1],["john",1],["large",1],["four",1],["need",1],["big",1],["within",1],["become",1],["felt",1],["along",1],["children",1],["saw",1],["best",1],["church",1],["ever",1],["least",1],["power",1],["development",1],["thing",1],["seemed",1],["light",1],["family",1],["interest",1],["want",1],["mind",1],["members",1],["area",1],["country",1],["others",1],["although",1],["turned",1],["done",1],["open",1],["god",1],["service",1],["problem",1],["kind",1],["certain",1],["different",1],["thus",1],["began",1],["door",1],["sense",1],["help",1],["means",1],["whole",1],["matter",1],["perhaps",1],["itself",1],["york",1],["times",1],["human",1],["law",1],["line",1],["above",1],["name",1],["example",1],["action",1],["company",1],["hands",1],["local",1],["show",1],["whether",1],["history",1],["five",1],["gave",1],["either",1],["today",1],["act",1],["feet",1],["across",1],["quite",1],["taken",1],["past",1],["anything",1],["having",1],["seen",1],["death",1],["experience",1],["body",1],["really",1],["half",1],["week",1],["car",1],["field",1],["words",1],["word",1],["already",1],["themselves",1],["information",1],["tell",1],["shall",1],["together",1],["college",1],["money",1],["period",1],["keep",1],["held",1],["sure",1],["probably",1],["free",1],["seems",1],["behind",1],["real",1],["cannot",1],["political",1],["question",1],["air",1],["making",1],["office",1],["brought",1],["miss",1],["whose",1],["special",1],["problems",1],["major",1],["heard",1],["became",1],["moment",1],["study",1],["ago",1],["federal",1],["known",1],["available",1],["street",1],["result",1],["economic",1],["boy",1],["reason",1],["position",1],["south",1],["change",1],["board",1],["individual",1],["job",1],["society",1],["am",1],["areas",1],["west",1],["close",1],["turn",1],["love",1],["community",1],["true",1],["force",1],["full",1],["court",1],["seem",1],["cost",1],["wife",1],["age",1],["future",1],["wanted",1],["voice",1],["department",1],["center",1],["woman",1],["common",1],["control",1],["necessary",1],["policy",1],["front",1],["following",1],["sometimes",1],["girl",1],["six",1],["clear",1],["further",1],["land",1],["music",1],["feel",1],["mother",1],["able",1],["party",1],["provide",1],["university",1],["education",1],["level",1],["run",1],["students",1],["effect",1],["child",1],["stood",1],["town",1],["short",1],["military",1],["total",1],["morning",1],["outside",1],["rate",1],["figure",1],["art",1],["class",1],["century",1],["usually",1],["north",1],["washington",1],["leave",1],["therefore",1],["plan",1],["top",1],["evidence",1],["sound",1],["million",1],["black",1],["hard",1],["strong",1],["various",1],["tax",1],["believe",1],["type",1],["value",1],["says",1],["play",1],["surface",1],["mean",1],["soon",1],["modern",1],["near",1],["peace",1],["lines",1],["table",1],["book",1],["road",1],["red",1],["situation",1],["personal",1],["minutes",1],["process",1],["nor",1],["idea",1],["alone",1],["women",1],["english",1],["schools",1],["gone",1],["increase",1],["living",1],["america",1],["started",1],["longer",1],["cut",1],["finally",1],["private",1],["nature",1],["secretary",1],["third",1],["section",1],["months",1],["greater",1],["call",1],["fire",1],["needed",1],["expected",1],["view",1],["values",1],["kept",1],["ground",1],["everything",1],["pressure",1],["dark",1],["basis",1],["space",1],["father",1],["east",1],["spirit",1],["required",1],["union",1],["complete",1],["wrote",1],["except",1],["moved",1],["support",1],["return",1],["conditions",1],["recent",1],["particular",1],["attention",1],["late",1],["hope",1],["live",1],["else",1],["costs",1],["brown",1],["beyond",1],["stage",1],["taking",1],["material",1],["nations",1],["forces",1],["report",1],["inside",1],["dead",1],["read",1],["coming",1],["person",1],["hours",1],["heart",1],["instead",1],["low",1],["miles",1],["lost",1],["looking",1],["data",1],["added",1],["makes",1],["single",1],["followed",1],["amount",1],["pay",1],["feeling",1],["basic",1],["including",1],["simply",1],["move",1],["cold",1],["hundred",1],["research",1],["industry",1],["tried",1],["developed",1],["reached",1],["hold",1],["committee",1],["defense",1],["equipment",1],["island",1],["actually",1],["shown",1],["son",1],["central",1],["religious",1],["river",1],["getting",1],["beginning",1],["ten",1],["sort",1],["received",1],["rest",1],["terms",1],["doing",1],["trying",1],["indeed",1],["care",1],["friends",1],["medical",1],["picture",1],["especially",1],["administration",1],["subject",1],["fine",1],["difficult",1],["building",1],["higher",1],["simple",1],["wall",1],["walked",1],["meeting",1],["bring",1],["floor",1],["foreign",1],["passed",1],["similar",1],["paper",1],["range",1],["natural",1],["final",1],["property",1],["training",1],["police",1],["international",1],["cent",1],["county",1],["market",1],["growth",1],["england",1],["talk",1],["written",1],["start",1],["story",1],["suddenly",1],["hear",1],["issue",1],["hall",1],["answer",1],["needs",1],["congress",1],["working",1],["likely",1],["considered",1],["countries",1],["earth",1],["sat",1],["entire",1],["meet",1],["purpose",1],["labor",1],["happened",1],["results",1],["difference",1],["cases",1],["stand",1],["hair",1],["william",1],["production",1],["stock",1],["involved",1],["fall",1],["food",1],["whom",1],["earlier",1],["increased",1],["particularly",1],["thinking",1],["club",1],["letter",1],["knowledge",1],["below",1],["effort",1],["hour",1],["using",1],["paid",1],["sent",1],["christian",1],["yes",1],["points",1],["boys",1],["industrial",1],["bill",1],["deal",1],["ready",1],["ideas",1],["certainly",1],["square",1],["blue",1],["trade",1],["moral",1],["bad",1],["due",1],["methods",1],["girls",1],["method",1],["addition",1],["neither",1],["showed",1],["throughout",1],["directly",1],["nearly",1],["statement",1],["decided",1],["reading",1],["weeks",1],["according",1],["questions",1],["anyone",1],["try",1],["color",1],["kennedy",1],["programs",1],["nation",1],["services",1],["lay",1],["french",1],["remember",1],["physical",1],["size",1],["member",1],["understand",1],["western",1],["strength",1],["record",1],["comes",1],["southern",1],["normal",1],["population",1],["appeared",1],["merely",1],["concerned",1],["district",1],["volume",1],["temperature",1],["direction",1],["trial",1],["trouble",1],["summer",1],["aid",1],["ran",1],["friend",1],["sales",1],["literature",1],["list",1],["maybe",1],["continued",1],["evening",1],["association",1],["generally",1],["influence",1],["led",1],["provided",1],["met",1],["army",1],["science",1],["changes",1],["husband",1],["step",1],["chance",1],["student",1],["opened",1],["former",1],["average",1],["month",1],["series",1],["hot",1],["works",1],["cause",1],["lead",1],["direct",1],["systems",1],["planning",1],["stopped",1],["myself",1],["theory",1],["piece",1],["wrong",1],["effective",1],["george",1],["soviet",1],["freedom",1],["worked",1],["ask",1],["movement",1],["organization",1],["clearly",1],["ways",1],["treatment",1],["fear",1],["beautiful",1],["meaning",1],["consider",1],["bed",1],["spring",1],["somewhat",1],["press",1],["lot",1],["forms",1],["efforts",1],["note",1],["truth",1],["placed",1],["hotel",1],["carried",1],["numbers",1],["respect",1],["plant",1],["herself",1],["apparently",1],["groups",1],["degree",1],["wide",1],["easy",1],["manner",1],["reaction",1],["farm",1],["lower",1],["recently",1],["approach",1],["immediately",1],["game",1],["larger",1],["running",1],["daily",1],["charge",1],["couple",1],["eye",1],["oh",1],["performance",1],["feed",1],["de",1],["persons",1],["c",1],["understanding",1],["arms",1],["blood",1],["opportunity",1],["march",1],["progress",1],["additional",1],["described",1],["technical",1],["stop",1],["fiscal",1],["radio",1],["religion",1],["test",1],["reported",1],["main",1],["based",1],["steps",1],["image",1],["chief",1],["served",1],["window",1],["decision",1],["determined",1],["responsibility",1],["character",1],["middle",1],["british",1],["aj",1],["europe",1],["gun",1],["writing",1],["horse",1],["appear",1],["learned",1],["account",1],["serious",1],["ones",1],["types",1],["activity",1],["green",1],["length",1],["letters",1],["nuclear",1],["specific",1],["slowly",1],["forward",1],["returned",1],["lived",1],["activities",1],["corner",1],["obtained",1],["audience",1],["justice",1],["doubt",1],["latter",1],["hit",1],["plane",1],["gives",1],["moving",1],["obviously",1],["straight",1],["design",1],["quality",1],["saying",1],["function",1],["choice",1],["staff",1],["include",1],["pattern",1],["figures",1],["born",1],["parts",1],["stay",1],["seven",1],["poor",1],["plans",1],["operation",1],["shot",1],["whatever",1],["cars",1],["sun",1],["faith",1],["pool",1],["extent",1],["speak",1],["heavy",1],["completely",1],["lack",1],["waiting",1],["standard",1],["hospital",1],["ball",1],["wish",1],["corps",1],["visit",1],["language",1],["ahead",1],["deep",1],["income",1],["principle",1],["democratic",1],["firm",1],["importance",1],["growing",1],["analysis",1],["designed",1],["effects",1],["price",1],["none",1],["distance",1],["indicated",1],["established",1],["products",1],["expect",1],["division",1],["continue",1],["leaders",1],["existence",1],["determine",1],["serve",1],["stress",1],["attitude",1],["pretty",1],["elements",1],["easily",1],["cities",1],["negro",1],["remained",1],["factors",1],["hardly",1],["applied",1],["limited",1],["closed",1],["agreement",1],["scene",1],["write",1],["afternoon",1],["b",1],["suggested",1],["health",1],["professional",1],["attack",1],["reach",1],["drive",1],["season",1],["interested",1],["station",1],["rhode",1],["married",1],["despite",1],["covered",1],["role",1],["becomes",1],["eight",1],["current",1],["played",1],["spent",1],["reasons",1],["council",1],["unit",1],["built",1],["commission",1],["date",1],["mouth",1],["exactly",1],["original",1],["studies",1],["race",1],["machine",1]];

    return Model;
  }());

  // As a wordlist model, we can rely on our "model library" implementation, models.WordListModel.
  LMLayerWorker.loadModel(new models.WordListModel(Model.wordlist));
})();
{'fraction_non_alphanumeric': 0.39172165566886624, 'fraction_numerical': 0.043362756020224526, 'mean_word_length': 17.6712, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2280207', 'n_tokens_mistral': 9297, 'n_tokens_neox': 8657, 'n_words': 1110}