<commit_before>package com.eharmony.matching.aloha.feature

import scala.collection.mutable.ListBuffer

/**
 * @author msokolowski
 *
 * Implements k-skip-n-grams e.g
 *
 * from http://homepages.inf.ed.ac.uk/ballison/pdf/lrec_skipgrams.pdf
 *
 * "Insurgents killed in ongoing fighting.”
 * Bi-grams = {insurgents killed, killed in, in ongoing, ongoing fighting}.
 * 2-skip-2-grams = {insurgents killed, insurgents in, insurgents ongoing, killed in, killed ongoing, killed fighting, in ongoing, in fighting, ongoing fighting} Tri-grams = {insurgents killed in, killed in ongoing, in ongoing fighting}.
 * 2-skip-3-grams = {insurgents killed in, insurgents killed ongoing, insurgents killed fighting, insurgents in ongoing, insurgents in fighting, insurgents ongoing fighting, killed in ongoing, killed in fighting, killed ongoing fighting, in ongoing fighting}.
 */
trait nGrams {
  def skipGrams(s: String, n: Int, k: Int): Iterable[(String, Double)] = {
    s.split(" ").sliding(n + k).foldLeft(Iterable.empty[(String, Double)])((acc, l) => acc ++ l.combinations(n).map(e => e.mkString("_") -> 1.0).toMap)<commit_msg>MAT-3566: Added link to paper dealing with skip grams and an example I used to create the original function.<commit_after>package com.eharmony.matching.aloha.feature

import scala.collection.mutable.ListBuffer

/**
 * @author msokolowski
 *
 * Implements k-skip-n-grams e.g
 *
 * from http://homepages.inf.ed.ac.uk/ballison/pdf/lrec_skipgrams.pdf (Introduction)
 *
 * "Insurgents killed in ongoing fighting.”
 * Bi-grams = {insurgents killed, killed in, in ongoing, ongoing fighting}.
 * 2-skip-2-grams = {insurgents killed, insurgents in, insurgents ongoing, killed in, killed ongoing, killed fighting, in ongoing, in fighting, ongoing fighting} Tri-grams = {insurgents killed in, killed in ongoing, in ongoing fighting}.
 * 2-skip-3-grams = {insurgents killed in, insurgents killed ongoing, insurgents killed fighting, insurgents in ongoing, insurgents in fighting, insurgents ongoing fighting, killed in ongoing, killed in fighting, killed ongoing fighting, in ongoing fighting}.
 */
trait nGrams {
  def skipGrams(s: String, n: Int, k: Int): Iterable[(String, Double)] = {
    s.split(" ").sliding(n + k).foldLeft(Iterable.empty[(String, Double)])((acc, l) => acc ++ l.combinations(n).map(e => e.mkString("_") -> 1.0).toMap)
{'fraction_non_alphanumeric': 0.12915601023017903, 'fraction_numerical': 0.0068201193520886615, 'mean_word_length': 4.780788177339901, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7150010', 'n_tokens_mistral': 754, 'n_tokens_neox': 710, 'n_words': 228}