<commit_before>package org.deeplearning4j.word2vec.util;

import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

public class Windows {
	/**
	 * Constructs a list of window of size windowSize.
	 * Note that padding for each window is created as well.
	 * @param words the words to tokenize and construct windows from
	 * @return the list of windows for the tokenized string
	 */
	public static List<Window> windows(String words) {
		StringTokenizer tokenizer = new StringTokenizer(words);
		List<String> list = new ArrayList<String>();
		while(tokenizer.hasMoreTokens())
			list.add(tokenizer.nextToken());
		return windows(list,5);
	}

	/**
	 * Creates a sliding window from text
	 * @param windowSize the window size to use<commit_msg> allow customization of tokenizer factory in Windows.windows<commit_after>package org.deeplearning4j.word2vec.util;

import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

import org.deeplearning4j.word2vec.tokenizer.Tokenizer;
import org.deeplearning4j.word2vec.tokenizer.TokenizerFactory;

public class Windows {
	/**
	 * Constructs a list of window of size windowSize.
	 * Note that padding for each window is created as well.
	 * @param words the words to tokenize and construct windows from
	 * @return the list of windows for the tokenized string
	 */
	public static List<Window> windows(String words) {
		StringTokenizer tokenizer = new StringTokenizer(words);
		List<String> list = new ArrayList<String>();
		while(tokenizer.hasMoreTokens())
			list.add(tokenizer.nextToken());
		return windows(list,5);
	}
	
	/**
	 * Constructs a list of window of size windowSize.
	 * Note that padding for each window is created as well.
	 * @param words the words to tokenize and construct windows from
	 * @param tokenizerFactory tokenizer factory to use
	 * @return the list of windows for the tokenized string
	 */
	public static List<Window> windows(String words,TokenizerFactory tokenizerFactory) {
		Tokenizer tokenizer = tokenizerFactory.create(words);
		List<String> list = new ArrayList<String>();
		while(tokenizer.hasMoreTokens())
			list.add(tokenizer.nextToken());
		return windows(list,5);
	}


	/**
	 * Creates a sliding window from text
	 * @param windowSize the window size to use
{'fraction_non_alphanumeric': 0.08817110432125709, 'fraction_numerical': 0.004801396769969445, 'mean_word_length': 4.590243902439024, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6547282', 'n_tokens_mistral': 672, 'n_tokens_neox': 631, 'n_words': 226}