<commit_before>/* Copyright (C) 2002 Univ. of Massachusetts Amherst, Computer Science Dept.
   This file is part of "MALLET" (MAchine Learning for LanguagE Toolkit).
   http://www.cs.umass.edu/~mccallum/mallet
   This software is provided under the terms of the Common Public License,
   version 1.0, as published by http://www.opensource.org.  For further
   information, see the file `LICENSE' included with this distribution. */




/** 
   @author <NAME> <a href="mailto:<EMAIL>"><EMAIL></a>
 */

package cc.mallet.util;

import java.io.*;
import java.lang.CharSequence;
import java.util.Iterator;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.io.*;

import cc.mallet.util.Lexer;

public class CharSequenceLexer implements Lexer, Serializable
{
	// Some predefined lexing rules
	public static final Pattern LEX_ALPHA = Pattern.compile ("\\p{Alpha}+");
	public static final Pattern LEX_WORDS = Pattern.compile ("\\w+");
	public static final Pattern LEX_NONWHITESPACE_TOGETHER = Pattern.compile ("\\S+");
	public static final Pattern LEX_WORD_CLASSES	=
	Pattern.compile ("\\p{Alpha}+|\\p{Digit}+");
	public static final Pattern LEX_NONWHITESPACE_CLASSES	=
	Pattern.compile ("\\p{Alpha}+|\\p{Digit}+|\\p{Punct}");

	Pattern regex;
	Matcher matcher = null;
	CharSequence input;
	String matchText;<commit_msg>added unicode pattern<commit_after>/* Copyright (C) 2002 Univ. of Massachusetts Amherst, Computer Science Dept.
   This file is part of "MALLET" (MAchine Learning for LanguagE Toolkit).
   http://www.cs.umass.edu/~mccallum/mallet
   This software is provided under the terms of the Common Public License,
   version 1.0, as published by http://www.opensource.org.  For further
   information, see the file `LICENSE' included with this distribution. */




/** 
   @author <NAME> <a href="mailto:<EMAIL>"><EMAIL></a>
 */

package cc.mallet.util;

import java.io.*;
import java.lang.CharSequence;
import java.util.Iterator;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.io.*;

import cc.mallet.util.Lexer;

public class CharSequenceLexer implements Lexer, Serializable
{
	// Some predefined lexing rules
	public static final Pattern LEX_ALPHA = Pattern.compile ("\\p{Alpha}+");
	public static final Pattern LEX_WORDS = Pattern.compile ("\\w+");
	public static final Pattern LEX_NONWHITESPACE_TOGETHER = Pattern.compile ("\\S+");
	public static final Pattern LEX_WORD_CLASSES	=
	Pattern.compile ("\\p{Alpha}+|\\p{Digit}+");
	public static final Pattern LEX_NONWHITESPACE_CLASSES	=
	Pattern.compile ("\\p{Alpha}+|\\p{Digit}+|\\p{Punct}");

	// Lowercase letters and uppercase letters
	public static final Pattern UNICODE_LETTERS =
		Pattern.compile("[\\u{Ll}&&\\u{Lu}]+");

	Pattern regex;
	Matcher matcher = null;
	CharSequence input;
	String matchText;
{'fraction_non_alphanumeric': 0.12916962384669978, 'fraction_numerical': 0.0042583392476933995, 'mean_word_length': 4.505859375, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7379616', 'n_tokens_mistral': 945, 'n_tokens_neox': 891, 'n_words': 264}