<commit_before>package com.dongdongxia.myfastjson.parser;

import java.io.Closeable;
import java.io.IOException;
/**
 * 
 * <P>Description: JSON的词法分析器默认实现</P>
 * @ClassName: JSONLexerBase
 * @author <EMAIL>  2017年6月29日 上午11:31:33
 */
public abstract class JSONLexerBase implements JSONLexer, Closeable{

	protected int bp;
	
	
	protected int features;
	
	/**
	 * 本地缓存的数据
	 */
	protected char[] sbuf;
	
	/**
	 * 缓存本地数据
	 */
	private final static ThreadLocal<char[]> SBUF_LOCAL = new ThreadLocal<char[]>();
	
	protected String stringDefaultValue = null;
	
	public JSONLexerBase(int features) {
		this.features = features;<commit_msg>添加: 标识字段<commit_after>package com.dongdongxia.myfastjson.parser;

import java.io.Closeable;
import java.io.IOException;
/**
 * 
 * <P>Description: JSON的词法分析器默认实现</P>
 * @ClassName: JSONLexerBase
 * @author <EMAIL>  2017年6月29日 上午11:31:33
 */
public abstract class JSONLexerBase implements JSONLexer, Closeable{

	protected char ch;
	protected int bp;
	
	
	protected int features;
	
	/**
	 * 本地缓存的数据
	 */
	protected char[] sbuf;
	
	/**
	 * 缓存本地数据
	 */
	private final static ThreadLocal<char[]> SBUF_LOCAL = new ThreadLocal<char[]>();
	
	protected String stringDefaultValue = null;
	
	public JSONLexerBase(int features) {
		this.features = features;
{'fraction_non_alphanumeric': 0.11746280344557557, 'fraction_numerical': 0.020360219263899765, 'mean_word_length': 4.325, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '532958', 'n_tokens_mistral': 510, 'n_tokens_neox': 459, 'n_words': 99}