<commit_before>/*
 * Copyright 2015 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.heliosapm.opentsdb.client.jvmjmx.custom;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.AttributeKeyResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.AttributeKeyTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.AttributeValueTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.DescriptorKeyTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.DescriptorValueTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.MBeanClassNameTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.MBeanDescriptionTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.ObjectNameDomainTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.ObjectNameKeyTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.ObjectNameValueTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.ScriptTokenResolver;

/**
 * <p>Title: Tokener</p>
 * <p>Description: Parses and resolves expression tokens</p> 
 * <p>Company: Helios Development Group LLC</p>
 * @author Whitehead (nwhitehead AT heliosdev DOT org)
 * <p><code>com.heliosapm.opentsdb.client.jvmjmx.custom.Tokener</code></p>
 */

public enum Tokener {
	ATTRIBUTE_KEYR("akr", new AttributeKeyTokenResolver(false, true)),
	ATTRIBUTE_KEYT("akt", new AttributeKeyTokenResolver(true, false)),
	ATTRIBUTE_KEY("ak", new AttributeKeyResolver()),
	ATTRIBUTE_VALUE("av", new AttributeValueTokenResolver()),
	OBJECTNAME_KEYT("onkt", new ObjectNameKeyTokenResolver(false, true)),
	OBJECTNAME_KEYR("onkr", new ObjectNameKeyTokenResolver(true, false)),
	OBJECTNAME_VALUE("onv", new ObjectNameValueTokenResolver()),
	OBJECTNAME_DOMAIN("od", new ObjectNameDomainTokenResolver()),
	SCRIPT_EXEC("s", new ScriptTokenResolver()),
	MBEAN_CLASS("mc", new MBeanClassNameTokenResolver()),
	MBEAN_DESCRIPTION("md", new MBeanDescriptionTokenResolver()),
	// FIXME:  These are broken
	DESCRIPTOR_KEYR("dkr", new DescriptorKeyTokenResolver(false, true)),
	DESCRIPTOR_KEYT("dkt", new DescriptorKeyTokenResolver(true, false)),
	DESCRIPTOR_VALUE("dv", new DescriptorValueTokenResolver());

	
	public static final Map<String, Tokener> TOKEN2ENUM;
	
	static {
		Tokener[] values = Tokener.values();
		Map<String, Tokener> tmp = new HashMap<String, Tokener>(values.length);
		for(Tokener t: values) {
			tmp.put(t.token, t);
		}
		TOKEN2ENUM = Collections.unmodifiableMap(tmp);
	}
	
	private Tokener(final String token, final TokenResolver resolver) {
		this.token = token;
		this.resolver = resolver;
	}
	
	/** The target of the value extraction */
	public final String token;
	/** The token's token resolver */
	public final TokenResolver resolver;
	
	/**
	 * Decodes the passed name to a tokener.
	 * @param name Either the tokener enum name or the define token value. Arguments are trimmed and cased appropriately.
	 * @return the decoded tokener
	 */
	public static Tokener forName(final String name) {
		if(name==null || name.trim().isEmpty()) throw new IllegalArgumentException("The passed name [" + name + "] is not a valid tokener");
		Tokener tok = TOKEN2ENUM.get(name.trim().toLowerCase());
		if(tok!=null) return tok;
		try {
			return Tokener.valueOf(name.trim().toUpperCase());
		} catch (Exception ex) {
			throw new IllegalArgumentException("The passed name [" + name + "] is not a valid tokener");
		}
	}
	
	/*
$a{attrname} : the value of an attribute
$pa{attrname(n, n)} : the pattern key expression applied to an attribute valie
$ond{(n,n)}: the object name domain
$onp{(k)}: the keyed property in the object name
$s{scriptname}: eval a script against kitchen sink
$d{descriptorname}
MBean Info:
	classname
	description

	 */
	
	/**
	 * Resolves the expression, replacing each recognized token with the matched value in the data context
	 * @param dctx The monitor data context
	 * @param fullExpression The expression to resolve
	 * @return the resolved expression
	 */
	public static String resolve(final ExpressionDataContext dctx, final String fullExpression) {
		if(fullExpression==null || fullExpression.trim().isEmpty()) return null;
		final StringBuffer sb = new StringBuffer();
		final Matcher m = TOKEN_PATTERN.matcher(fullExpression.trim());
		while(m.find()) {
			ParsedToken pt = ParsedToken.getParsedToken(m.group(0));
			if(pt!=null) {
				Object resolved = pt.getToken().resolver.resolve(dctx, pt.getKey(), pt.getQual(), pt.getRanges());
				if(resolved!=null) {
					m.appendReplacement(sb, resolved.toString());
				} else {
					m.appendReplacement(sb, "");
				}
			}
		}
		m.appendTail(sb);
		return sb.toString();
	}
	
	
	/** The token splitter */
	public static final Pattern TOKEN_PATTERN = Pattern.compile("\\$(.*)?\\{(.*?)(?:\\((\\d+(?:[,-]\\d+)*)\\))?(?:\\[(.*)?\\])?\\}?");
	
	public static class IntRange {
		/** A cache of IntRanges keyed by the original expression */
		private static final ConcurrentHashMap<String, IntRange> rangeCache = new ConcurrentHashMap<String, IntRange>();
		/** The int range expression */
		public static final Pattern RANGE_PATTERN = Pattern.compile("(\\d+(?:[,-]\\d+)*)");
		/** Comma splitter pattern */
		public static final Pattern COMMA_SPLIT = Pattern.compile(",");
		private static final IntRange[] EMPTY_ARR = {};
		/** The range which is an array of ints with a length of one for a single value, or a length of two for a wider range */
		private final int[] range;<commit_msg>DevSynch<commit_after>/*
 * Copyright 2015 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.heliosapm.opentsdb.client.jvmjmx.custom;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.AttributeKeyResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.AttributeKeyTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.AttributeValueTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.DescriptorKeyTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.DescriptorValueTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.MBeanClassNameTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.MBeanDescriptionTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.ObjectNameDomainTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.ObjectNameKeyTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.ObjectNameValueTokenResolver;
import com.heliosapm.opentsdb.client.jvmjmx.custom.TokenResolvers.ScriptTokenResolver;

/**
 * <p>Title: Tokener</p>
 * <p>Description: Parses and resolves expression tokens</p> 
 * <p>Company: Helios Development Group LLC</p>
 * @author Whitehead (nwhitehead AT heliosdev DOT org)
 * <p><code>com.heliosapm.opentsdb.client.jvmjmx.custom.Tokener</code></p>
 */

public enum Tokener {
	ATTRIBUTE_KEYR("akr", new AttributeKeyTokenResolver(false, true)),
	ATTRIBUTE_KEYT("akt", new AttributeKeyTokenResolver(true, false)),
	ATTRIBUTE_KEY("ak", new AttributeKeyResolver()),
	ATTRIBUTE_VALUE("av", new AttributeValueTokenResolver()),
	OBJECTNAME_KEYT("onkt", new ObjectNameKeyTokenResolver(false, true)),
	OBJECTNAME_KEYR("onkr", new ObjectNameKeyTokenResolver(true, false)),
	OBJECTNAME_VALUE("onv", new ObjectNameValueTokenResolver()),
	OBJECTNAME_DOMAIN("od", new ObjectNameDomainTokenResolver()),
	SCRIPT_EXEC("s", new ScriptTokenResolver()),
	MBEAN_CLASS("mc", new MBeanClassNameTokenResolver()),
	MBEAN_DESCRIPTION("md", new MBeanDescriptionTokenResolver()),
	// FIXME:  These are broken
	DESCRIPTOR_KEYR("dkr", new DescriptorKeyTokenResolver(false, true)),
	DESCRIPTOR_KEYT("dkt", new DescriptorKeyTokenResolver(true, false)),
	DESCRIPTOR_VALUE("dv", new DescriptorValueTokenResolver());

	
	public static final Map<String, Tokener> TOKEN2ENUM;
	
	static {
		Tokener[] values = Tokener.values();
		Map<String, Tokener> tmp = new HashMap<String, Tokener>(values.length);
		for(Tokener t: values) {
			tmp.put(t.token, t);
		}
		TOKEN2ENUM = Collections.unmodifiableMap(tmp);
	}
	
	private Tokener(final String token, final TokenResolver resolver) {
		this.token = token;
		this.resolver = resolver;
	}
	
	/** The target of the value extraction */
	public final String token;
	/** The token's token resolver */
	public final TokenResolver resolver;
	
	/**
	 * Decodes the passed name to a tokener.
	 * @param name Either the tokener enum name or the define token value. Arguments are trimmed and cased appropriately.
	 * @return the decoded tokener
	 */
	public static Tokener forName(final String name) {
		if(name==null || name.trim().isEmpty()) throw new IllegalArgumentException("The passed name [" + name + "] is not a valid tokener");
		Tokener tok = TOKEN2ENUM.get(name.trim().toLowerCase());
		if(tok!=null) return tok;
		try {
			return Tokener.valueOf(name.trim().toUpperCase());
		} catch (Exception ex) {
			throw new IllegalArgumentException("The passed name [" + name + "] is not a valid tokener");
		}
	}
	
	/*
$a{attrname} : the value of an attribute
$pa{attrname(n, n)} : the pattern key expression applied to an attribute valie
$ond{(n,n)}: the object name domain
$onp{(k)}: the keyed property in the object name
$s{scriptname}: eval a script against kitchen sink
$d{descriptorname}
MBean Info:
	classname
	description

	 */
	
	/**
	 * Resolves the expression, replacing each recognized token with the matched value in the data context
	 * @param dctx The monitor data context
	 * @param fullExpression The expression to resolve
	 * @return the resolved expression
	 */
	public static String resolve(final ExpressionDataContext dctx, final String fullExpression) {
		if(fullExpression==null || fullExpression.trim().isEmpty()) return null;
		final StringBuffer sb = new StringBuffer();
		final Matcher m = TOKEN_PATTERN.matcher(fullExpression.trim());
		while(m.find()) {
			final String op = m.group(1);
			final String params = m.group(3);
			ParsedToken pt = ParsedToken.getParsedToken(m.group(0));
			if(pt!=null) {
				Object resolved = pt.getToken().resolver.resolve(dctx, pt.getKey(), pt.getQual(), pt.getRanges());
				if(resolved!=null) {
					m.appendReplacement(sb, resolved.toString());
				} else {
					m.appendReplacement(sb, "");
				}
			}
		}
		m.appendTail(sb);
		return sb.toString();
	}
	
	
	/** The token splitter */
	public static final Pattern TOKEN_PATTERN = Pattern.compile("\\$\\{(.*?)(:(.*?)?)?\\}");
	
	public static class IntRange {
		/** A cache of IntRanges keyed by the original expression */
		private static final ConcurrentHashMap<String, IntRange> rangeCache = new ConcurrentHashMap<String, IntRange>();
		/** The int range expression */
		public static final Pattern RANGE_PATTERN = Pattern.compile("(\\d+(?:[,-]\\d+)*)");
		/** Comma splitter pattern */
		public static final Pattern COMMA_SPLIT = Pattern.compile(",");
		private static final IntRange[] EMPTY_ARR = {};
		/** The range which is an array of ints with a length of one for a single value, or a length of two for a wider range */
		private final int[] range;
{'fraction_non_alphanumeric': 0.11917458845351264, 'fraction_numerical': 0.002009428858489837, 'mean_word_length': 5.106654082114205, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2146886', 'n_tokens_mistral': 4164, 'n_tokens_neox': 3967, 'n_words': 1059}