<commit_before>/*
 * Copyright 2015-2018 JKOOL, LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.jkoolcloud.tnt4j.streams.parsers;

import static com.jkoolcloud.tnt4j.streams.fields.StreamFieldType.*;

import java.security.MessageDigest;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.jkoolcloud.tnt4j.core.OpLevel;
import com.jkoolcloud.tnt4j.streams.configure.SyslogParserProperties;
import com.jkoolcloud.tnt4j.streams.utils.StreamsResources;
import com.jkoolcloud.tnt4j.streams.utils.SyslogStreamConstants;
import com.jkoolcloud.tnt4j.streams.utils.Utils;

/**
 * Base class for abstract syslog entries data parser that assumes each activity data item is an Syslog server event
 * {@link org.graylog2.syslog4j.server.SyslogServerEventIF} or Syslog log line {@link String}. Parser resolved Syslog
 * entry data fields are put into {@link Map} afterwards mapped into activity fields and properties according to defined
 * parser configuration.
 * <p>
 * This parser supports the following properties (in addition to those supported by {@link AbstractActivityMapParser}):
 * <ul>
 * <li>SuppressMessagesLevel - Syslog messages suppression level:
 * <ul>
 * <li>{@code 0} - output all Syslog messages</li>
 * <li>{@code -1} - output only the first occurrence of Syslog message</li>
 * <li>any other positive number - suppresses all Syslog messages except those that are multiples of that number</li>
 * </ul>
 * Default value - {@value #DEFAULT_SUPPRESSION_LEVEL}. (Optional)</li>
 * <li>SuppressIgnoredFields - Syslog message ignored fields list used to compare if message contents are same. Default
 * value - ['EndTime', 'ElapsedTime', 'Tag']. (Optional)</li>
 * <li>SuppressCacheSize - maximal Syslog messages suppression cache entries count. Default value -
 * {@value #DEFAULT_MAX_CACHE_SIZE}. (Optional)</li>
 * <li>SuppressCacheExpireDurationMinutes - Syslog messages suppression cache entries expiration duration value in
 * minutes. Default value - {@value #DEFAULT_CACHE_EXPIRE_DURATION}. (Optional)</li>
 * <li>FlattenStructuredData - flag indicating to flatten structured data map if there is only one structure available.
 * Default value - {@code false}. (Optional)</li>
 * </ul>
 *
 * @version $Revision: 1 $
 */
public abstract class AbstractSyslogParser extends AbstractActivityMapParser {

	/**
	 * Constant for default messages suppression cache maximum entries count.
	 */
	public static final long DEFAULT_MAX_CACHE_SIZE = 100;
	/**
	 * Constant for default messages suppression cache entries expiration duration in minutes.
	 */
	public static final long DEFAULT_CACHE_EXPIRE_DURATION = 10;
	/**
	 * Constant for default messages suppression level.
	 */
	public static final int DEFAULT_SUPPRESSION_LEVEL = 0;

	/**
	 * Constant for default array of log entry suppression ignored fields.
	 */
	final static String[] DEFAULT_IGNORED_FIELDS = { EndTime.name(), ElapsedTime.name(), Tag.name() };

	private int suppressionLevel = DEFAULT_SUPPRESSION_LEVEL;
	private long cacheSize = DEFAULT_MAX_CACHE_SIZE;
	private long cacheExpireDuration = DEFAULT_CACHE_EXPIRE_DURATION;
	private List<String> ignoredFields = Arrays.asList(DEFAULT_IGNORED_FIELDS);
	private boolean flattenStructuredData = false;

	private static final MessageDigest MSG_DIGEST = Utils.getMD5Digester();
	private final Map<String, Long> EVENT_TIMESTAMP_MAP = new HashMap<>(8);

	private Cache<String, AtomicInteger> msc;

	protected final ReentrantLock digestLock = new ReentrantLock();
	protected final ReentrantLock cacheLock = new ReentrantLock();

	@Override
	public void setProperty(String name, String value) {
		super.setProperty(name, value);<commit_msg>Set default data type setting for parser according to resolved RAW activity values.<commit_after>/*
 * Copyright 2015-2018 JKOOL, LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.jkoolcloud.tnt4j.streams.parsers;

import static com.jkoolcloud.tnt4j.streams.fields.StreamFieldType.*;

import java.security.MessageDigest;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.jkoolcloud.tnt4j.core.OpLevel;
import com.jkoolcloud.tnt4j.streams.configure.SyslogParserProperties;
import com.jkoolcloud.tnt4j.streams.utils.StreamsResources;
import com.jkoolcloud.tnt4j.streams.utils.SyslogStreamConstants;
import com.jkoolcloud.tnt4j.streams.utils.Utils;

/**
 * Base class for abstract syslog entries data parser that assumes each activity data item is an Syslog server event
 * {@link org.graylog2.syslog4j.server.SyslogServerEventIF} or Syslog log line {@link String}. Parser resolved Syslog
 * entry data fields are put into {@link Map} afterwards mapped into activity fields and properties according to defined
 * parser configuration.
 * <p>
 * This parser supports the following properties (in addition to those supported by {@link AbstractActivityMapParser}):
 * <ul>
 * <li>SuppressMessagesLevel - Syslog messages suppression level:
 * <ul>
 * <li>{@code 0} - output all Syslog messages</li>
 * <li>{@code -1} - output only the first occurrence of Syslog message</li>
 * <li>any other positive number - suppresses all Syslog messages except those that are multiples of that number</li>
 * </ul>
 * Default value - {@value #DEFAULT_SUPPRESSION_LEVEL}. (Optional)</li>
 * <li>SuppressIgnoredFields - Syslog message ignored fields list used to compare if message contents are same. Default
 * value - ['EndTime', 'ElapsedTime', 'Tag']. (Optional)</li>
 * <li>SuppressCacheSize - maximal Syslog messages suppression cache entries count. Default value -
 * {@value #DEFAULT_MAX_CACHE_SIZE}. (Optional)</li>
 * <li>SuppressCacheExpireDurationMinutes - Syslog messages suppression cache entries expiration duration value in
 * minutes. Default value - {@value #DEFAULT_CACHE_EXPIRE_DURATION}. (Optional)</li>
 * <li>FlattenStructuredData - flag indicating to flatten structured data map if there is only one structure available.
 * Default value - {@code false}. (Optional)</li>
 * </ul>
 *
 * @version $Revision: 1 $
 */
public abstract class AbstractSyslogParser extends AbstractActivityMapParser {

	/**
	 * Constant for default messages suppression cache maximum entries count.
	 */
	public static final long DEFAULT_MAX_CACHE_SIZE = 100;
	/**
	 * Constant for default messages suppression cache entries expiration duration in minutes.
	 */
	public static final long DEFAULT_CACHE_EXPIRE_DURATION = 10;
	/**
	 * Constant for default messages suppression level.
	 */
	public static final int DEFAULT_SUPPRESSION_LEVEL = 0;

	/**
	 * Constant for default array of log entry suppression ignored fields.
	 */
	final static String[] DEFAULT_IGNORED_FIELDS = { EndTime.name(), ElapsedTime.name(), Tag.name() };

	private int suppressionLevel = DEFAULT_SUPPRESSION_LEVEL;
	private long cacheSize = DEFAULT_MAX_CACHE_SIZE;
	private long cacheExpireDuration = DEFAULT_CACHE_EXPIRE_DURATION;
	private List<String> ignoredFields = Arrays.asList(DEFAULT_IGNORED_FIELDS);
	private boolean flattenStructuredData = false;

	private static final MessageDigest MSG_DIGEST = Utils.getMD5Digester();
	private final Map<String, Long> EVENT_TIMESTAMP_MAP = new HashMap<>(8);

	private Cache<String, AtomicInteger> msc;

	protected final ReentrantLock digestLock = new ReentrantLock();
	protected final ReentrantLock cacheLock = new ReentrantLock();

	/**
	 * Constructs a new AbstractSyslogParser.
	 */
	protected AbstractSyslogParser() {
		super();
	}

	@Override
	public void setProperty(String name, String value) {
		super.setProperty(name, value);
{'fraction_non_alphanumeric': 0.09058050383351589, 'fraction_numerical': 0.007447973713033954, 'mean_word_length': 5.019116677653263, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6650116', 'n_tokens_mistral': 2618, 'n_tokens_neox': 2514, 'n_words': 866}