<commit_before>package com.emc.mongoose.tests.system.base;

import com.emc.mongoose.common.api.SizeInBytes;
import com.emc.mongoose.common.env.PathUtil;
import com.emc.mongoose.model.io.IoType;
import com.emc.mongoose.model.io.task.IoTask;
import com.emc.mongoose.tests.system.util.BufferingOutputStream;
import com.emc.mongoose.tests.system.util.LogPatterns;
import com.emc.mongoose.ui.log.LogUtil;
import static com.emc.mongoose.common.Constants.K;
import static com.emc.mongoose.common.Constants.KEY_STEP_NAME;
import static com.emc.mongoose.common.Constants.MIB;
import static com.emc.mongoose.common.env.DateUtil.FMT_DATE_ISO8601;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.SystemUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.ThreadContext;

import org.junit.AfterClass;
import org.junit.BeforeClass;

import static com.emc.mongoose.load.monitor.metrics.MetricsStdoutLogMessage.TABLE_BORDER;
import static com.emc.mongoose.load.monitor.metrics.MetricsStdoutLogMessage.TABLE_HEADER;
import static com.emc.mongoose.model.io.task.IoTask.Status.CANCELLED;
import static com.emc.mongoose.model.io.task.IoTask.Status.SUCC;
import static com.emc.mongoose.tests.system.util.LogPatterns.CELL_BORDER;
import static com.emc.mongoose.tests.system.util.LogPatterns.WHITESPACES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Paths;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 Created by andrey on 19.01.17.
 */
public abstract class LoggingTestBase {
	
	protected static Logger LOG;
	protected static String JOB_NAME;
	protected static BufferingOutputStream STD_OUT_STREAM;
	protected static int LOG_FILE_TIMEOUT_SEC = 50;

	@BeforeClass
	public static void setUpClass()
	throws Exception {
		final URL u = LoggingTestBase.class.getClassLoader().getResource("logging.json");
		System.setProperty("log4j.configurationFile", u.toString());
		LogUtil.init();
		JOB_NAME = ThreadContext.get(KEY_STEP_NAME);
		// remove previous logs if exist
		FileUtils.deleteDirectory(Paths.get(PathUtil.getBaseDir(), "log", JOB_NAME).toFile());
		LOG = LogManager.getLogger();
		STD_OUT_STREAM = new BufferingOutputStream(System.out);
	}
	
	@AfterClass
	public static void tearDownClass()
	throws Exception {
		STD_OUT_STREAM.close();
		//LogUtil.shutdown();
	}

	private static List<String> getLogFileLines(final String fileName)
	throws IOException {
		final File logFile = Paths.get(PathUtil.getBaseDir(), "log", JOB_NAME, fileName).toFile();
		try(final BufferedReader br = new BufferedReader(new FileReader(logFile))) {
			return br.lines().collect(Collectors.toList());
		}
	}

	protected static List<String> getMessageLogLines()
	throws IOException {
		return getLogFileLines("messages.log");
	}

	protected static List<String> getErrorsLogLines()
	throws IOException {
		return getLogFileLines("errors.log");
	}

	protected static List<String> getConfigLogLines()
	throws IOException {
		return getLogFileLines("config.log");
	}

	protected static List<String> getPartsUploadLogLines()
	throws IOException {
		return getLogFileLines("parts.upload.csv");
	}

	protected static List<CSVRecord> getLogFileCsvRecords(final String fileName)
	throws IOException {
		final File logFile = Paths.get(PathUtil.getBaseDir(), "log", JOB_NAME, fileName).toFile();
		long prevSize = -1, nextSize;
		for(int t = 0; t < LOG_FILE_TIMEOUT_SEC; t ++) {
			if(logFile.exists()) {
				nextSize = logFile.length();
				if(prevSize == nextSize) {
					break;
				}
				prevSize = nextSize;
			}
			try {
				TimeUnit.SECONDS.sleep(1);
			} catch(final InterruptedException e) {
				return null;
			}
		}
		try(final BufferedReader br = new BufferedReader(new FileReader(logFile))) {
			try(final CSVParser csvParser = CSVFormat.RFC4180.withHeader().parse(br)) {
				final List<CSVRecord> csvRecords = new ArrayList<>();
				for(final CSVRecord csvRecord : csvParser) {
					csvRecords.add(csvRecord);
				}
				return csvRecords;
			}
		}
	}

	protected static List<CSVRecord> getMetricsMedLogRecords()
	throws IOException {<commit_msg>system tests fix proposal #2<commit_after>package com.emc.mongoose.tests.system.base;

import com.emc.mongoose.common.api.SizeInBytes;
import com.emc.mongoose.common.env.PathUtil;
import com.emc.mongoose.model.io.IoType;
import com.emc.mongoose.model.io.task.IoTask;
import com.emc.mongoose.tests.system.util.BufferingOutputStream;
import com.emc.mongoose.tests.system.util.LogPatterns;
import com.emc.mongoose.ui.log.LogUtil;
import static com.emc.mongoose.common.Constants.K;
import static com.emc.mongoose.common.Constants.KEY_STEP_NAME;
import static com.emc.mongoose.common.Constants.MIB;
import static com.emc.mongoose.common.env.DateUtil.FMT_DATE_ISO8601;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.SystemUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.ThreadContext;

import org.junit.AfterClass;
import org.junit.BeforeClass;

import static com.emc.mongoose.load.monitor.metrics.MetricsStdoutLogMessage.TABLE_BORDER;
import static com.emc.mongoose.load.monitor.metrics.MetricsStdoutLogMessage.TABLE_HEADER;
import static com.emc.mongoose.model.io.task.IoTask.Status.CANCELLED;
import static com.emc.mongoose.model.io.task.IoTask.Status.SUCC;
import static com.emc.mongoose.tests.system.util.LogPatterns.CELL_BORDER;
import static com.emc.mongoose.tests.system.util.LogPatterns.WHITESPACES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Paths;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 Created by andrey on 19.01.17.
 */
public abstract class LoggingTestBase {
	
	protected static Logger LOG;
	protected static String JOB_NAME;
	protected static BufferingOutputStream STD_OUT_STREAM;
	protected static int LOG_FILE_TIMEOUT_SEC = 50;

	@BeforeClass
	public static void setUpClass()
	throws Exception {
		final URL u = LoggingTestBase.class.getClassLoader().getResource("logging.json");
		System.setProperty("log4j.configurationFile", u.toString());
		LogUtil.init();
		JOB_NAME = ThreadContext.get(KEY_STEP_NAME);
		// remove previous logs if exist
		FileUtils.deleteDirectory(Paths.get(PathUtil.getBaseDir(), "log", JOB_NAME).toFile());
		LOG = LogManager.getLogger();
		STD_OUT_STREAM = new BufferingOutputStream(System.out);
	}
	
	@AfterClass
	public static void tearDownClass()
	throws Exception {
		STD_OUT_STREAM.close();
		//LogUtil.shutdown();
	}

	private static List<String> getLogFileLines(final String fileName)
	throws IOException {
		final File logFile = Paths.get(PathUtil.getBaseDir(), "log", JOB_NAME, fileName).toFile();
		try(final BufferedReader br = new BufferedReader(new FileReader(logFile))) {
			return br.lines().collect(Collectors.toList());
		}
	}

	protected static List<String> getMessageLogLines()
	throws IOException {
		return getLogFileLines("messages.log");
	}

	protected static List<String> getErrorsLogLines()
	throws IOException {
		return getLogFileLines("errors.log");
	}

	protected static List<String> getConfigLogLines()
	throws IOException {
		return getLogFileLines("config.log");
	}

	protected static List<String> getPartsUploadLogLines()
	throws IOException {
		return getLogFileLines("parts.upload.csv");
	}

	protected static List<CSVRecord> getLogFileCsvRecords(final String fileName)
	throws IOException {
		final File logFile = Paths.get(PathUtil.getBaseDir(), "log", JOB_NAME, fileName).toFile();
		long prevSize = 1, nextSize;
		for(int t = 0; t < LOG_FILE_TIMEOUT_SEC; t ++) {
			if(logFile.exists()) {
				nextSize = logFile.length();
				if(prevSize == nextSize) {
					break;
				}
				prevSize = nextSize;
			}
			try {
				TimeUnit.SECONDS.sleep(1);
			} catch(final InterruptedException e) {
				return null;
			}
		}
		try(final BufferedReader br = new BufferedReader(new FileReader(logFile))) {
			try(final CSVParser csvParser = CSVFormat.RFC4180.withHeader().parse(br)) {
				final List<CSVRecord> csvRecords = new ArrayList<>();
				for(final CSVRecord csvRecord : csvParser) {
					csvRecords.add(csvRecord);
				}
				return csvRecords;
			}
		}
	}

	protected static List<CSVRecord> getMetricsMedLogRecords()
	throws IOException {
{'fraction_non_alphanumeric': 0.11386554621848739, 'fraction_numerical': 0.004936974789915966, 'mean_word_length': 5.174448767833982, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1698321', 'n_tokens_mistral': 3286, 'n_tokens_neox': 3120, 'n_words': 598}