<commit_before>/*
 * Copyright 2011-15 <NAME>
 *
 * This file is part of OpenMUC.
 * For more information visit http://www.openmuc.org
 *
 * OpenMUC is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * OpenMUC is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with OpenMUC.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
package org.openmuc.framework.datalogger.ascii;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.Locale;

import org.openmuc.framework.data.ByteArrayValue;
import org.openmuc.framework.data.DoubleValue;
import org.openmuc.framework.data.Flag;
import org.openmuc.framework.data.Record;
import org.openmuc.framework.data.StringValue;
import org.openmuc.framework.datalogger.ascii.utils.Const;
import org.openmuc.framework.datalogger.ascii.utils.LoggerUtils;
import org.openmuc.framework.datalogger.spi.LogChannel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class LogFileReader {

	private static final Logger logger = LoggerFactory.getLogger(LogFileReader.class);

	private final String channelId;
	private final String path;
	private final int loggingInterval;
	private final int logTimeOffset;
	private int unixTimestampColumn;
	private long startTimestamp;
	private long endTimestamp;
	private long firstTimestampFromFile;

	/**
	 * Constructor
	 * 
	 * @param path
	 * @param id
	 * @param loggingInterval
	 */
	public LogFileReader(String path, LogChannel logChannel) {

		this.path = path;
		channelId = logChannel.getId();
		this.loggingInterval = logChannel.getLoggingInterval();
		this.logTimeOffset = logChannel.getLoggingTimeOffset();
		firstTimestampFromFile = -1;
	}

	/**
	 * @return All records of the given time span
	 */
	public List<Record> getValues(long startTimestamp, long endTimestamp) {

		this.startTimestamp = startTimestamp;
		this.endTimestamp = endTimestamp;

		List<Record> allRecords = new ArrayList<Record>();

		List<String> filenames = LoggerUtils.getFilenames(loggingInterval, logTimeOffset, this.startTimestamp,
				this.endTimestamp);

		for (int i = 0; i < filenames.size(); i++) {
			Boolean nextFile = false;
			logger.trace("using " + filenames.get(i));

			String filepath;
			if (path.endsWith("/")) {
				filepath = path + filenames.get(i);
			}
			else {
				filepath = path + "/" + filenames.get(i);
			}

			if (i > 0) {
				nextFile = true;
			}
			List<Record> fileRecords = processFile(filepath, nextFile);
			if (fileRecords != null) {
				allRecords.addAll(fileRecords);
				logger.trace("read records: " + fileRecords.size());
			}
			else {
				// some error occurred while processing the file so no records will be added
			}

		}
		return allRecords;
	}

	/**
	 * @param timestamp
	 * @return Record on success, otherwise null
	 */
	public Record getValue(long timestamp) {

		// Returns a record which lays within the interval [timestamp, timestamp + loggingInterval]
		// The interval is necessary for a requested time stamp which lays between the time stamps of two logged values
		// e.g.: t_request = 7, t1_logged = 5, t2_logged = 10, loggingInterval = 5
		// method will return the record of t2_logged because this lays within the interval [7,12]
		// If the t_request matches exactly a logged time stamp, then the according record is returned.

		Record record = null;
		List<Record> records = getValues(timestamp, timestamp);// + loggingInterval);
		if (records.size() == 0) {
			// no record found for requested timestamp

			// TODO statt null flag 3 setzen!
			record = null;
		}
		else if (records.size() == 1) {
			// t_request lays between two logged values
			record = records.get(0);
		}
		else if (records.size() == 2) {
			// t_request matches exactly a logged value
			// so getVaules returns a record for t_request and one for t_request+loggingInterval
			record = records.get(0);
		}

		// nur wert zurückgeben wenn zeitstempel identisch ist
		// sonst

		return record;
	}

	/**
	 * Reads the file line by line
	 * 
	 * @param br
	 * @return records on success, otherwise null
	 */
	private List<Record> processFile(String filepath, Boolean nextFile) {

		List<Record> records = new ArrayList<Record>();
		String line = null;
		long currentPosition = 0;
		long rowSize;
		long firstTimestamp = 0;
		String firstValueLine = null;
		long actualTimestamp = 0;

		RandomAccessFile raf = LoggerUtils.getRandomAccessFile(new File(filepath), "r");
		if (raf == null) {
			return null;
		}
		try {
			int channelColumn = -1;
			while (channelColumn <= 0) {
				line = raf.readLine();
				channelColumn = LoggerUtils.getColumnNumberByName(line, channelId);
				unixTimestampColumn = LoggerUtils.getColumnNumberByName(line, Const.TIMESTAMP_STRING);
			}

			firstValueLine = raf.readLine();
			rowSize = firstValueLine.length() + 1;

			currentPosition = raf.getFilePointer() - rowSize;
			firstTimestamp = (long) (Double.valueOf((firstValueLine.split(Const.SEPARATOR))[unixTimestampColumn]) * 1000);

			if (nextFile || startTimestamp < firstTimestamp) {
				startTimestamp = firstTimestamp;
			}

			if (startTimestamp >= firstTimestamp) {
				long filepos = getFilePosition(loggingInterval, startTimestamp, firstTimestamp, currentPosition,
						rowSize);
				raf.seek(filepos);
				actualTimestamp = startTimestamp;
				while ((line = raf.readLine()) != null && actualTimestamp <= endTimestamp) {
					processLine(line, channelColumn, records);
					actualTimestamp += loggingInterval;
				}
				raf.close();
			}
			else {
				records = null; // because the column of the channel was not identified
			}
		} catch (IOException e) {
			e.printStackTrace();
			records = null;
		}<commit_msg>fix calculate log file position<commit_after>/*
 * Copyright 2011-15 Fraunhofer ISE
 *
 * This file is part of OpenMUC.
 * For more information visit http://www.openmuc.org
 *
 * OpenMUC is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * OpenMUC is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with OpenMUC.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
package org.openmuc.framework.datalogger.ascii;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.Locale;

import org.openmuc.framework.data.ByteArrayValue;
import org.openmuc.framework.data.DoubleValue;
import org.openmuc.framework.data.Flag;
import org.openmuc.framework.data.Record;
import org.openmuc.framework.data.StringValue;
import org.openmuc.framework.datalogger.ascii.utils.Const;
import org.openmuc.framework.datalogger.ascii.utils.LoggerUtils;
import org.openmuc.framework.datalogger.spi.LogChannel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class LogFileReader {

	private static final Logger logger = LoggerFactory.getLogger(LogFileReader.class);

	private final String channelId;
	private final String path;
	private final int loggingInterval;
	private final int logTimeOffset;
	private int unixTimestampColumn;
	private long startTimestamp;
	private long endTimestamp;
	private long firstTimestampFromFile;

	/**
	 * Constructor
	 * 
	 * @param path
	 * @param id
	 * @param loggingInterval
	 */
	public LogFileReader(String path, LogChannel logChannel) {

		this.path = path;
		channelId = logChannel.getId();
		this.loggingInterval = logChannel.getLoggingInterval();
		this.logTimeOffset = logChannel.getLoggingTimeOffset();
		firstTimestampFromFile = -1;
	}

	/**
	 * @return All records of the given time span
	 */
	public List<Record> getValues(long startTimestamp, long endTimestamp) {

		this.startTimestamp = startTimestamp;
		this.endTimestamp = endTimestamp;

		List<Record> allRecords = new ArrayList<Record>();

		List<String> filenames = LoggerUtils.getFilenames(loggingInterval, logTimeOffset, this.startTimestamp,
				this.endTimestamp);

		for (int i = 0; i < filenames.size(); i++) {
			Boolean nextFile = false;
			logger.trace("using " + filenames.get(i));

			String filepath;
			if (path.endsWith("/")) {
				filepath = path + filenames.get(i);
			}
			else {
				filepath = path + "/" + filenames.get(i);
			}

			if (i > 0) {
				nextFile = true;
			}
			List<Record> fileRecords = processFile(filepath, nextFile);
			if (fileRecords != null) {
				allRecords.addAll(fileRecords);
				logger.trace("read records: " + fileRecords.size());
			}
			else {
				// some error occurred while processing the file so no records will be added
			}

		}
		return allRecords;
	}

	/**
	 * @param timestamp
	 * @return Record on success, otherwise null
	 */
	public Record getValue(long timestamp) {

		// Returns a record which lays within the interval [timestamp, timestamp + loggingInterval]
		// The interval is necessary for a requested time stamp which lays between the time stamps of two logged values
		// e.g.: t_request = 7, t1_logged = 5, t2_logged = 10, loggingInterval = 5
		// method will return the record of t2_logged because this lays within the interval [7,12]
		// If the t_request matches exactly a logged time stamp, then the according record is returned.

		Record record = null;
		List<Record> records = getValues(timestamp, timestamp);// + loggingInterval);
		if (records.size() == 0) {
			// no record found for requested timestamp

			// TODO statt null flag 3 setzen!
			record = null;
		}
		else if (records.size() == 1) {
			// t_request lays between two logged values
			record = records.get(0);
		}
		else if (records.size() == 2) {
			// t_request matches exactly a logged value
			// so getVaules returns a record for t_request and one for t_request+loggingInterval
			record = records.get(0);
		}

		// nur wert zurückgeben wenn zeitstempel identisch ist
		// sonst

		return record;
	}

	/**
	 * Reads the file line by line
	 * 
	 * @param br
	 * @return records on success, otherwise null
	 */
	private List<Record> processFile(String filepath, Boolean nextFile) {

		List<Record> records = new ArrayList<Record>();
		String line = null;
		long currentPosition = 0;
		long rowSize;
		long firstTimestamp = 0;
		String firstValueLine = null;
		long actualTimestamp = 0;

		RandomAccessFile raf = LoggerUtils.getRandomAccessFile(new File(filepath), "r");
		if (raf == null) {
			return null;
		}
		try {
			int channelColumn = -1;
			while (channelColumn <= 0) {
				line = raf.readLine();
				channelColumn = LoggerUtils.getColumnNumberByName(line, channelId);
				unixTimestampColumn = LoggerUtils.getColumnNumberByName(line, Const.TIMESTAMP_STRING);
			}

            currentPosition = raf.getFilePointer();
            firstValueLine = raf.readLine();
            raf.seek(raf.getFilePointer() - 2);
            int offset = raf.read() == '\r' ? 2 : 1;
            rowSize = firstValueLine.length() + offset;

			firstTimestamp = (long) (Double.valueOf((firstValueLine.split(Const.SEPARATOR))[unixTimestampColumn]) * 1000);

			if (nextFile || startTimestamp < firstTimestamp) {
				startTimestamp = firstTimestamp;
			}

			if (startTimestamp >= firstTimestamp) {
				long filepos = getFilePosition(loggingInterval, startTimestamp, firstTimestamp, currentPosition,
						rowSize);
				raf.seek(filepos);
				actualTimestamp = startTimestamp;
				while ((line = raf.readLine()) != null && actualTimestamp <= endTimestamp) {
					processLine(line, channelColumn, records);
					actualTimestamp += loggingInterval;
				}
				raf.close();
			}
			else {
				records = null; // because the column of the channel was not identified
			}
		} catch (IOException e) {
			e.printStackTrace();
			records = null;
		}
{'fraction_non_alphanumeric': 0.10128477969575156, 'fraction_numerical': 0.0063056672184125485, 'mean_word_length': 3.7627627627627627, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6445329', 'n_tokens_mistral': 4193, 'n_tokens_neox': 3662, 'n_words': 1152}