<commit_before>/*******************************************************************************
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *******************************************************************************/
package ctlogger;

//---------------------------------------------------------------------------------	
// CTlogger:  parse data logger file into CloudTurbine (zip) files
// <NAME>, Cycronix
// 02/25/2014  	Initial version
// 08/06/2014	Add -H specify header line via argument

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Timer;
import java.util.TimerTask;

import cycronix.ctlib.CTreader;
import cycronix.ctlib.CTwriter;

public class CTlogger {
	static boolean zipmode=true;			// true:  zip files
	static boolean debug=false;				// debug prints
	static boolean gzipmode=false;			// gzip for extra compression
	static boolean appendMode=true;			// append to end of existing CT data
	static boolean newFileMode=false;		// re-parse file every time 
	static long autoflush = 3600;			// autoflush interval (sec)
	static long pollInterval=60;			// default 1 minute (60 sec) polling interval
	static long skipLines = 2;				// # lines to skip between first chanNames line to data
	static boolean repeatFetch = false;		// auto-fetch data loop 
	static boolean storeTime = false;		// store time string as channel
	static String nanVal = "0";				// replace NAN with this
	static String loggerFileName=null;		// name of input data file to parse
	static String CTrootfolder=null;		// top level folder to write CT data
	static String leadingID=null;			// leading ID string (IWG1 compliant)
	static String SourceName="CTlogger";	// CT source name
	static String HeaderLine=null;			// optional header line of channel names argument
	static String helpMsg = "CTlogger -x -r -z -g -k <skiplines> -f <flush_sec> -p <poll_sec> -n <nanVal> -i <leadingID> -s <SourceName> -H <HeaderLine> <logger.dat> <CTfolder>";
	static double trimTime=0.;				// amount of data to keep (ring buffer).  
	static boolean noBackwards=false;		// no backwards-going time allowed
	static boolean blockMode=false;			// block-mode CTwrite
	
	public static void main(String args[]) {

     	int dirArg = 0;
     	while((dirArg<args.length) && args[dirArg].startsWith("-")) {		// arg parsing
     		if(args[dirArg].equals("-h")) 	{ 
    			System.err.println(helpMsg);
    			System.exit(0);
     		}
     		if(args[dirArg].equals("-x")) 	{ debug = true;   }
     		if(args[dirArg].equals("-b")) 	{ noBackwards = true;   }
     		if(args[dirArg].equals("-g")) 	{ gzipmode = true;   }			// default false
     		if(args[dirArg].equals("-a")) 	{ appendMode = false;   }		// default true
     		if(args[dirArg].equals("-z"))  	{ zipmode = false; } 			// default true
     		if(args[dirArg].equals("-N")) 	{ newFileMode = true;   }		// default false
     		if(args[dirArg].equals("-f"))   { autoflush = Long.parseLong(args[++dirArg]); }	
     		if(args[dirArg].equals("-p"))   { pollInterval = Long.parseLong(args[++dirArg]); }	
     		if(args[dirArg].equals("-k"))   { skipLines = Long.parseLong(args[++dirArg]); }	
     		if(args[dirArg].equals("-r"))   { repeatFetch = true; }	
     		if(args[dirArg].equals("-B"))   { blockMode = true; }	
     		if(args[dirArg].equals("-t"))   { storeTime = true; }	
     		if(args[dirArg].equals("-T"))   { trimTime = Double.parseDouble(args[++dirArg]); }	
     		if(args[dirArg].equals("-n"))	{ nanVal = args[++dirArg]; }
     		if(args[dirArg].equals("-i"))	{ leadingID = args[++dirArg]; }
     		if(args[dirArg].equals("-s"))	{ SourceName = args[++dirArg]; }
     		if(args[dirArg].equals("-H"))	{ HeaderLine = args[++dirArg]; }
     		dirArg++;
     	}
		if(args.length < (dirArg+2)) {
			System.err.println(helpMsg);
			System.exit(0);
		}
		loggerFileName = args[dirArg++];		// args[0]:  logger.dat file
		CTrootfolder = args[dirArg++];			// args[1]:  CT destination folder     	

		System.err.println("CTlogger: "+loggerFileName+", CTrootfolder: "+CTrootfolder+", pollInterval: "+pollInterval);
		
		if(!repeatFetch) getData(true);		// run once
		else {
			Timer timer = new Timer();
			TimerTask fetchTask = new TimerTask() {
				@Override public void run() {
					if(newFileMode) getData(true);
					else
					if(getData(false)) {		// pick up from old data if you can
						System.err.println("Failed to pick up from old data, refetch from start of file...");
						boolean status = getData(true);	
						System.err.println("refetch status: "+status);
					}
					if(debug) System.err.println("Waiting for data, pollInterval: "+pollInterval+" sec...");
				};
			};
			// repeatFetch@autoflush interval, convert to msec
			if((autoflush>0) && (pollInterval > autoflush)) pollInterval = autoflush;
			timer.scheduleAtFixedRate(fetchTask,  0,  pollInterval*1000);		
		}<commit_msg>CTlogger updates: (1) add Apache header; (2) convert command line parsing to use Apache Commons CLI<commit_after>/*
Copyright 2014-2017 Cycronix

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ctlogger;

//---------------------------------------------------------------------------------	
// CTlogger:  parse data logger file into CloudTurbine (zip) files
// <NAME>, Cycronix
// 02/25/2014  	Initial version
// 08/06/2014	Add -H specify header line via argument
// 02/18/2017	JPW, add command line parsing using Apache Commons CLI

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Timer;
import java.util.TimerTask;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
// Because java.text.ParseException is imported above, we can't import the CLI ParseException here
// import org.apache.commons.cli.ParseException;

import cycronix.ctlib.CTreader;
import cycronix.ctlib.CTwriter;

public class CTlogger {
	static boolean zipmode=true;			// true:  zip files
	static boolean debug=false;				// debug prints
	static boolean gzipmode=false;			// gzip for extra compression
	static boolean appendMode=true;			// append to end of existing CT data
	static boolean newFileMode=false;		// re-parse file every time 
	static long autoflush = 3600;			// autoflush interval (sec)
	static long pollInterval=60;			// default 1 minute (60 sec) polling interval
	static long skipLines = 2;				// # lines to skip between first chanNames line to data
	static boolean repeatFetch = false;		// auto-fetch data loop 
	static boolean storeTime = false;		// store time string as channel
	static String nanVal = "0";				// replace NAN with this
	static String loggerFileName=null;		// name of input data file to parse
	static String CTrootfolder=null;		// top level folder to write CT data
	static String leadingID=null;			// leading ID string (IWG1 compliant)
	static String SourceName="CTlogger";	// CT source name
	static String HeaderLine=null;			// optional header line of channel names argument
	static String helpMsg = "CTlogger -x -r -z -g -k <skiplines> -f <flush_sec> -p <poll_sec> -n <nanVal> -i <leadingID> -s <SourceName> -H <HeaderLine> <logger.dat> <CTfolder>";
	static double trimTime=0.;				// amount of data to keep (ring buffer).  
	static boolean noBackwards=false;		// no backwards-going time allowed
	static boolean blockMode=false;			// block-mode CTwrite
	
	public static void main(String args[]) {
		
		/**
		 * 
		 * Original code for command line parsing
		 * 
     	int dirArg = 0;
     	while((dirArg<args.length) && args[dirArg].startsWith("-")) {		// arg parsing
     		if(args[dirArg].equals("-h")) 	{ 
    			System.err.println(helpMsg);
    			System.exit(0);
     		}
     		if(args[dirArg].equals("-x")) 	{ debug = true;   }
     		if(args[dirArg].equals("-b")) 	{ noBackwards = true;   }
     		if(args[dirArg].equals("-g")) 	{ gzipmode = true;   }			// default false
     		if(args[dirArg].equals("-a")) 	{ appendMode = false;   }		// default true
     		if(args[dirArg].equals("-z"))  	{ zipmode = false; } 			// default true
     		if(args[dirArg].equals("-N")) 	{ newFileMode = true;   }		// default false
     		if(args[dirArg].equals("-f"))   { autoflush = Long.parseLong(args[++dirArg]); }	
     		if(args[dirArg].equals("-p"))   { pollInterval = Long.parseLong(args[++dirArg]); }	
     		if(args[dirArg].equals("-k"))   { skipLines = Long.parseLong(args[++dirArg]); }	
     		if(args[dirArg].equals("-r"))   { repeatFetch = true; }	
     		if(args[dirArg].equals("-B"))   { blockMode = true; }	
     		if(args[dirArg].equals("-t"))   { storeTime = true; }	
     		if(args[dirArg].equals("-T"))   { trimTime = Double.parseDouble(args[++dirArg]); }	
     		if(args[dirArg].equals("-n"))	{ nanVal = args[++dirArg]; }
     		if(args[dirArg].equals("-i"))	{ leadingID = args[++dirArg]; }
     		if(args[dirArg].equals("-s"))	{ SourceName = args[++dirArg]; }
     		if(args[dirArg].equals("-H"))	{ HeaderLine = args[++dirArg]; }
     		dirArg++;
     	}
		if(args.length < (dirArg+2)) {
			System.err.println(helpMsg);
			System.exit(0);
		}
		loggerFileName = args[dirArg++];		// args[0]:  logger.dat file
		CTrootfolder = args[dirArg++];			// args[1]:  CT destination folder     	
		*/
		
		//
		// Parse command line arguments
		//
		// 1. Setup command line options
		//
		Options options = new Options();
		// Boolean options (only the flag, no argument)
		options.addOption("h", "help", false, "Print this message");
		options.addOption("x", "debug", false, "turn on debug output");
		options.addOption("b", "nobackwards", false, "no backwards-going time allowed");
		options.addOption("g", "gzipmode", false, "turn on gzip for extra compression");
		options.addOption("a", "noappend", false, "turn off append mode (i.e., do not append to end of existing CT data)");
		options.addOption("z", "nozip", false, "turn off zip mode (it is on by default)");
		options.addOption("N", "newfilemode", false, "re-parse file every time");
		options.addOption("r", "repeatFetch", false, "turn on repeat fetch (auto-fetch data loop)");
		options.addOption("B", "blockMode", false, "turn on CloudTurbine writer block mode");
		options.addOption("t", "storeTime", false, "store time string as a channel");
		// Options with an argument
		Option outputFolderOption = Option.builder("f")
				.argName("autoflush")
                .hasArg()
                .desc("flush interval (sec); default = \"" + autoflush + "\"")
                .build();
		options.addOption(outputFolderOption);
		outputFolderOption = Option.builder("p")
				.argName("pollInterval")
                .hasArg()
                .desc("polling interval (sec); default = \"" + pollInterval + "\"")
                .build();
		options.addOption(outputFolderOption);
		outputFolderOption = Option.builder("k")
				.argName("skipLines")
                .hasArg()
                .desc("in logger file, the num lines to skip after the header line to get to the first line of data; default = \"" + skipLines + "\"")
                .build();
		options.addOption(outputFolderOption);
		outputFolderOption = Option.builder("T")
				.argName("trimTime")
                .hasArg()
                .desc("trim (ring-buffer loop) time (sec) (trimTime=0 for indefinite); default = \"" + trimTime + "\"")
                .build();
		options.addOption(outputFolderOption);
		outputFolderOption = Option.builder("n")
				.argName("nanVal")
                .hasArg()
                .desc("replace NAN with this; default = \"" + nanVal + "\"")
                .build();
		options.addOption(outputFolderOption);
		outputFolderOption = Option.builder("i")
				.argName("leadingID")
                .hasArg()
                .desc("leading ID string (IWG1 compliant)")
                .build();
		options.addOption(outputFolderOption);
		outputFolderOption = Option.builder("s")
				.argName("sourceName")
                .hasArg()
                .desc("CloudTurbine source name; default = \"" + SourceName + "\"")
                .build();
		options.addOption(outputFolderOption);
		outputFolderOption = Option.builder("H")
				.argName("HeaderLine")
                .hasArg()
                .desc("optional header line of channel names; if not supplied, this is read from the first line in the logger file")
                .build();
		options.addOption(outputFolderOption);
		outputFolderOption = Option.builder("l")
				.argName("loggerfilename")
                .hasArg()
                .desc("name of the logger data file")
                .build();
		options.addOption(outputFolderOption);
		outputFolderOption = Option.builder("o")
                .longOpt("outputfolder")
				.argName("folder")
                .hasArg()
                .desc("Location of output files (source is created under this folder)")
                .build();
		options.addOption(outputFolderOption);

		//
		// 2. Parse command line options
		//
	    CommandLineParser parser = new DefaultParser();
	    CommandLine line = null;
	    try {
	        line = parser.parse( options, args );
	    }
	    catch( org.apache.commons.cli.ParseException exp ) {
	        // oops, something went wrong
	        System.err.println( "Command line argument parsing failed: " + exp.getMessage() );
	     	return;
	    }
	    
	    //
	    // 3. Retrieve the command line values
	    //
	    if (line.hasOption("help")) {
	    	// Display help message and quit
	    	HelpFormatter formatter = new HelpFormatter();
	    	formatter.printHelp( "CTscreencap", options );
	    	return;
	    }
	    
	    //
	    // MAKE SURE logger filename and output folder have been specified!!!!!
		//
	    
		
		System.err.println("CTlogger: "+loggerFileName+", CTrootfolder: "+CTrootfolder+", pollInterval: "+pollInterval);
		
		if(!repeatFetch) getData(true);		// run once
		else {
			Timer timer = new Timer();
			TimerTask fetchTask = new TimerTask() {
				@Override public void run() {
					if(newFileMode) getData(true);
					else
					if(getData(false)) {		// pick up from old data if you can
						System.err.println("Failed to pick up from old data, refetch from start of file...");
						boolean status = getData(true);	
						System.err.println("refetch status: "+status);
					}
					if(debug) System.err.println("Waiting for data, pollInterval: "+pollInterval+" sec...");
				};
			};
			// repeatFetch@autoflush interval, convert to msec
			if((autoflush>0) && (pollInterval > autoflush)) pollInterval = autoflush;
			timer.scheduleAtFixedRate(fetchTask,  0,  pollInterval*1000);		
		}
{'fraction_non_alphanumeric': 0.1522694903709208, 'fraction_numerical': 0.006997337296427024, 'mean_word_length': 2.9690341607274515, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3222579', 'n_tokens_mistral': 5287, 'n_tokens_neox': 4873, 'n_words': 1374}