<commit_before>//---------------------------------------------------------------------------
// csved_split.cpp
//
// split single field to multple fields
// can split on specific character or on transitions between char types
//
// Copyright (C) 2009 <NAME>
//---------------------------------------------------------------------------

#include "a_base.h"
#include "a_collect.h"
#include "csved_cli.h"
#include "csved_split.h"
#include "csved_strings.h"

using std::string;
using std::vector;

namespace CSVED {

//----------------------------------------------------------------------------
// Transition split values
//----------------------------------------------------------------------------

const char * const FLAG_TRANA2N	= "-tan";
const char * const FLAG_TRANN2A	= "-tna";

//---------------------------------------------------------------------------
// Register split command
//---------------------------------------------------------------------------


static RegisterCommand <SplitFixed> rc1_(
	CMD_SPLFIX,
	"split input using fixed sizes"
);

static RegisterCommand <SplitChar> rc2_(
	CMD_SPLCHR,
	"split using regular expression"
);

static RegisterCommand <SplitRegex> rc3_(
	CMD_SPLREGEX,
	"split at character or character type transition"
);

//----------------------------------------------------------------------------
// Help text
//----------------------------------------------------------------------------

const char * const FSPLIT_HELP = {
	"split CSV field at fixed positions\n"
	"usage: csvfix split_fixed  [flags] [file ...]\n"
	"where flags are:\n"
	"  -f field\tindex of the field to be split\n"
	"  -p plist\tlist of positions to split, in start:len format\n"
	"  -l lengths\tlengths of fields to split (mutually exclusive with -p)\n"
	"  -k\t\tretain field being split in output (default is discard it)\n"
	"#ALL,SKIP,PASS"
};

const char * const RESPLIT_HELP = {
	"split using regular expression\n"
	"usage: csvfix split_fixed  [flags] [file ...]\n"
	"where flags are:\n"
	"  -f field\tindex of the field to be split\n"
	"  -r regex\tregular expression to use to perform split\n"
	"  -k\t\tretain field being split in output (default is discard it)\n"
	"#ALL,SKIP,PASS"
};
const char * const CSPLIT_HELP = {
	"split CSV field at specific character(s) or character type transition\n"
	"usage: csvfix split_char  [flags] [file ...]\n"
	"where flags are:\n"
	"  -f field\tindex of the field to be split (required)\n"
	"  -c char\tcharacter to split at (default is space)\n"
	"  -tcn\tsplit at first transition from character to number\n"
	"  -tnc\tsplit at first transition from number to character\n"
	"  -k\t\tretain field being split in output (default is discard it)\n"
	"#ALL,SKIP,PASS"
};

//---------------------------------------------------------------------------
// Standard Command ctor
//---------------------------------------------------------------------------

SplitBase :: SplitBase( const string & name,
						const string & desc,
						const string & help )
		: Command( name, desc, help ) {

	AddFlag( ALib::CommandLineFlag( FLAG_COLS, true, 1 ) );
	AddFlag( ALib::CommandLineFlag( FLAG_KEEP, false, 0 ) );
}

//---------------------------------------------------------------------------
// Get flags common to all split commands
//---------------------------------------------------------------------------

void SplitBase :: GetCommonFlags( ALib::CommandLine & cl ) {
	mKeep = cl.HasFlag( FLAG_KEEP );
	string nf = cl.GetValue( FLAG_COLS, "" );
	if ( ! ALib::IsInteger( nf ) ) {
		CSVTHROW( "Field specified by " << FLAG_COLS << " must be integer" );
	}
	if ( ALib::ToInteger( nf ) - 1 < 0 ) {
		CSVTHROW( "Invalid field index: " << nf );
	}
	mField = ALib::ToInteger( nf ) - 1;
}

//---------------------------------------------------------------------------
// Helper to insert the split fields into the row at the position
// specified by the -f flag.
//---------------------------------------------------------------------------

void SplitBase :: Insert( CSVRow & row, const CSVRow & split ) {
	CSVRow tmp;
	for ( unsigned int j = 0; j < row.size(); j++ ) {
		if ( j == Field() ) {
			for ( unsigned int k = 0; k < split.size(); k++ ) {
				tmp.push_back( split[k] );
			}
			if ( Keep() ) {
				tmp.push_back( row[j] );
			}
		}
		else {
			tmp.push_back( row[j] );
		}
	}
	row.swap( tmp );
}

//---------------------------------------------------------------------------
// Split via regular expression
//---------------------------------------------------------------------------

SplitRegex :: SplitRegex( const string & name,
							const string & desc )
	: SplitBase( name, desc, RESPLIT_HELP ) {
	AddFlag( ALib::CommandLineFlag( FLAG_REGEX, true, 1 ) );
}

//---------------------------------------------------------------------------
// Split using bracketed exressions then recall and insert into row.
//---------------------------------------------------------------------------

void SplitRegex :: Split( CSVRow & row ) {
    CSVRow tmp;
    string target = Field() < row.size() ? row[ Field() ] : "";
    mRegex.FindIn( target );
    for( unsigned int i = 0; i < mRegex.SavedMatchCount(); i++ ) {
        tmp.push_back( mRegex.SavedMatch(i) );
    }
    Insert( row, tmp );
}

//---------------------------------------------------------------------------
// Perform regex split.
//---------------------------------------------------------------------------

int SplitRegex :: Execute( ALib::CommandLine & cmd ) {

	GetSkipOptions( cmd );
	GetCommonFlags( cmd );

    string rs = cmd.GetValue( FLAG_REGEX );
    mRegex = ALib::RegEx( rs );

	IOManager io( cmd );
	CSVRow row;

	while( io.ReadCSV( row ) ) {
		if ( Skip( io, row ) ) {
			continue;
		}
		if( ! Pass( io,row ) ) {
            Split( row );
		}<commit_msg>added -ic option to split_regex to ignore case when splitting<commit_after>//---------------------------------------------------------------------------
// csved_split.cpp
//
// split single field to multple fields
// can split on specific character or on transitions between char types
//
// Copyright (C) 2009 <NAME>
//---------------------------------------------------------------------------

#include "a_base.h"
#include "a_collect.h"
#include "csved_cli.h"
#include "csved_split.h"
#include "csved_strings.h"

using std::string;
using std::vector;

namespace CSVED {

//----------------------------------------------------------------------------
// Transition split values
//----------------------------------------------------------------------------

const char * const FLAG_TRANA2N	= "-tan";
const char * const FLAG_TRANN2A	= "-tna";

//---------------------------------------------------------------------------
// Register split command
//---------------------------------------------------------------------------


static RegisterCommand <SplitFixed> rc1_(
	CMD_SPLFIX,
	"split input using fixed sizes"
);

static RegisterCommand <SplitChar> rc2_(
	CMD_SPLCHR,
	"split using regular expression"
);

static RegisterCommand <SplitRegex> rc3_(
	CMD_SPLREGEX,
	"split at character or character type transition"
);

//----------------------------------------------------------------------------
// Help text
//----------------------------------------------------------------------------

const char * const FSPLIT_HELP = {
	"split CSV field at fixed positions\n"
	"usage: csvfix split_fixed  [flags] [file ...]\n"
	"where flags are:\n"
	"  -f field\tindex of the field to be split\n"
	"  -p plist\tlist of positions to split, in start:len format\n"
	"  -l lengths\tlengths of fields to split (mutually exclusive with -p)\n"
	"  -k\t\tretain field being split in output (default is discard it)\n"
	"#ALL,SKIP,PASS"
};

const char * const RESPLIT_HELP = {
	"split using regular expression\n"
	"usage: csvfix split_fixed  [flags] [file ...]\n"
	"where flags are:\n"
	"  -f field\tindex of the field to be split\n"
	"  -r regex\tregular expression to use to perform split\n"
	"  -ic\t\tignore case when matching regular expressions\n"
	"  -k\t\tretain field being split in output (default is discard it)\n"
	"#ALL,SKIP,PASS"
};
const char * const CSPLIT_HELP = {
	"split CSV field at specific character(s) or character type transition\n"
	"usage: csvfix split_char  [flags] [file ...]\n"
	"where flags are:\n"
	"  -f field\tindex of the field to be split (required)\n"
	"  -c char\tcharacter to split at (default is space)\n"
	"  -tcn\tsplit at first transition from character to number\n"
	"  -tnc\tsplit at first transition from number to character\n"
	"  -k\t\tretain field being split in output (default is discard it)\n"
	"#ALL,SKIP,PASS"
};

//---------------------------------------------------------------------------
// Standard Command ctor
//---------------------------------------------------------------------------

SplitBase :: SplitBase( const string & name,
						const string & desc,
						const string & help )
		: Command( name, desc, help ) {

	AddFlag( ALib::CommandLineFlag( FLAG_COLS, true, 1 ) );
	AddFlag( ALib::CommandLineFlag( FLAG_KEEP, false, 0 ) );
}

//---------------------------------------------------------------------------
// Get flags common to all split commands
//---------------------------------------------------------------------------

void SplitBase :: GetCommonFlags( ALib::CommandLine & cl ) {
	mKeep = cl.HasFlag( FLAG_KEEP );
	string nf = cl.GetValue( FLAG_COLS, "" );
	if ( ! ALib::IsInteger( nf ) ) {
		CSVTHROW( "Field specified by " << FLAG_COLS << " must be integer" );
	}
	if ( ALib::ToInteger( nf ) - 1 < 0 ) {
		CSVTHROW( "Invalid field index: " << nf );
	}
	mField = ALib::ToInteger( nf ) - 1;
}

//---------------------------------------------------------------------------
// Helper to insert the split fields into the row at the position
// specified by the -f flag.
//---------------------------------------------------------------------------

void SplitBase :: Insert( CSVRow & row, const CSVRow & split ) {
	CSVRow tmp;
	for ( unsigned int j = 0; j < row.size(); j++ ) {
		if ( j == Field() ) {
			for ( unsigned int k = 0; k < split.size(); k++ ) {
				tmp.push_back( split[k] );
			}
			if ( Keep() ) {
				tmp.push_back( row[j] );
			}
		}
		else {
			tmp.push_back( row[j] );
		}
	}
	row.swap( tmp );
}

//---------------------------------------------------------------------------
// Split via regular expression
//---------------------------------------------------------------------------

SplitRegex :: SplitRegex( const string & name,
							const string & desc )
	: SplitBase( name, desc, RESPLIT_HELP ) {
	AddFlag( ALib::CommandLineFlag( FLAG_REGEX, true, 1 ) );
	AddFlag( ALib::CommandLineFlag( FLAG_ICASE, false, 0 ) );
}

//---------------------------------------------------------------------------
// Split using bracketed exressions then recall and insert into row.
//---------------------------------------------------------------------------

void SplitRegex :: Split( CSVRow & row ) {
    CSVRow tmp;
    string target = Field() < row.size() ? row[ Field() ] : "";
    mRegex.FindIn( target );
    for( unsigned int i = 0; i < mRegex.SavedMatchCount(); i++ ) {
        tmp.push_back( mRegex.SavedMatch(i) );
    }
    Insert( row, tmp );
}

//---------------------------------------------------------------------------
// Perform regex split.
//---------------------------------------------------------------------------

int SplitRegex :: Execute( ALib::CommandLine & cmd ) {

	GetSkipOptions( cmd );
	GetCommonFlags( cmd );

    string rs = cmd.GetValue( FLAG_REGEX );
    bool ic = cmd.HasFlag( FLAG_ICASE );
    ALib::RegEx::CaseSense cs = ic  ? ALib::RegEx::Insensitive
                                    : ALib::RegEx::Sensitive;
    mRegex = ALib::RegEx( rs, cs );

	IOManager io( cmd );
	CSVRow row;

	while( io.ReadCSV( row ) ) {
		if ( Skip( io, row ) ) {
			continue;
		}
		if( ! Pass( io,row ) ) {
            Split( row );
		}
{'fraction_non_alphanumeric': 0.3584541872747087, 'fraction_numerical': 0.003101684969402297, 'mean_word_length': 1.290266845843732, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2981005', 'n_tokens_mistral': 3444, 'n_tokens_neox': 3240, 'n_words': 992}