<commit_before><?php
// Copyright (C) 2010 Combodo SARL
//
//   This program is free software; you can redistribute it and/or modify
//   it under the terms of the GNU General Public License as published by
//   the Free Software Foundation; version 3 of the License.
//
//   This program is distributed in the hope that it will be useful,
//   but WITHOUT ANY WARRANTY; without even the implied warranty of
//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//   GNU General Public License for more details.
//
//   You should have received a copy of the GNU General Public License
//   along with this program; if not, write to the Free Software
//   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

/**
 * Import web service 
 *
 * @author      <NAME> <<EMAIL>>
 * @author      <NAME> <<EMAIL>>
 * @author      <NAME> <<EMAIL>>
 * @license     http://www.opensource.org/licenses/gpl-3.0.html LGPL
 */

//
// Known limitations
// - reconciliation is made on the first column
//
// Known issues
// - ALMOST impossible to troubleshoot when an externl key has a wrong value
// - no character escaping in the xml output (yes !?!?!)
// - not outputing xml when a wrong input is given (class, attribute names)
//

if (!defined('__DIR__')) define('__DIR__', dirname(__FILE__));
require_once(__DIR__.'/../approot.inc.php');
require_once(APPROOT.'/application/application.inc.php');
require_once(APPROOT.'/application/webpage.class.inc.php');
require_once(APPROOT.'/application/csvpage.class.inc.php');
require_once(APPROOT.'/application/clipage.class.inc.php');

require_once(APPROOT.'/application/startup.inc.php');

class BulkLoadException extends Exception
{
}

$aPageParams = array
(
	'auth_user' => array
	(
		'mandatory' => true,
		'modes' => 'cli',
		'default' => null,
		'description' => 'login (must have enough rights to create objects of the given class)',
	),
	'auth_pwd' => array
	(
		'mandatory' => true,
		'modes' => 'cli',
		'default' => null,
		'description' => 'password',
	),
	'class' => array
	(
		'mandatory' => true,
		'modes' => 'http,cli',
		'default' => null,
		'description' => 'class of loaded objects',
	),
	'csvdata' => array
	(
		'mandatory' => true,
		'modes' => 'http',
		'default' => null,
		'description' => 'data',
	),
	'csvfile' => array
	(
		'mandatory' => true,
		'modes' => 'cli',
		'default' => '',
		'description' => 'local data file, replaces csvdata if specified',
	),
	'charset' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => 'UTF-8',
		'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15',
	),
	'date_format' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '',
		'description' => 'Input date format (used both for dates and datetimes) - Examples: %Y-%m-%d, %d/%m/%Y (Europe) - no transformation is applied if the argument is omitted',
	),
	'separator' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => ',',
		'description' => 'column separator in CSV data',
	),
	'qualifier' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '"',
		'description' => 'test qualifier in CSV data',
	),
	'output' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => 'summary',
		'description' => '[retcode] to return the count of lines in error, [summary] to return a concise report, [details] to get a detailed report (each line listed)',
	),
/*
	'reportlevel' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => 'errors|warnings|created|changed|unchanged',
		'description' => 'combination of flags to limit the detailed output',
	),
*/
	'reconciliationkeys' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '',
		'description' => 'name of the columns used to identify existing objects and update them, or create a new one',
	),
	'simulate' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '0',
		'description' => 'If set to 1, then the load will not be executed, but the expected report will be produced',
	),
	'comment' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '',
		'description' => 'Comment to be added into the change log',
	),
);

function UsageAndExit($oP)
{
	global $aPageParams;
	$bModeCLI = utils::IsModeCLI();

	$oP->p("USAGE:\n");
	foreach($aPageParams as $sParam => $aParamData)
	{
		$aModes = explode(',', $aParamData['modes']);
		if ($bModeCLI)
		{
			if (in_array('cli', $aModes))
			{
				$sDesc = $aParamData['description'].', '.($aParamData['mandatory'] ? 'mandatory' : 'optional, defaults to ['.$aParamData['default'].']');
				$oP->p("$sParam = $sDesc");
			}
		}
		else
		{
			if (in_array('http', $aModes))
			{
				$sDesc = $aParamData['description'].', '.($aParamData['mandatory'] ? 'mandatory' : 'optional, defaults to ['.$aParamData['default'].']');
				$oP->p("$sParam = $sDesc");
			}
		}
	}
	$oP->output();
	exit;
}


function ReadParam($oP, $sParam)
{
	global $aPageParams;
	assert(isset($aPageParams[$sParam]));
	assert(!$aPageParams[$sParam]['mandatory']);
	$sValue = utils::ReadParam($sParam, $aPageParams[$sParam]['default'], true /* Allow CLI */);
	return trim($sValue);
}

function ReadMandatoryParam($oP, $sParam)
{
	global $aPageParams;
	assert(isset($aPageParams[$sParam]));
	assert($aPageParams[$sParam]['mandatory']);

	$sValue = utils::ReadParam($sParam, null, true /* Allow CLI */);
	if (is_null($sValue))
	{
		$oP->p("ERROR: Missing argument '$sParam'\n");
		UsageAndExit($oP);
	}
	return trim($sValue);
}

/////////////////////////////////
// Main program

if (utils::IsModeCLI())
{
	$oP = new CLIPage("iTop - Bulk import");
}
else
{
	$oP = new CSVPage("iTop - Bulk import");
}

try
{
	utils::UseParamFile();
}
catch(Exception $e)
{
	$oP->p("Error: ".$e->GetMessage());
	$oP->output();
	exit -2;
}

if (utils::IsModeCLI())
{
	// Next steps:
	//   specific arguments: 'csvfile'
	//   
	$sAuthUser = ReadMandatoryParam($oP, 'auth_user');
	$sAuthPwd = ReadMandatoryParam($oP, 'auth_pwd');
	$sCsvFile = ReadMandatoryParam($oP, 'csvfile');
	if (UserRights::CheckCredentials($sAuthUser, $sAuthPwd))
	{
		UserRights::Login($sAuthUser); // Login & set the user's language
	}
	else
	{
		$oP->p("Access restricted or wrong credentials ('$sAuthUser')");
		$oP->output();
		exit -1;
	}

	if (!is_readable($sCsvFile))
	{
		$oP->p("Input file could not be found or could not be read: '$sCsvFile'");
		$oP->output();
		exit -1;
	}
	$sCSVData = file_get_contents($sCsvFile);

}
else
{
	$_SESSION['login_mode'] = 'basic';
	require_once(APPROOT.'/application/loginwebpage.class.inc.php');
	LoginWebPage::DoLogin(); // Check user rights and prompt if needed

	$sCSVData = utils::ReadPostedParam('csvdata');
}


try
{
	//////////////////////////////////////////////////
	//
	// Read parameters
	//
	$sClass = ReadMandatoryParam($oP, 'class');
	$sSep = ReadParam($oP, 'separator');
	$sQualifier = ReadParam($oP, 'qualifier');
	$sCharSet = ReadParam($oP, 'charset');
	$sDateFormat = ReadParam($oP, 'date_format');
	$sOutput = ReadParam($oP, 'output');
//	$sReportLevel = ReadParam($oP, 'reportlevel');
	$sReconcKeys = ReadParam($oP, 'reconciliationkeys');
	$sSimulate = ReadParam($oP, 'simulate');
	$sComment = ReadParam($oP, 'comment');

	//////////////////////////////////////////////////
	//
	// Check parameters format/consistency
	//
	if (strlen($sCSVData) == 0)
	{
		throw new ExchangeException("Missing data - at least one line is expected");
	}

	if (!MetaModel::IsValidClass($sClass))
	{
		throw new BulkLoadException("Unknown class: '$sClass'");
	}

	if (strlen($sSep) > 1)
	{
		throw new BulkLoadException("Separator is limited to one character, found '$sSep'");
	}

	if (strlen($sQualifier) > 1)
	{
		throw new BulkLoadException("Text qualifier is limited to one character, found '$sQualifier'");
	}

	if (!in_array($sOutput, array('retcode', 'summary', 'details')))
	{
		throw new BulkLoadException("Unknown output format: '$sOutput'");
	}

	if (strlen($sDateFormat) == 0)
	{
		$sDateFormat = null;
	}
	
/*
	$aReportLevels = explode('|', $sReportLevel);
	foreach($aReportLevels as $sLevel)
	{
		if (!in_array($sLevel, explode('|', 'errors|warnings|created|changed|unchanged')))
		{
			throw new BulkLoadException("Unknown level in reporting level: '$sLevel'");
		}
	}
*/

	if ($sSimulate == '1')
	{
		$bSimulate = true;
	}
	else
	{
		$bSimulate = false;
	}

	if (($sOutput == "summary") || ($sOutput == 'details'))
	{
		$oP->add_comment("Output format: ".$sOutput);
		$oP->add_comment("Class: ".$sClass);
		$oP->add_comment("Separator: ".$sSep);
		$oP->add_comment("Qualifier: ".$sQualifier);
		$oP->add_comment("Charset Encoding:".$sCharSet);
		if (strlen($sDateFormat) > 0)
		{
			$oP->add_comment("Date format: '$sDateFormat'");
		}
		else
		{
			$oP->add_comment("Date format: <none>");
		}
		$oP->add_comment("Data Size: ".strlen($sCSVData));
	}
	//////////////////////////////////////////////////
	//
	// Security
	//
	if (!UserRights::IsActionAllowed($sClass, UR_ACTION_BULK_MODIFY))
	{
		throw new SecurityException(Dict::Format('UI:Error:BulkModifyNotAllowedOn_Class', $sClass));
	}

	//////////////////////////////////////////////////
	//
	// Make translated column reference
	//
	// array of <LowercaseTranslatedName> => <ExtendedAttCode>
	//
	// Examples:
	//   'organization' => 'org_id'
	//   'organization->name' => 'org_id->name'
	//
	// Note: it may happen that an external field has the same label as the external key
	//       in that case, we consider that the external key has precedence
	//
	$aFriendlyToInternalAttCode = array();
	foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode => $oAttDef)
	{
	  	$sFriendlyName = strtolower(BulkChange::GetFriendlyAttCodeName($sClass, $sAttCode));
	  	if (!$oAttDef->IsExternalField() || !array_key_exists($sFriendlyName, $aFriendlyToInternalAttCode))
	  	{
		  	$aFriendlyToInternalAttCode[$sFriendlyName] = $sAttCode;
		}
	  	if ($oAttDef->IsExternalKey(EXTKEY_RELATIVE))
	  	{
	  		$sRemoteClass = $oAttDef->GetTargetClass();
			foreach(MetaModel::ListAttributeDefs($sRemoteClass) as $sRemoteAttCode => $oRemoteAttDef)
		  	{
	  			$sAttCodeEx = $sAttCode.'->'.$sRemoteAttCode;
	  			$sFriendlyName = strtolower(BulkChange::GetFriendlyAttCodeName($sClass, $sAttCodeEx));
		  		if (!array_key_exists($sFriendlyName, $aFriendlyToInternalAttCode))
		  		{
		  			$aFriendlyToInternalAttCode[$sFriendlyName] = $sAttCodeEx;
		  		}
		  	}
		}
   }
   
	//////////////////////////////////////////////////
	//
	// Parse first line, check attributes, analyse the request
	//
	if ($sCharSet == 'UTF-8')
	{
		$sUTF8Data = $sCSVData;		
	}
	else
	{
		$sUTF8Data = iconv($sCharSet, 'UTF-8//IGNORE//TRANSLIT', $sCSVData);
	}
	$oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier); 

	// Limitation: as the attribute list is in the first line, we can not match external key by a third-party attribute
	$aRawFieldList = $oCSVParser->ListFields();
	$iColCount = count($aRawFieldList);

	// Translate into internal names
	$aFieldList = array();
	foreach($aRawFieldList as $iFieldId => $sFieldName)
	{
		$sFieldName = trim($sFieldName);
		if (array_key_exists(strtolower($sFieldName), $aFriendlyToInternalAttCode))
		{
			$aFieldList[$iFieldId] = $aFriendlyToInternalAttCode[strtolower($sFieldName)];
		}
		else
		{
			$aFieldList[$iFieldId] = $sFieldName;
		}
	}	

	$aAttList = array();
	$aExtKeys = array();
	foreach($aFieldList as $iFieldId => $sFieldName)
	{
		$aMatches = array();
		if (preg_match('/^(.+)\*$/', $sFieldName, $aMatches))
		{
			// Ignore any trailing "star" (*) that simply indicates a mandatory field
			$sFieldName = $aMatches[1];
		}
		if (preg_match('/^(.+)->(.+)$/', trim($sFieldName), $aMatches))
		{
			// The column has been specified as "extkey->attcode"
			//
			$sExtKeyAttCode = $aMatches[1];
			$sRemoteAttCode = $aMatches[2];
			if (!MetaModel::IsValidAttCode($sClass, $sExtKeyAttCode))
			{
				throw new BulkLoadException("Unknown attribute '$sExtKeyAttCode' (class: '$sClass')");
			}
			$oAtt = MetaModel::GetAttributeDef($sClass, $sExtKeyAttCode);
			if (!$oAtt->IsExternalKey())
			{
				throw new BulkLoadException("Not an external key '$sExtKeyAttCode' (class: '$sClass')");
			}
			$sTargetClass = $oAtt->GetTargetClass();<commit_msg>Bug fix: properly process field names followed by a star (which simply indicates an important field in the CSV import)<commit_after><?php
// Copyright (C) 2010 Combodo SARL
//
//   This program is free software; you can redistribute it and/or modify
//   it under the terms of the GNU General Public License as published by
//   the Free Software Foundation; version 3 of the License.
//
//   This program is distributed in the hope that it will be useful,
//   but WITHOUT ANY WARRANTY; without even the implied warranty of
//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//   GNU General Public License for more details.
//
//   You should have received a copy of the GNU General Public License
//   along with this program; if not, write to the Free Software
//   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

/**
 * Import web service 
 *
 * @author      <NAME> <<EMAIL>>
 * @author      <NAME> <<EMAIL>>
 * @author      <NAME> <<EMAIL>>
 * @license     http://www.opensource.org/licenses/gpl-3.0.html LGPL
 */

//
// Known limitations
// - reconciliation is made on the first column
//
// Known issues
// - ALMOST impossible to troubleshoot when an externl key has a wrong value
// - no character escaping in the xml output (yes !?!?!)
// - not outputing xml when a wrong input is given (class, attribute names)
//

if (!defined('__DIR__')) define('__DIR__', dirname(__FILE__));
require_once(__DIR__.'/../approot.inc.php');
require_once(APPROOT.'/application/application.inc.php');
require_once(APPROOT.'/application/webpage.class.inc.php');
require_once(APPROOT.'/application/csvpage.class.inc.php');
require_once(APPROOT.'/application/clipage.class.inc.php');

require_once(APPROOT.'/application/startup.inc.php');

class BulkLoadException extends Exception
{
}

$aPageParams = array
(
	'auth_user' => array
	(
		'mandatory' => true,
		'modes' => 'cli',
		'default' => null,
		'description' => 'login (must have enough rights to create objects of the given class)',
	),
	'auth_pwd' => array
	(
		'mandatory' => true,
		'modes' => 'cli',
		'default' => null,
		'description' => 'password',
	),
	'class' => array
	(
		'mandatory' => true,
		'modes' => 'http,cli',
		'default' => null,
		'description' => 'class of loaded objects',
	),
	'csvdata' => array
	(
		'mandatory' => true,
		'modes' => 'http',
		'default' => null,
		'description' => 'data',
	),
	'csvfile' => array
	(
		'mandatory' => true,
		'modes' => 'cli',
		'default' => '',
		'description' => 'local data file, replaces csvdata if specified',
	),
	'charset' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => 'UTF-8',
		'description' => 'Character set encoding of the CSV data: UTF-8, ISO-8859-1, WINDOWS-1251, WINDOWS-1252, ISO-8859-15',
	),
	'date_format' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '',
		'description' => 'Input date format (used both for dates and datetimes) - Examples: %Y-%m-%d, %d/%m/%Y (Europe) - no transformation is applied if the argument is omitted',
	),
	'separator' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => ',',
		'description' => 'column separator in CSV data',
	),
	'qualifier' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '"',
		'description' => 'test qualifier in CSV data',
	),
	'output' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => 'summary',
		'description' => '[retcode] to return the count of lines in error, [summary] to return a concise report, [details] to get a detailed report (each line listed)',
	),
/*
	'reportlevel' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => 'errors|warnings|created|changed|unchanged',
		'description' => 'combination of flags to limit the detailed output',
	),
*/
	'reconciliationkeys' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '',
		'description' => 'name of the columns used to identify existing objects and update them, or create a new one',
	),
	'simulate' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '0',
		'description' => 'If set to 1, then the load will not be executed, but the expected report will be produced',
	),
	'comment' => array
	(
		'mandatory' => false,
		'modes' => 'http,cli',
		'default' => '',
		'description' => 'Comment to be added into the change log',
	),
);

function UsageAndExit($oP)
{
	global $aPageParams;
	$bModeCLI = utils::IsModeCLI();

	$oP->p("USAGE:\n");
	foreach($aPageParams as $sParam => $aParamData)
	{
		$aModes = explode(',', $aParamData['modes']);
		if ($bModeCLI)
		{
			if (in_array('cli', $aModes))
			{
				$sDesc = $aParamData['description'].', '.($aParamData['mandatory'] ? 'mandatory' : 'optional, defaults to ['.$aParamData['default'].']');
				$oP->p("$sParam = $sDesc");
			}
		}
		else
		{
			if (in_array('http', $aModes))
			{
				$sDesc = $aParamData['description'].', '.($aParamData['mandatory'] ? 'mandatory' : 'optional, defaults to ['.$aParamData['default'].']');
				$oP->p("$sParam = $sDesc");
			}
		}
	}
	$oP->output();
	exit;
}


function ReadParam($oP, $sParam)
{
	global $aPageParams;
	assert(isset($aPageParams[$sParam]));
	assert(!$aPageParams[$sParam]['mandatory']);
	$sValue = utils::ReadParam($sParam, $aPageParams[$sParam]['default'], true /* Allow CLI */);
	return trim($sValue);
}

function ReadMandatoryParam($oP, $sParam)
{
	global $aPageParams;
	assert(isset($aPageParams[$sParam]));
	assert($aPageParams[$sParam]['mandatory']);

	$sValue = utils::ReadParam($sParam, null, true /* Allow CLI */);
	if (is_null($sValue))
	{
		$oP->p("ERROR: Missing argument '$sParam'\n");
		UsageAndExit($oP);
	}
	return trim($sValue);
}

/////////////////////////////////
// Main program

if (utils::IsModeCLI())
{
	$oP = new CLIPage("iTop - Bulk import");
}
else
{
	$oP = new CSVPage("iTop - Bulk import");
}

try
{
	utils::UseParamFile();
}
catch(Exception $e)
{
	$oP->p("Error: ".$e->GetMessage());
	$oP->output();
	exit -2;
}

if (utils::IsModeCLI())
{
	// Next steps:
	//   specific arguments: 'csvfile'
	//   
	$sAuthUser = ReadMandatoryParam($oP, 'auth_user');
	$sAuthPwd = ReadMandatoryParam($oP, 'auth_pwd');
	$sCsvFile = ReadMandatoryParam($oP, 'csvfile');
	if (UserRights::CheckCredentials($sAuthUser, $sAuthPwd))
	{
		UserRights::Login($sAuthUser); // Login & set the user's language
	}
	else
	{
		$oP->p("Access restricted or wrong credentials ('$sAuthUser')");
		$oP->output();
		exit -1;
	}

	if (!is_readable($sCsvFile))
	{
		$oP->p("Input file could not be found or could not be read: '$sCsvFile'");
		$oP->output();
		exit -1;
	}
	$sCSVData = file_get_contents($sCsvFile);

}
else
{
	$_SESSION['login_mode'] = 'basic';
	require_once(APPROOT.'/application/loginwebpage.class.inc.php');
	LoginWebPage::DoLogin(); // Check user rights and prompt if needed

	$sCSVData = utils::ReadPostedParam('csvdata');
}


try
{
	//////////////////////////////////////////////////
	//
	// Read parameters
	//
	$sClass = ReadMandatoryParam($oP, 'class');
	$sSep = ReadParam($oP, 'separator');
	$sQualifier = ReadParam($oP, 'qualifier');
	$sCharSet = ReadParam($oP, 'charset');
	$sDateFormat = ReadParam($oP, 'date_format');
	$sOutput = ReadParam($oP, 'output');
//	$sReportLevel = ReadParam($oP, 'reportlevel');
	$sReconcKeys = ReadParam($oP, 'reconciliationkeys');
	$sSimulate = ReadParam($oP, 'simulate');
	$sComment = ReadParam($oP, 'comment');

	//////////////////////////////////////////////////
	//
	// Check parameters format/consistency
	//
	if (strlen($sCSVData) == 0)
	{
		throw new ExchangeException("Missing data - at least one line is expected");
	}

	if (!MetaModel::IsValidClass($sClass))
	{
		throw new BulkLoadException("Unknown class: '$sClass'");
	}

	if (strlen($sSep) > 1)
	{
		throw new BulkLoadException("Separator is limited to one character, found '$sSep'");
	}

	if (strlen($sQualifier) > 1)
	{
		throw new BulkLoadException("Text qualifier is limited to one character, found '$sQualifier'");
	}

	if (!in_array($sOutput, array('retcode', 'summary', 'details')))
	{
		throw new BulkLoadException("Unknown output format: '$sOutput'");
	}

	if (strlen($sDateFormat) == 0)
	{
		$sDateFormat = null;
	}
	
/*
	$aReportLevels = explode('|', $sReportLevel);
	foreach($aReportLevels as $sLevel)
	{
		if (!in_array($sLevel, explode('|', 'errors|warnings|created|changed|unchanged')))
		{
			throw new BulkLoadException("Unknown level in reporting level: '$sLevel'");
		}
	}
*/

	if ($sSimulate == '1')
	{
		$bSimulate = true;
	}
	else
	{
		$bSimulate = false;
	}

	if (($sOutput == "summary") || ($sOutput == 'details'))
	{
		$oP->add_comment("Output format: ".$sOutput);
		$oP->add_comment("Class: ".$sClass);
		$oP->add_comment("Separator: ".$sSep);
		$oP->add_comment("Qualifier: ".$sQualifier);
		$oP->add_comment("Charset Encoding:".$sCharSet);
		if (strlen($sDateFormat) > 0)
		{
			$oP->add_comment("Date format: '$sDateFormat'");
		}
		else
		{
			$oP->add_comment("Date format: <none>");
		}
		$oP->add_comment("Data Size: ".strlen($sCSVData));
	}
	//////////////////////////////////////////////////
	//
	// Security
	//
	if (!UserRights::IsActionAllowed($sClass, UR_ACTION_BULK_MODIFY))
	{
		throw new SecurityException(Dict::Format('UI:Error:BulkModifyNotAllowedOn_Class', $sClass));
	}

	//////////////////////////////////////////////////
	//
	// Make translated column reference
	//
	// array of <LowercaseTranslatedName> => <ExtendedAttCode>
	//
	// Examples:
	//   'organization' => 'org_id'
	//   'organization->name' => 'org_id->name'
	//
	// Note: it may happen that an external field has the same label as the external key
	//       in that case, we consider that the external key has precedence
	//
	$aFriendlyToInternalAttCode = array();
	foreach(MetaModel::ListAttributeDefs($sClass) as $sAttCode => $oAttDef)
	{
	  	$sFriendlyName = strtolower(BulkChange::GetFriendlyAttCodeName($sClass, $sAttCode));
	  	if (!$oAttDef->IsExternalField() || !array_key_exists($sFriendlyName, $aFriendlyToInternalAttCode))
	  	{
		  	$aFriendlyToInternalAttCode[$sFriendlyName] = $sAttCode;
		}
	  	if ($oAttDef->IsExternalKey(EXTKEY_RELATIVE))
	  	{
	  		$sRemoteClass = $oAttDef->GetTargetClass();
			foreach(MetaModel::ListAttributeDefs($sRemoteClass) as $sRemoteAttCode => $oRemoteAttDef)
		  	{
	  			$sAttCodeEx = $sAttCode.'->'.$sRemoteAttCode;
	  			$sFriendlyName = strtolower(BulkChange::GetFriendlyAttCodeName($sClass, $sAttCodeEx));
		  		if (!array_key_exists($sFriendlyName, $aFriendlyToInternalAttCode))
		  		{
		  			$aFriendlyToInternalAttCode[$sFriendlyName] = $sAttCodeEx;
		  		}
		  	}
		}
   }
   
	//////////////////////////////////////////////////
	//
	// Parse first line, check attributes, analyse the request
	//
	if ($sCharSet == 'UTF-8')
	{
		$sUTF8Data = $sCSVData;		
	}
	else
	{
		$sUTF8Data = iconv($sCharSet, 'UTF-8//IGNORE//TRANSLIT', $sCSVData);
	}
	$oCSVParser = new CSVParser($sUTF8Data, $sSep, $sQualifier); 

	// Limitation: as the attribute list is in the first line, we can not match external key by a third-party attribute
	$aRawFieldList = $oCSVParser->ListFields();
	$iColCount = count($aRawFieldList);

	// Translate into internal names
	$aFieldList = array();
	foreach($aRawFieldList as $iFieldId => $sFieldName)
	{
		$sFieldName = trim($sFieldName);
		$aMatches = array();
		if (preg_match('/^(.+)\*$/', $sFieldName, $aMatches))
		{
			// Ignore any trailing "star" (*) that simply indicates a mandatory field
			$sFieldName = $aMatches[1];
		}
		if (array_key_exists(strtolower($sFieldName), $aFriendlyToInternalAttCode))
		{
			$aFieldList[$iFieldId] = $aFriendlyToInternalAttCode[strtolower($sFieldName)];
		}
		else
		{
			$aFieldList[$iFieldId] = $sFieldName;
		}
	}	

	$aAttList = array();
	$aExtKeys = array();
	foreach($aFieldList as $iFieldId => $sFieldName)
	{
		$aMatches = array();
		if (preg_match('/^(.+)->(.+)$/', trim($sFieldName), $aMatches))
		{
			// The column has been specified as "extkey->attcode"
			//
			$sExtKeyAttCode = $aMatches[1];
			$sRemoteAttCode = $aMatches[2];
			if (!MetaModel::IsValidAttCode($sClass, $sExtKeyAttCode))
			{
				throw new BulkLoadException("Unknown attribute '$sExtKeyAttCode' (class: '$sClass')");
			}
			$oAtt = MetaModel::GetAttributeDef($sClass, $sExtKeyAttCode);
			if (!$oAtt->IsExternalKey())
			{
				throw new BulkLoadException("Not an external key '$sExtKeyAttCode' (class: '$sClass')");
			}
			$sTargetClass = $oAtt->GetTargetClass();
{'fraction_non_alphanumeric': 0.19772782209330433, 'fraction_numerical': 0.004914994762710499, 'mean_word_length': 3.7165114953448604, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 251, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7450481', 'n_tokens_mistral': 9403, 'n_tokens_neox': 8856, 'n_words': 2004}