<commit_before><?php

/**
 * @package         Billing
 * @copyright       Copyright (C) 2012-2018 S.D.O.C. LTD. All rights reserved.
 * @license         GNU Affero General Public License Version 3; see LICENSE.txt
 */

/**
 * Billing abstract exporter class
 * Exporter class should handle:
 *   1. fetching data (from DB) according to configured query
 *   2. mapping the data according to configuration
 *   3. export the data (to a file or any other export option)
 *   4. send the exported data to a configured location (FTP, SFTP, etc...)
 * 
 * @package  Billing
 * @since    5.9
 */
class Billrun_Exporter extends Billrun_Generator_File {

    use Billrun_Traits_ConditionsCheck;

    /**
     * Type of exporter
     *
     * @var string
     */
    static protected $type = 'exporter';

    const SEQUENCE_NUM_INIT = 1;
    const DEFAULT_FILENAME = 'EXPORT_[[param1]]_[[param2]].CSV';
    const DEFAULT_FILENAME_PARMS = [
        [
            "param" => "param1",
            "type" => "autoinc",
            "min_value" => 1,
            "date_group" => "e",
            "padding" => [
                "character" => "0",
                "length" => 6,
                "direction" => "left"
            ],
            "value" => "now"
        ],
        [
            "param" => "param2",
            "type" => "date",
            "format" => "YmdHis",
            "value" => "now"
        ]
    ];

    /**
     * the name of the log collection in the DB
     * @var string
     */
    protected $logCollection = null;

    /**
     * sequence number unique for the specific export
     * @var string
     */
    protected $sequenceNum = null;

    /**
     * datetime the export has started
     * @var unixtimestamp 
     */
    protected $exportTime = null;

    /**
     * unique stamp for the export
     * @var string
     */
    protected $exportStamp = null;

    /**
     * unique stamp for log collection
     * @var string
     */
    protected $logStamp = array();

    /**
     * collection name (DB) from which data should be fetched
     * @var string
     */
    protected $collection = null;

    /**
     * query by which data should be fetched from DB
     * @var array
     */
    protected $query = array();
    
    /**
     * maximum number of records to fetch (in order to allow pagination)
     *
     * @var int
     */
    protected $limit = null;
    
    /**
     * was the file moved (uploaded) successfully
     *
     * @var bool
     */
    protected $moved = false;

    public function __construct($options = array()) {
        parent::__construct($options);
        $this->exportTime = time();
        $this->exportStamp = $this->getExportStamp();
        $this->query = $this->getFiltrationQuery();
        $this->limit = $this->getLimit();
        $this->logCollection = Billrun_Factory::db()->logCollection();
    }

    protected function getLinkedEntityData($entity, $params, $field) {
        switch ($entity) {
            case 'line':
                $value = Billrun_Util::getIn($params, $field, '');
                return $value;
            default:
                $message = "Unknown entity: " . $entity . ", as 'linked entity' in the config.";
                Billrun_Factory::log($message, Zend_Log::ERR);
        }
    }

    /**
     * get stamp for the current run of the exporter
     */
    protected function getExportStamp() {
        if (is_null($this->exportStamp)) {
            $this->exportStamp = uniqid();
        }
        return $this->exportStamp;
    }

    /**
     * gets collection to load data from DB
     * 
     * @return Mongodloid_Collection
     */
    protected function getCollection() {
        if (is_null($this->collection)) {
            $collectionName = $this->getCollectionName();
            $this->collection = Billrun_Factory::db()->{"{$collectionName}Collection"}();
        }
        return $this->collection;
    }

    /**
     * get query to load data from the DB
     */
    protected function getFiltrationQuery() {
        $querySettings = $this->config['filtration'][0]; // TODO: currenly, supporting 1 query might support more in the future
        $query = $this->getConditionsQuery($querySettings['query']);
        if ($query === true) {
            return [];
        }
        if (isset($querySettings['time_range'])) {
            $timeRange = $querySettings['time_range'];
            if (isset($querySettings['time_range_hour'])) {
                $hour = $querySettings['time_range_hour'];
                $endTime = strtotime($hour, $this->exportTime);
                $startTime = strtotime($timeRange . ' ' . $hour, $endTime);
            } else {
                $endTime = $this->exportTime;
                $startTime = strtotime($timeRange, $endTime);
            }
            $query['urt'] = array(
                '$gte' => new MongoDate($startTime),
                '$lt' => new MongoDate($endTime),
            );
        }
        return $query;
    }

    /**
     * get limitation for rows to be exported
     *
     * @return int
     */
    protected function getLimit() {
        $querySettings = $this->config['filtration'][0]; // TODO: currenly, supporting 1 query might support more in the future
        return $querySettings['limit'] ?? null;
    }

    /**
     * general function to handle the export
     *
     * @return array list of lines exported
     */
    public function generate() {
        Billrun_Factory::log()->log("Billrun_Exporter::generate - starting to generate", Zend_Log::INFO);
        Billrun_Factory::dispatcher()->trigger('beforeExport', array($this));
        $this->beforeExport();
        $className = $this->getGeneratorClassName();
        $generatorOptions = $this->buildGeneratorOptions();
        $this->createLogDB($this->getLogStamp());
        $this->fileGenerator = new $className($generatorOptions);
        $this->fileGenerator->generate();
        $transactionCounter = $this->fileGenerator->getTransactionsCounter();
        Billrun_Factory::log("Exported " . $transactionCounter . " lines from " . $this->getCollectionName() . " collection");
    }

    /**
     * gets record type according to configuration mapping
     * 
     * @return string
     */
    protected function getRecordType($row) {
        foreach (Billrun_Util::getIn($this->config, 'generator.record_type_mapping', array()) as $recordTypeMapping) {
            foreach ($recordTypeMapping['conditions'] as $condition) {
                $query = $this->getConditionQuery($row, $condition);
                if (!$this->isConditionMeet($row, $query)) {
                    continue 2;
                }
            }
            return $recordTypeMapping['record_type'];
        }
        
        Billrun_Factory::log()->log("Billrun_Exporter::getRecordType - Cannot get record type for line {$row['stamp']}", Zend_Log::ERR);
        return '';
    }

    /**
     * translate row to the format it should be exported
     * 
     * @param array $row
     * @return array
     */
    protected function getRecordData($row) {
        Billrun_Factory::dispatcher()->trigger('ExportBeforeGetRecordData', array(&$row, $this));
        $recordType = $this->getRecordType($row);
        $ret = $this->getDataLine($row, $recordType);
        Billrun_Factory::dispatcher()->trigger('ExportAfterGetRecordData', array(&$row, &$ret, $this));
        return $ret;
    }

    /**
     * get rows to be exported
     * 
     * @return array
     */
    protected function loadRows() {
        Billrun_Factory::log()->log("Billrun_Exporter::loadRows - starting to load rows", Zend_Log::INFO);
        $collection = $this->getCollection();
        Billrun_Factory::dispatcher()->trigger('ExportBeforeLoadRows', array(&$this->query, $collection, $this));
        $rows = $collection->query($this->query)
            ->cursor()
            ->hint(['stamp' => 1])
            ->timeout(Billrun_Factory::config()->getConfigValue('db.long_queries_timeout', 10800000));
        $data = array();
        $count = 0;
        foreach ($rows as $row) {
            Billrun_Factory::log()->log("start getting data for row {$count} with stamp {$row['stamp']}", Zend_Log::DEBUG);
            $rawRow = $row->getRawData();
            $this->rowsStamps[] = $rawRow['stamp'];
            $data[] = $this->getRecordData($rawRow);
            Billrun_Factory::log()->log("done getting data for row {$count} with stamp {$row['stamp']}", Zend_Log::DEBUG);
            $count++;
        }
        Billrun_Factory::dispatcher()->trigger('ExportAfterLoadRows', array(&$this->rowsStamps, &$this->rowsToExport, $this));
        Billrun_Factory::log()->log("Billrun_Exporter::loadRows - done", Zend_Log::INFO);
        return $data;
    }

    /**
     * method to log the export process
     */
    protected function logDB($stamp, $data) {
        if (empty($stamp)) {
            Billrun_Factory::log()->log("Billrun_Exporter::logDB - got export with empty stamp. data: " . print_R($data, 1), Zend_Log::NOTICE);
            return false;
        }
        $log = Billrun_Factory::db()->logCollection();
        Billrun_Factory::dispatcher()->trigger('beforeLogExport', array(&$data, $stamp, $this));

        $query = array(
            'stamp' => $stamp,
            'source' => 'export',
            'type' => static::$type,<commit_msg>BRCD-3252: prevent exporting the same line more than once (to duplications caused by mongo sync process)<commit_after><?php

/**
 * @package         Billing
 * @copyright       Copyright (C) 2012-2018 S.D.O.C. LTD. All rights reserved.
 * @license         GNU Affero General Public License Version 3; see LICENSE.txt
 */

/**
 * Billing abstract exporter class
 * Exporter class should handle:
 *   1. fetching data (from DB) according to configured query
 *   2. mapping the data according to configuration
 *   3. export the data (to a file or any other export option)
 *   4. send the exported data to a configured location (FTP, SFTP, etc...)
 * 
 * @package  Billing
 * @since    5.9
 */
class Billrun_Exporter extends Billrun_Generator_File {

    use Billrun_Traits_ConditionsCheck;

    /**
     * Type of exporter
     *
     * @var string
     */
    static protected $type = 'exporter';

    const SEQUENCE_NUM_INIT = 1;
    const DEFAULT_FILENAME = 'EXPORT_[[param1]]_[[param2]].CSV';
    const DEFAULT_FILENAME_PARMS = [
        [
            "param" => "param1",
            "type" => "autoinc",
            "min_value" => 1,
            "date_group" => "e",
            "padding" => [
                "character" => "0",
                "length" => 6,
                "direction" => "left"
            ],
            "value" => "now"
        ],
        [
            "param" => "param2",
            "type" => "date",
            "format" => "YmdHis",
            "value" => "now"
        ]
    ];

    /**
     * the name of the log collection in the DB
     * @var string
     */
    protected $logCollection = null;

    /**
     * sequence number unique for the specific export
     * @var string
     */
    protected $sequenceNum = null;

    /**
     * datetime the export has started
     * @var unixtimestamp 
     */
    protected $exportTime = null;

    /**
     * unique stamp for the export
     * @var string
     */
    protected $exportStamp = null;

    /**
     * unique stamp for log collection
     * @var string
     */
    protected $logStamp = array();

    /**
     * collection name (DB) from which data should be fetched
     * @var string
     */
    protected $collection = null;

    /**
     * query by which data should be fetched from DB
     * @var array
     */
    protected $query = array();
    
    /**
     * maximum number of records to fetch (in order to allow pagination)
     *
     * @var int
     */
    protected $limit = null;
    
    /**
     * was the file moved (uploaded) successfully
     *
     * @var bool
     */
    protected $moved = false;

    public function __construct($options = array()) {
        parent::__construct($options);
        $this->exportTime = time();
        $this->exportStamp = $this->getExportStamp();
        $this->query = $this->getFiltrationQuery();
        $this->limit = $this->getLimit();
        $this->logCollection = Billrun_Factory::db()->logCollection();
    }

    protected function getLinkedEntityData($entity, $params, $field) {
        switch ($entity) {
            case 'line':
                $value = Billrun_Util::getIn($params, $field, '');
                return $value;
            default:
                $message = "Unknown entity: " . $entity . ", as 'linked entity' in the config.";
                Billrun_Factory::log($message, Zend_Log::ERR);
        }
    }

    /**
     * get stamp for the current run of the exporter
     */
    protected function getExportStamp() {
        if (is_null($this->exportStamp)) {
            $this->exportStamp = uniqid();
        }
        return $this->exportStamp;
    }

    /**
     * gets collection to load data from DB
     * 
     * @return Mongodloid_Collection
     */
    protected function getCollection() {
        if (is_null($this->collection)) {
            $collectionName = $this->getCollectionName();
            $this->collection = Billrun_Factory::db()->{"{$collectionName}Collection"}();
        }
        return $this->collection;
    }

    /**
     * get query to load data from the DB
     */
    protected function getFiltrationQuery() {
        $querySettings = $this->config['filtration'][0]; // TODO: currenly, supporting 1 query might support more in the future
        $query = $this->getConditionsQuery($querySettings['query']);
        if ($query === true) {
            return [];
        }
        if (isset($querySettings['time_range'])) {
            $timeRange = $querySettings['time_range'];
            if (isset($querySettings['time_range_hour'])) {
                $hour = $querySettings['time_range_hour'];
                $endTime = strtotime($hour, $this->exportTime);
                $startTime = strtotime($timeRange . ' ' . $hour, $endTime);
            } else {
                $endTime = $this->exportTime;
                $startTime = strtotime($timeRange, $endTime);
            }
            $query['urt'] = array(
                '$gte' => new MongoDate($startTime),
                '$lt' => new MongoDate($endTime),
            );
        }
        return $query;
    }

    /**
     * get limitation for rows to be exported
     *
     * @return int
     */
    protected function getLimit() {
        $querySettings = $this->config['filtration'][0]; // TODO: currenly, supporting 1 query might support more in the future
        return $querySettings['limit'] ?? null;
    }

    /**
     * general function to handle the export
     *
     * @return array list of lines exported
     */
    public function generate() {
        Billrun_Factory::log()->log("Billrun_Exporter::generate - starting to generate", Zend_Log::INFO);
        Billrun_Factory::dispatcher()->trigger('beforeExport', array($this));
        $this->beforeExport();
        $className = $this->getGeneratorClassName();
        $generatorOptions = $this->buildGeneratorOptions();
        $this->createLogDB($this->getLogStamp());
        $this->fileGenerator = new $className($generatorOptions);
        $this->fileGenerator->generate();
        $transactionCounter = $this->fileGenerator->getTransactionsCounter();
        Billrun_Factory::log("Exported " . $transactionCounter . " lines from " . $this->getCollectionName() . " collection");
    }

    /**
     * gets record type according to configuration mapping
     * 
     * @return string
     */
    protected function getRecordType($row) {
        foreach (Billrun_Util::getIn($this->config, 'generator.record_type_mapping', array()) as $recordTypeMapping) {
            foreach ($recordTypeMapping['conditions'] as $condition) {
                $query = $this->getConditionQuery($row, $condition);
                if (!$this->isConditionMeet($row, $query)) {
                    continue 2;
                }
            }
            return $recordTypeMapping['record_type'];
        }
        
        Billrun_Factory::log()->log("Billrun_Exporter::getRecordType - Cannot get record type for line {$row['stamp']}", Zend_Log::ERR);
        return '';
    }

    /**
     * translate row to the format it should be exported
     * 
     * @param array $row
     * @return array
     */
    protected function getRecordData($row) {
        Billrun_Factory::dispatcher()->trigger('ExportBeforeGetRecordData', array(&$row, $this));
        $recordType = $this->getRecordType($row);
        $ret = $this->getDataLine($row, $recordType);
        Billrun_Factory::dispatcher()->trigger('ExportAfterGetRecordData', array(&$row, &$ret, $this));
        return $ret;
    }

    /**
     * get rows to be exported
     * 
     * @return array
     */
    protected function loadRows() {
        Billrun_Factory::log()->log("Billrun_Exporter::loadRows - starting to load rows", Zend_Log::INFO);
        $collection = $this->getCollection();
        Billrun_Factory::dispatcher()->trigger('ExportBeforeLoadRows', array(&$this->query, $collection, $this));
        $rows = $collection->query($this->query)
            ->cursor()
            ->hint(['stamp' => 1])
            ->timeout(Billrun_Factory::config()->getConfigValue('db.long_queries_timeout', 10800000));
        $data = array();
        $count = 0;
        foreach ($rows as $row) {
            if (in_array($row['stamp'], $this->rowsStamps)) {
                Billrun_Factory::log()->log("Skipping stamp {$row['stamp']} as it was already loaded", Zend_Log::DEBUG);
                continue;
            }
            
            Billrun_Factory::log()->log("start getting data for row {$count} with stamp {$row['stamp']}", Zend_Log::DEBUG);
            $rawRow = $row->getRawData();
            $this->rowsStamps[] = $rawRow['stamp'];
            $data[] = $this->getRecordData($rawRow);
            Billrun_Factory::log()->log("done getting data for row {$count} with stamp {$row['stamp']}", Zend_Log::DEBUG);
            $count++;
        }
        Billrun_Factory::dispatcher()->trigger('ExportAfterLoadRows', array(&$this->rowsStamps, &$this->rowsToExport, $this));
        Billrun_Factory::log()->log("Billrun_Exporter::loadRows - done", Zend_Log::INFO);
        return $data;
    }

    /**
     * method to log the export process
     */
    protected function logDB($stamp, $data) {
        if (empty($stamp)) {
            Billrun_Factory::log()->log("Billrun_Exporter::logDB - got export with empty stamp. data: " . print_R($data, 1), Zend_Log::NOTICE);
            return false;
        }
        $log = Billrun_Factory::db()->logCollection();
        Billrun_Factory::dispatcher()->trigger('beforeLogExport', array(&$data, $stamp, $this));

        $query = array(
            'stamp' => $stamp,
            'source' => 'export',
            'type' => static::$type,
{'fraction_non_alphanumeric': 0.14281212248542466, 'fraction_numerical': 0.004306948894374704, 'mean_word_length': 2.194094950511659, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 173, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363605', 'n_tokens_mistral': 5414, 'n_tokens_neox': 5226, 'n_words': 1281}