<commit_before><?php

/*
 * (c) Xidea <NAME> <<EMAIL>>
 *
 * For the full copyright and license intypeion, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Xidea\Component\Dataflow\Import;

use Xidea\Component\Dataflow\Model\ImportInterface;

/**
 * @author <NAME> <<EMAIL>>
 */
abstract class AbstractService implements ServiceInterface
{    
    /*
     * @var array
     */
    protected $fields;
    
    /*
     * @var array
     */
    protected $options;
    
    /*
     * @var string
     */
    protected $idFieldName = null;
    
    /*
     * @var array
     */
    protected $data;
    
    /**
     * @inheritDoc
     */
    public function getFields()
    {
        return $this->fields;
    }
    
    /**
     * @inheritDoc
     */
    public function getReaderFields()
    {
        $fields = [];
        
        foreach($this->getFields() as $name => $config) {
            $fields[] = isset($config['alias']) ? $config['alias'] : $name;
        }
        return $fields;
    }
    
    /**
     * @inheritDoc
     */
    public function getIdFieldName()
    {
        if($this->idFieldName !== null) {
            return $this->idFieldName;
        }
        
        foreach($this->getFields() as $name => $config) {
            if(isset($config['id']) && $config['id']) {
                $this->idFieldName = $name;
            }
        }
        
        return $this->idFieldName;
    }
    
    /**
     * @inheritDoc
     */
    public function configure(array $options = [], array $fields = [])
    {
        $this->configureOptions($options);
        $this->configureFields($fields);
    }
    
    /**
     * @inheritDoc
     */
    public function configureFields(array $fields = [])
    {
        if(empty($fields))
            return;
        
        $results = [];
        foreach($fields as $name => $config) {
            if(isset($this->fields[$name])) {
                $results[$name] = array_merge($this->fields[$name], $config);
            }
        }
        $this->fields = $results;
    }
    
    /**
     * @inheritDoc
     */
    public function configureOptions(array $options = [])
    {
        $this->options = array_merge([
           'behavior' => ImportInterface::BEHAVIOR_INSERT,
           'batch_size' => 50
        ], $options);
    }
    
    /**
     * @inheritDoc
     */
    public function add(array $record)
    {
        $record = $this->convert($record);
        if($filteredRecord = $this->filter($record)) {
            $this->data[] = $filteredRecord;
        }
    }
    
    /**
     * @inheritDoc
     */
    public function filter(array $record)
    {
        return $record;
    }
    
    /**
     * @inheritDoc
     */
    public function convert(array $record)
    {
        $idField = $this->getIdFieldName();
        
        $result = [];
        foreach($data as $record) {
            foreach($this->fields as $name => $config) {
                $id = isset($config[$idField]) ? $config[$idField] : $name;
                if(array_key_exists($name, $record)) {
                    $result[$id] = $record[$name];
                } elseif(isset($config['alias']) && array_key_exists($config['alias'], $record)) {
                    $result[$id] = $record[$config['alias']];
                }
            }
        }
        
        return $result;
    }
    
    /**
     * @inheritDoc
     */
    public function import(\Closure $callback = null)
    {
        try {
            switch($this->options['behavior']) {<commit_msg>Updated import service<commit_after><?php

/*
 * (c) Xidea <NAME> <<EMAIL>>
 *
 * For the full copyright and license intypeion, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Xidea\Component\Dataflow\Import;

use Xidea\Component\Dataflow\Model\ImportInterface;

/**
 * @author <NAME> <<EMAIL>>
 */
abstract class AbstractService implements ServiceInterface
{    
    /*
     * @var array
     */
    protected $fields;
    
    /*
     * @var array
     */
    protected $options;
    
    /*
     * @var string
     */
    protected $idFieldName = null;
    
    /*
     * @var array
     */
    protected $data;
    
    /**
     * @inheritDoc
     */
    public function getFields()
    {
        return $this->fields;
    }
    
    /**
     * @inheritDoc
     */
    public function getReaderFields()
    {
        $fields = [];
        
        foreach($this->getFields() as $name => $config) {
            $fields[] = isset($config['alias']) ? $config['alias'] : $name;
        }
        return $fields;
    }
    
    /**
     * @inheritDoc
     */
    public function getIdFieldName()
    {
        if($this->idFieldName !== null) {
            return $this->idFieldName;
        }
        
        foreach($this->getFields() as $name => $config) {
            if(isset($config['id']) && $config['id']) {
                $this->idFieldName = $name;
            }
        }
        
        return $this->idFieldName;
    }
    
    /**
     * @inheritDoc
     */
    public function configure(array $options = [], array $fields = [])
    {
        $this->configureOptions($options);
        $this->configureFields($fields);
    }
    
    /**
     * @inheritDoc
     */
    public function configureFields(array $fields = [])
    {
        if(empty($fields))
            return;
        
        $results = [];
        foreach($fields as $name => $config) {
            if(isset($this->fields[$name])) {
                $results[$name] = array_merge($this->fields[$name], $config);
            }
        }
        $this->fields = $results;
    }
    
    /**
     * @inheritDoc
     */
    public function configureOptions(array $options = [])
    {
        $this->options = array_merge([
           'behavior' => ImportInterface::BEHAVIOR_INSERT,
           'batch_size' => 50
        ], $options);
    }
    
    /**
     * @inheritDoc
     */
    public function add(array $record)
    {
        $record = $this->convert($record);
        if($filteredRecord = $this->filter($record)) {
            $id = isset($filteredRecord[$this->getIdFieldName()]) ? $filteredRecord[$this->getIdFieldName()] : null;
            if($id) {
                $this->data[$id] = $filteredRecord;
            }
        }
    }
    
    /**
     * @inheritDoc
     */
    public function filter(array $record)
    {
        return $record;
    }
    
    /**
     * @inheritDoc
     */
    public function convert(array $record)
    {
        $result = [];
        foreach($this->fields as $name => $config) {
            if(array_key_exists($name, $record)) {
                $result[$name] = $record[$name];
            } elseif(isset($config['alias']) && array_key_exists($config['alias'], $record)) {
                $result[$name] = $record[$config['alias']];
            }
        }
        
        return $result;
    }
    
    /**
     * @inheritDoc
     */
    public function import(\Closure $callback = null)
    {
        try {
            switch($this->options['behavior']) {
{'fraction_non_alphanumeric': 0.1538135593220339, 'fraction_numerical': 0.0005649717514124294, 'mean_word_length': 1.6264836795252227, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '209372', 'n_tokens_mistral': 2072, 'n_tokens_neox': 2052, 'n_words': 372}