<commit_before><?php
namespace Craft;

class ImportService extends BaseApplicationComponent 
{

    public $log = array();

    public function columns($file) 
    {
                
        // Open CSV file       
        $data = $this->_open($file);
        
        // Return only column names
        return array_shift($data);
       
    }
    
    public function data($file) 
    {
        
        // Open CSV file
        $data = $this->_open($file);
        
        // Skip first row
        array_shift($data);

        // Return all data
        return $data;
    
    }
    
    public function row($row, $data, $settings) 
    {
    
        // Get max power
        craft()->config->maxPowerCaptain();
        
        // See if map and data match (could not be due to malformed csv)
        if(count($settings['map']) != count($data)) {
        
            // Log errors when unsuccessful
            $this->log[$row] = craft()->import_history->log($settings['history'], $row, array(array(Craft::t('Columns and data did not match, could be due to malformed CSV row.'))));            
            return;
        
        }
        
        // Check what service we're gonna need
        $service = 'import_' . strtolower($settings['type']);
            
        // Map data to fields
        $fields = array_combine($settings['map'], $data);
        
        // If set, remove fields that will not be imported
        if(isset($fields['dont'])) {
            unset($fields['dont']);
        }
        
        // Set up a model to save according to element type
        $entry = craft()->$service->setModel($settings);
        
        // If unique is non-empty array, we're replacing or deleting
        if(is_array($settings['unique']) && count($settings['unique']) > 1) {
            
            // Set criteria according to elementtype
            $criteria = craft()->$service->setCriteria($settings);
                        
            // Set up criteria model for matching        
            $cmodel = array();    
            foreach($settings['map'] as $key => $value) {
                if(isset($criteria->$settings['map'][$key]) && isset($settings['unique'][$key]) && intval($settings['unique'][$key]) == 1 && !empty($fields[$value])) {
                    $criteria->$settings['map'][$key] = $cmodel[$settings['map'][$key]] = $fields[$value];
                }
            }
                        
            // If there's a match...
            if(count($cmodel) && $criteria->count()) {
                
                // If we're deleting
                if($settings['behavior'] == ImportModel::BehaviorDelete) {
                
                    // Get elements to delete
                    $elements = $criteria->find();
                
                    // Fire an 'onBeforeImportDelete' event
                    Craft::import('plugins.import.events.BeforeImportDeleteEvent');
                    $event = new BeforeImportDeleteEvent($this, array('elements' => $elements));
                    $this->onBeforeImportDelete($event);
                    
                    // Give event the chance to blow off deletion
                    if($event->proceed) {
                    
                        try {
                                
                            // Do it
                            if(!craft()->$service->delete($elements)) {
                            
                                // Log errors when unsuccessful
                                $this->log[$row] = craft()->import_history->log($settings['history'], $row, array(array(Craft::t('Something went wrong while deleting this row.'))));            
                                    
                            }
                            
                        } catch(Exception $e) {
                        
                            // Something went terribly wrong, assume its only this row
                            $this->log[$row] = craft()->import_history->log($settings['history'], $row, array('exception' => array($e->getMessage())));
                        
                        }
                        
                    }
                    
                    // Skip rest and continue
                    return;
                    
                } else {
                
                    // Fill new EntryModel with match
                    $entry = $criteria->first();
                
                } 
                
            } else {
            
                // Else do nothing
                return;
            
            } 
        
        }
        
        // Prepare element model
        $entry = craft()->$service->prepForElementModel($fields, $entry);
        
        try {
        
            // Hook to prepare as appropriate fieldtypes
            array_walk($fields, function(&$data, $handle) {
                return craft()->plugins->call('registerImportOperation', array(&$data, $handle));
            });
        
        } catch(Exception $e) {
        
            // Something went terribly wrong, assume its only this row
            $this->log[$row] = craft()->import_history->log($settings['history'], $row, array('exception' => array($e->getMessage())));
        
        }
        
        // Set fields on entry model
        $entry->setContentFromPost($fields);
        
        try {
        
            // Log
            if(!craft()->$service->save($entry, $settings)) {
            
                // Log errors when unsuccessful
                $this->log[$row] = craft()->import_history->log($settings['history'], $row, $entry->getErrors());
            
            } else {
            
                // Some functions need calling after saving
                craft()->$service->callback($fields, $entry);
                
            }
            
        } catch(Exception $e) {
        
            // Something went terribly wrong, assume its only this row
            $this->log[$row] = craft()->import_history->log($settings['history'], $row, array('exception' => array($e->getMessage())));
        
        }
    
    }
    
    public function finish($settings, $backup) 
    {
    
        craft()->import_history->end($settings['history'], ImportModel::StatusFinished);
        
        if($settings['email']) {
        
            // Gather results
            $results = array(
                'success' => $settings['rows'],
                'errors' => array()
            );
            
            // Gather errors
            foreach($this->log as $line => $result) {
                 $results['errors'][$line] = $result;
            }
            
            // Recalculate successful results
            $results['success'] -= count($results['errors']);
        
            // Prepare the mail
            $email = new EmailModel();
            $emailSettings = craft()->email->getSettings();
            $email->toEmail = $emailSettings['emailAddress'];
            
            // Zip the backup
            if($settings['backup'] && IOHelper::fileExists($backup)) {
                $destZip = craft()->path->getTempPath().IOHelper::getFileName($backup, false).'.zip';
                if(IOHelper::fileExists($destZip)) {
                    IOHelper::deleteFile($destZip, true);
                }
                IOHelper::createFile($destZip);
                if(Zip::add($destZip, $backup, craft()->path->getDbBackupPath())) {
                    $backup = $destZip;
                }
            }
            
            // Set email content
            $email->subject = Craft::t('The import task is finished');
            $email->htmlBody = TemplateHelper::getRaw(craft()->templates->render('import/_email', array(
                'results' => $results,
                'backup' => $backup
            )));
            
            // Send it
            craft()->email->sendEmail($email);
            
        }
    
    }
    
    // Prepare fields for fieldtypes
    public function prepForFieldType(&$data, $handle) 
    {
    
        // Fresh up $data
        $data = StringHelper::convertToUTF8($data);
        $data = trim($data);
                
        // Get field info
        $field = craft()->fields->getFieldByHandle($handle);
        
        // If it's a field ofcourse
        if(!is_null($field)) {
            
            // For some fieldtypes the're special rules
            switch($field->type) {
            
                case ImportModel::FieldTypeEntries:
                
                    // No newlines allowed
                    $data = str_replace("\n", "", $data);
                    $data = str_replace("\r", "", $data);
                    
                    // Don't connect empty fields
                    if(!empty($data)) {
                    
                        // Get field settings
                        $settings = $field->getFieldType()->getSettings();
                
                        // Get source id's for connecting
                        $sectionIds = array();
                        $sources = $settings->sources;
                        if(is_array($sources)) {
                            foreach($sources as $source) {
                                list($type, $id) = explode(':', $source);
                                $sectionIds[] = $id;
                            }
                        }
                                    
                        // Find matching element in sections       
                        $criteria = craft()->elements->getCriteria(ElementType::Entry);
                        $criteria->sectionId = $sectionIds;
                        $criteria->limit = $settings->limit;
 
                        // Get search strings
                        $search = ArrayHelper::stringToArray($data);
                        
                        // Ability to import multiple Assets at once
                        $data = array();
                        
                        // Loop through keywords
                        foreach($search as $query) {
                            
                            // Search
                            $criteria->search = $query;
                            
                            // Add to data
                            $data = array_merge($data, $criteria->ids());
                            
                        }
                    
                    } else {
                    
                        // Return empty array
                        $data = array();
                    
                    }
                                        
                    break;
                
                case ImportModel::FieldTypeCategories:
                    
                    // Don't connect empty fields
                    if(!empty($data)) {
                    
                        // Get field settings
                        $settings = $field->getFieldType()->getSettings();
                                                                                                
                        // Get source id
                        $source = $settings->source;
                        list($type, $id) = explode(':', $source);
                        
                        // Get category data
                        $category = new CategoryModel();
                        $category->groupId = $id;                  
                    
                        // This we append before the slugified path
                        $categoryUrl = str_replace('{slug}', '', $category->getUrlFormat());

                        // Find matching elements in categories
                        $criteria = craft()->elements->getCriteria(ElementType::Category);
                        $criteria->groupId = $id;
                        $criteria->limit = $settings->limit;

                        // Get search strings
                        $search = ArrayHelper::stringToArray($data);
                        
                        // Ability to import multiple Categories at once
                        $data = array();

                        // Loop through keywords
                        foreach($search as $query) {
                            
                            // Find matching element by URI (dirty, not all categories have URI's)
                            $criteria->uri = $categoryUrl . $this->slugify($data);;
                            
                            // Add to data
                            $data = array_merge($data, $criteria->ids());
                            
                        }
                        
                    } else {
                    
                        // Return empty array
                        $data = array();
                    
                    }<commit_msg>Fixed a bug with connecting to categories<commit_after><?php
namespace Craft;

class ImportService extends BaseApplicationComponent 
{

    public $log = array();

    public function columns($file) 
    {
                
        // Open CSV file       
        $data = $this->_open($file);
        
        // Return only column names
        return array_shift($data);
       
    }
    
    public function data($file) 
    {
        
        // Open CSV file
        $data = $this->_open($file);
        
        // Skip first row
        array_shift($data);

        // Return all data
        return $data;
    
    }
    
    public function row($row, $data, $settings) 
    {
    
        // Get max power
        craft()->config->maxPowerCaptain();
        
        // See if map and data match (could not be due to malformed csv)
        if(count($settings['map']) != count($data)) {
        
            // Log errors when unsuccessful
            $this->log[$row] = craft()->import_history->log($settings['history'], $row, array(array(Craft::t('Columns and data did not match, could be due to malformed CSV row.'))));            
            return;
        
        }
        
        // Check what service we're gonna need
        $service = 'import_' . strtolower($settings['type']);
            
        // Map data to fields
        $fields = array_combine($settings['map'], $data);
        
        // If set, remove fields that will not be imported
        if(isset($fields['dont'])) {
            unset($fields['dont']);
        }
        
        // Set up a model to save according to element type
        $entry = craft()->$service->setModel($settings);
        
        // If unique is non-empty array, we're replacing or deleting
        if(is_array($settings['unique']) && count($settings['unique']) > 1) {
            
            // Set criteria according to elementtype
            $criteria = craft()->$service->setCriteria($settings);
                        
            // Set up criteria model for matching        
            $cmodel = array();    
            foreach($settings['map'] as $key => $value) {
                if(isset($criteria->$settings['map'][$key]) && isset($settings['unique'][$key]) && intval($settings['unique'][$key]) == 1 && !empty($fields[$value])) {
                    $criteria->$settings['map'][$key] = $cmodel[$settings['map'][$key]] = $fields[$value];
                }
            }
                        
            // If there's a match...
            if(count($cmodel) && $criteria->count()) {
                
                // If we're deleting
                if($settings['behavior'] == ImportModel::BehaviorDelete) {
                
                    // Get elements to delete
                    $elements = $criteria->find();
                
                    // Fire an 'onBeforeImportDelete' event
                    Craft::import('plugins.import.events.BeforeImportDeleteEvent');
                    $event = new BeforeImportDeleteEvent($this, array('elements' => $elements));
                    $this->onBeforeImportDelete($event);
                    
                    // Give event the chance to blow off deletion
                    if($event->proceed) {
                    
                        try {
                                
                            // Do it
                            if(!craft()->$service->delete($elements)) {
                            
                                // Log errors when unsuccessful
                                $this->log[$row] = craft()->import_history->log($settings['history'], $row, array(array(Craft::t('Something went wrong while deleting this row.'))));            
                                    
                            }
                            
                        } catch(Exception $e) {
                        
                            // Something went terribly wrong, assume its only this row
                            $this->log[$row] = craft()->import_history->log($settings['history'], $row, array('exception' => array($e->getMessage())));
                        
                        }
                        
                    }
                    
                    // Skip rest and continue
                    return;
                    
                } else {
                
                    // Fill new EntryModel with match
                    $entry = $criteria->first();
                
                } 
                
            } else {
            
                // Else do nothing
                return;
            
            } 
        
        }
        
        // Prepare element model
        $entry = craft()->$service->prepForElementModel($fields, $entry);
        
        try {
        
            // Hook to prepare as appropriate fieldtypes
            array_walk($fields, function(&$data, $handle) {
                return craft()->plugins->call('registerImportOperation', array(&$data, $handle));
            });
        
        } catch(Exception $e) {
        
            // Something went terribly wrong, assume its only this row
            $this->log[$row] = craft()->import_history->log($settings['history'], $row, array('exception' => array($e->getMessage())));
        
        }
        
        // Set fields on entry model
        $entry->setContentFromPost($fields);
        
        try {
        
            // Log
            if(!craft()->$service->save($entry, $settings)) {
            
                // Log errors when unsuccessful
                $this->log[$row] = craft()->import_history->log($settings['history'], $row, $entry->getErrors());
            
            } else {
            
                // Some functions need calling after saving
                craft()->$service->callback($fields, $entry);
                
            }
            
        } catch(Exception $e) {
        
            // Something went terribly wrong, assume its only this row
            $this->log[$row] = craft()->import_history->log($settings['history'], $row, array('exception' => array($e->getMessage())));
        
        }
    
    }
    
    public function finish($settings, $backup) 
    {
    
        craft()->import_history->end($settings['history'], ImportModel::StatusFinished);
        
        if($settings['email']) {
        
            // Gather results
            $results = array(
                'success' => $settings['rows'],
                'errors' => array()
            );
            
            // Gather errors
            foreach($this->log as $line => $result) {
                 $results['errors'][$line] = $result;
            }
            
            // Recalculate successful results
            $results['success'] -= count($results['errors']);
        
            // Prepare the mail
            $email = new EmailModel();
            $emailSettings = craft()->email->getSettings();
            $email->toEmail = $emailSettings['emailAddress'];
            
            // Zip the backup
            if($settings['backup'] && IOHelper::fileExists($backup)) {
                $destZip = craft()->path->getTempPath().IOHelper::getFileName($backup, false).'.zip';
                if(IOHelper::fileExists($destZip)) {
                    IOHelper::deleteFile($destZip, true);
                }
                IOHelper::createFile($destZip);
                if(Zip::add($destZip, $backup, craft()->path->getDbBackupPath())) {
                    $backup = $destZip;
                }
            }
            
            // Set email content
            $email->subject = Craft::t('The import task is finished');
            $email->htmlBody = TemplateHelper::getRaw(craft()->templates->render('import/_email', array(
                'results' => $results,
                'backup' => $backup
            )));
            
            // Send it
            craft()->email->sendEmail($email);
            
        }
    
    }
    
    // Prepare fields for fieldtypes
    public function prepForFieldType(&$data, $handle) 
    {
    
        // Fresh up $data
        $data = StringHelper::convertToUTF8($data);
        $data = trim($data);
                
        // Get field info
        $field = craft()->fields->getFieldByHandle($handle);
        
        // If it's a field ofcourse
        if(!is_null($field)) {
            
            // For some fieldtypes the're special rules
            switch($field->type) {
            
                case ImportModel::FieldTypeEntries:
                
                    // No newlines allowed
                    $data = str_replace("\n", "", $data);
                    $data = str_replace("\r", "", $data);
                    
                    // Don't connect empty fields
                    if(!empty($data)) {
                    
                        // Get field settings
                        $settings = $field->getFieldType()->getSettings();
                
                        // Get source id's for connecting
                        $sectionIds = array();
                        $sources = $settings->sources;
                        if(is_array($sources)) {
                            foreach($sources as $source) {
                                list($type, $id) = explode(':', $source);
                                $sectionIds[] = $id;
                            }
                        }
                                    
                        // Find matching element in sections       
                        $criteria = craft()->elements->getCriteria(ElementType::Entry);
                        $criteria->sectionId = $sectionIds;
                        $criteria->limit = $settings->limit;
 
                        // Get search strings
                        $search = ArrayHelper::stringToArray($data);
                        
                        // Ability to import multiple Assets at once
                        $data = array();
                        
                        // Loop through keywords
                        foreach($search as $query) {
                            
                            // Search
                            $criteria->search = $query;
                            
                            // Add to data
                            $data = array_merge($data, $criteria->ids());
                            
                        }
                    
                    } else {
                    
                        // Return empty array
                        $data = array();
                    
                    }
                                        
                    break;
                
                case ImportModel::FieldTypeCategories:
                    
                    // Don't connect empty fields
                    if(!empty($data)) {
                    
                        // Get field settings
                        $settings = $field->getFieldType()->getSettings();
                                                                                                
                        // Get source id
                        $source = $settings->source;
                        list($type, $id) = explode(':', $source);
                        
                        // Get category data
                        $category = new CategoryModel();
                        $category->groupId = $id;                  
                    
                        // This we append before the slugified path
                        $categoryUrl = str_replace('{slug}', '', $category->getUrlFormat());

                        // Find matching elements in categories
                        $criteria = craft()->elements->getCriteria(ElementType::Category);
                        $criteria->groupId = $id;
                        $criteria->limit = $settings->limit;

                        // Get search strings
                        $search = ArrayHelper::stringToArray($data);
                        
                        // Ability to import multiple Categories at once
                        $data = array();

                        // Loop through keywords
                        foreach($search as $query) {
                            
                            // Find matching element by URI (dirty, not all categories have URI's)
                            $criteria->uri = $categoryUrl . $this->slugify($query);
                            
                            // Add to data
                            $data = array_merge($data, $criteria->ids());
                            
                        }
                        
                    } else {
                    
                        // Return empty array
                        $data = array();
                    
                    }
{'fraction_non_alphanumeric': 0.11372564122313795, 'fraction_numerical': 0.00023107140106292846, 'mean_word_length': 0.9787396174655185, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 205, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7629046', 'n_tokens_mistral': 5758, 'n_tokens_neox': 5400, 'n_words': 1252}