<commit_before><?php
Yii::import("ext.xupload.models.XUploadForm");
Yii::import("ext.runner.BConsoleRunner");

class ImportController extends GxController
{
    /**
     * Full path of the main uploading folder.
     * @var string
     */
    public $path;

    /**
     * Subfolder in which files will be stored
     * @var string
     */
    public $subfolder = "images";

    public function filters()
    {
        return array(
            'IPBlock',
            'accessControl',
        );
    }

    public function accessRules()
    {
        return array(
            array('allow',
                'actions' => array('view'),
                'roles' => array('*'),
            ),
            array('allow',
                'actions' => array('index', 'uploadfromlocal', 'queueprocess', 'uploadzip', 'uploadftp', 'uploadprocess', 'xuploadmedia', 'batch', 'delete'),
                'roles' => array('editor', 'dbmanager', 'admin'),
            ),
            array('deny',
                'users' => array('*'),
            ),
        );
    }

    public function actionIndex()
    {
        $this->layout = '//layouts/column1';

        if (Yii::app()->user->checkAccess('editor')) {
            $tools = array();

            $tools["import-local"] = array(
                "name" => Yii::t('app', "Import medias from your computer"),
                "description" => Yii::t('app', "Select media(s). This includes the ability to click and drag files to import <strong>(good for small media sets)</strong>."),
                "url" => $this->createUrl('/admin/import/uploadfromlocal'),
            );

            $tools["import-ftp"] = array(
                "name" => Yii::t('app', "Import medias that can be found in the server's '/uploads/ftp' folder"),
                "description" => Yii::t('app', "Place medias in this folder using a SFTP client and let the system do its work <strong>(recommended method for large media sets)</strong>."),
                "url" => $this->createUrl('/admin/import/uploadftp'),
            );

            $tools["import-zip"] = array(
                "name" => Yii::t('app', "Import medias in a ZIP file from your computer"),
                "description" => Yii::t('app', "Import .zip compressed archives of medias. Currently has a filesize limit of 32 MB."),
                "url" => $this->createUrl('/admin/import/uploadzip'),
            );

            $tools["process"] = array(
                "name" => Yii::t('app', "Process imported medias"),
                "description" => Yii::t('app', "Once you have imported medias into the system, use this to process them."),
                "url" => $this->createUrl('/admin/import/uploadprocess'),
            );

            $this->render('index',
                array(
                    'tools' => $tools
                )
            );
        } else {
            throw new CHttpException(403, Yii::t('app', 'Access Denied.'));
        }
    }

    public function actionImportSettings()
    {
        $this->layout = '//layouts/column1';
        $this->render('processimportedmedias', array());
    }

    public function actionUploadFromLocal()
    {
        $this->layout = '//layouts/column1';

        $model = new XUploadForm;
        $this->render('uploadfromlocal', array(
            'model' => $model,
        ));
    }

    public function actionUploadZip()
    {
        $this->layout = '//layouts/column1';
        $this->checkUploadFolder();

        $model = new ImportZipForm;

        if (isset($_POST['ImportZipForm'])) {
            $model->setAttributes($_POST['ImportZipForm']);

            if ($model->validate()) {
                $file_media = CUploadedFile::getInstance($model, 'zipfile');

                if ((is_object($file_media) && get_class($file_media) === 'CUploadedFile')) {
                    $pclzip = $this->module->zip;

                    $tmp_path = sys_get_temp_dir() . "/MG" . date('YmdHis');
                    if (!is_dir($tmp_path)) {
                        mkdir($tmp_path);
                        chmod($tmp_path, 0777);
                    }

                    if (is_dir($tmp_path)) {
                        $list = $pclzip->extractZip($file_media->tempName, $tmp_path);
                        if ($list) {
                            $cnt_added = 0;
                            $cnt_skipped = 0;

                            $path = $this->path . "/" . $this->subfolder . "/";
                            if (!is_dir($path)) {
                                mkdir($path);
                                chmod($path, 0777);
                            }

                            foreach ($list as $file) {
                                $file_info = pathinfo($file['stored_filename']);

                                if (!$file["folder"] && strpos($file['stored_filename'], "__MACOSX") === false) { // we don't want to process folder and MACOSX meta data file mirrors as the mirrored files also return the image/jpg mime type
                                    $mime_type = CFileHelper::getMimeType($file['filename']);
                                    $file_ok = $this->_checkMedia($file['filename'], $mime_type);
                                    //var_dump("TODO");
                                    list($media_type, $extention) = explode('/', $mime_type);
                                    //if ($media_type == "image" && $file_ok) {
                                    if (($media_type == "image" || $media_type == "video" || $media_type == "audio") && $file_ok) {
                                        $cnt_added++;

                                        $file['stored_filename'] = $this->checkFileName($path, $file_info["basename"]);
                                        rename($file['filename'], $path . $file['stored_filename']);
                                        $this->createMedia($file['stored_filename'], $file['size'], $_POST['ImportZipForm']["batch_id"], $mime_type);

                                    } else {
                                        if (!$file_ok)
                                            Flash::add('error', Yii::t('app', 'The file {file} is corrupt and could therefore not be imported.', array('{file}' => $file['filename'])), true);
                                        $cnt_skipped++;
                                    }
                                }
                            }
                            Flash::add("success", Yii::t('app', '{total} files found, {num} medias imported, {num_skipped} other files skipped', array("{num}" => $cnt_added, "{total}" => $cnt_added + $cnt_skipped, "{num_skipped}" => $cnt_skipped)));
                            $this->redirect("uploadprocess");
                        }
                    }

                } else {
                    $model->addError("zipfile", Yii::t('app', 'Please choose a zip file'));
                }
            }
        }

        if (!Yii::app()->getRequest()->getIsPostRequest())
            $model->batch_id = "B-" . date('Y-m-d-H:i:s');

        if (Yii::app()->getRequest()->getIsPostRequest() && !$model->hasErrors()) {
            $model->addError("zipfile", Yii::t('app', 'Please make sure to keep the file smaller than %dB', array('%d' => ini_get('upload_max_filesize'))));
            $model->batch_id = "B-" . date('Y-m-d-H:i:s');
            $model->addError("batch_id", Yii::t('app', 'Please check your upload batch id'));
        }

        $this->render('uploadzip', array(
            'model' => $model,
        ));
    }

    public function actionUploadFtp()
    {
        Yii::app()->clientScript->registerCoreScript('jquery');
        Yii::app()->clientScript->registerScriptFile(Yii::app()->baseUrl . '/js/mg.api.js', CClientScript::POS_END);
        Yii::app()->clientScript->registerCssFile(Yii::app()->baseUrl . '/css/jquery.fancybox-1.3.4.css');
        Yii::app()->clientScript->registerScriptFile(Yii::app()->baseUrl . '/js/jquery.fancybox-1.3.4.pack.js', CClientScript::POS_END);

        // add page resubmit if script time out is close.
        $this->layout = '//layouts/column1';
        $this->checkUploadFolder();

        $path = $this->path . "/" . $this->subfolder . "/";
        if (!is_dir($path)) {
            mkdir($path);
            chmod($path, 0777);
        }

        $model = new ImportFtpForm;
        $count_files = 0;

        $ftp_path = $this->path . "/ftp/";
        if (is_dir($ftp_path)) {
            $list = CFileHelper::findFiles($ftp_path);

            foreach ($list as $file) {
                $file_info = pathinfo($file);
                if ($file_info['basename'] != '.gitignore') {
                    $count_files++;
                }
            }

        }
        if (!Yii::app()->getRequest()->getIsPostRequest())
            $model->batch_id = "B-" . date('Y-m-d-H:i:s');

        $this->render('uploadftp', array(
            'model' => $model,
            'count_files' => $count_files,
        ));
    }

    private function _processFTPQueue()
    {
        $data = array();
        $data['status'] = 'ok';

        $this->checkUploadFolder();

        $path = $this->path . "/" . $this->subfolder . "/";
        if (!is_dir($path)) {
            mkdir($path);
            chmod($path, 0777);
        }

        $model = new ImportFtpForm;
        $count_files = 0;

        $ftp_path = $this->path . "/ftp/";
        if (is_dir($ftp_path)) {
            if (isset($_POST['ImportFtpForm'])) {
                $model->setAttributes($_POST['ImportFtpForm']);

                if ($model->validate()) {
                    $cnt_added = 0;
                    $cnt_skipped = 0;

                    $list = CFileHelper::findFiles($ftp_path);
                    foreach ($list as $file) {
                        $file_info = pathinfo($file);
                        if ($file_info['basename'] != '.gitignore') {
                            $count_files++;
                        }
                    }

                    if ($count_files > 0) {
                        $import_per_request = $model->import_per_request;
                        $model->import_skipped = 0;

                        foreach ($list as $file) {
                            if ($import_per_request > 0) {
                                $file_info = pathinfo($file);
                                $mime_type = CFileHelper::getMimeType($file);
                                $file_ok = $this->_checkMedia($file, $mime_type);

                                if ($file_info['basename'] != '.gitignore') {
                                    //var_dump("TODO");
                                    list($media_type, $extention) = explode('/', $mime_type);
                                    //if ($media_type == "image" && $file_ok) {
                                    if (($media_type == "image" || $media_type == "video" || $media_type == "audio") && $file_ok) {
                                        $model->import_processed++;
                                        $file_name = $this->checkFileName($path, $file_info["basename"]);
                                        rename(str_replace('//', '/', $file), $path . $file_name);
                                        $this->createMedia($file_name, filesize($path . $file_name), $_POST['ImportFtpForm']["batch_id"], $mime_type);
                                        $import_per_request--;
                                    } else {
                                        if (!$file_ok)
                                            Flash::add('error', Yii::t('app', 'The file {file} is corrupt and could therefore not be imported.', array('{file}' => $file)), true);
                                        $model->import_skipped++;
                                    }
                                    $count_files--;
                                }
                            }
                        }

                        if ($count_files == 0) {
                            $this->_finishFTPQueue($model->import_processed, $model->import_skipped);
                        } else {
                            $data['status'] = 'retry';
                            $data['files_left'] = $count_files;
                            $data['ImportFtpForm'] = $model;
                        }
                    } else {
                        $this->_finishFTPQueue($model->import_processed, $model->import_skipped);
                    }
                }
            }
        }
        $this->jsonResponse($data);
    }

    private function _finishFTPQueue($added, $skipped)
    {
        $data['status'] = 'done';
        $data['redirect'] = Yii::app()->createUrl('admin/import/uploadprocess');

        Flash::add("success", Yii::t('app', '{total} files found in \'/uploads/ftp\' folder, {num} medias imported, {num_skipped} other files skipped', array("{total}" => $added + $skipped, "{num}" => $added, "{num_skipped}" => $skipped)));
        if ($skipped > 0)
            Flash::add("warning", Yii::t('app', 'The {num_skipped} files that are still in the \'/uploads/ftp\' folder cannot be imported and should therfore be manually removed!', array("{total}" => $added + $skipped, "{num}" => $added, "{num_skipped}" => $skipped)), true);

        $this->jsonResponse($data);
    }

    public function actionQueueProcess($action)
    {
        switch ($action) {
            case 'ftp':
                $this->_processFTPQueue();
                break;
        }
    }

    public function actionUploadProcess()
    {
        $this->layout = '//layouts/column1';

        $model = new Media('search');
        $model->unsetAttributes();

        if (isset($_GET['Media']))
            $model->setAttributes($_GET['Media']);

        $this->render('uploadprocess', array(
            'model' => $model,
        ));
    }

    public function actionDelete($id)
    {
        if (Yii::app()->getRequest()->getIsPostRequest()) {
            $model = $this->loadModel($id, 'Media');
            if ($model->hasAttribute("locked") && $model->locked) {
                throw new CHttpException(400, Yii::t('app', 'Your request is invalid.'));
            } else {
                $model->delete();
                MGHelper::log('delete', 'Deleted Media with ID(' . $id . ')');

                Flash::add('success', Yii::t('app', "Media deleted"));

                if (!Yii::app()->getRequest()->getIsAjaxRequest())
                    $this->redirect(array('uploadprocess'));
            }
        } else
            throw new CHttpException(400, Yii::t('app', 'Your request is invalid.'));
    }

    public function actionBatch($op)
    {
        $this->layout = '//layouts/column1';
        if (Yii::app()->getRequest()->getIsPostRequest()) {
            switch ($op) {
                case "delete":
                    $this->_batchDelete();
                    break;

                case "process":
                    $this->_batchProcess();
                    break;
            }
            if (!Yii::app()->getRequest()->getIsAjaxRequest())
                $this->actionUploadProcess();
        } else
            throw new CHttpException(400, Yii::t('app', 'Your request is invalid.'));

    }

    private function _batchDelete()
    {
        if (isset($_POST['media-ids'])) {
            $medias = Media::model()->findAllByPk($_POST['media-ids']);

            if ($medias) {
                foreach ($medias as $media) {
                    $media->delete();
                }
            }
            MGHelper::log('batch-delete', 'Batch deleted Medias with IDs(' . implode(',', $_POST['media-ids']) . ')');
        }
    }

    private function _batchProcess()
    {
        $errors = array();
        $processedIDs = array();

        if (isset($_POST['media-ids']) || isset($_POST['massProcess'])) {
            if (isset($_POST['media-ids'])) {
                $medias = Media::model()->findAllByPk($_POST['media-ids']);
            } else {
                $condition = new CDbCriteria;
                $condition->limit = (int)$_POST['massProcess'];
                $condition->order = 'created DESC';
                $medias = Media::model()->findAllByAttributes(array('locked' => 0), $condition);
            }

            if ($medias) {
                $firstModel = $medias[0];

                $plugins = PluginsModule::getAccessiblePlugins("import");
                if (count($plugins) > 0) {
                    foreach ($plugins as $plugin) {
                        if (method_exists($plugin->component, "validate")) {
                            $plugin->component->validate($firstModel, $errors);
                        }
                    }
                }

                if (count($errors) == 0) {
                    if (count($plugins) > 0) {
                        foreach ($plugins as $plugin) {
                            if (method_exists($plugin->component, "process")) {
                                $plugin->component->process($medias);
                            }
                        }
                    }

                    foreach ($medias as $media) {
                        $media->locked = 1;
                        $media->save();
                        $processedIDs[] = $media->id;
                    }
                    MGHelper::log('batch-import-process', 'Batch processed Media with IDs(' . implode(',', $processedIDs) . ')');
                    Flash::add('success', Yii::t('app', 'Processed {count} medias with the IDs({ids})', array("{count}" => count($processedIDs), "{ids}" => implode(',', $processedIDs))));
                }
            }
        } else {
            $errors["noMedias"] = array(Yii::t('ui', 'Please check at least one media you would like to process!'));
        }

        if (count($errors) > 0) {
            if (Yii::app()->getRequest()->getIsAjaxRequest()) {
                $this->jsonResponse($errors);
            } else {
                $model = new Media('search');
                $model->unsetAttributes();

                if (isset($_GET['Media']))
                    $model->setAttributes($_GET['Media']);

                $model->addErrors($errors);

                $this->render('uploadprocess', array(
                    'model' => $model,
                ));
            }
            Yii::app()->end();
        }
    }

    public function actionXUploadMedia()
    {
        $info = array();

        $this->checkUploadFolder();

        $model = new XUploadForm;
        $model->file = CUploadedFile::getInstance($model, 'file');

        if (isset($model->file) && isset($_POST["batch_id"]) && trim($_POST["batch_id"]) != "") {
            $model->mime_type = $model->file->getType();
            $model->size = $model->file->getSize();

            // Remove path information and dots around the filename, to prevent uploading
            // into different directories or replacing hidden system files.
            // Also remove control characters and spaces (\x00..\x20) around the filename:
            $model->name = trim(basename(stripslashes($model->file->getName())), ".\x00..\x20");
            $isMedia = false;
            $thumbUrl = "";
            if ($model->validate()) {
                list($media_type, $extention) = explode('/', $model->mime_type);

                //var_dump("TODO");
                if ($media_type == 'image') {
                    $path = $this->path . "/" . $this->subfolder . "/";
                    if (!is_dir($path)) {
                        mkdir($path);
                        chmod($path, 0777);
                    }

                    $model->name = $this->checkFileName($path, $model->name);
                    $model->file->saveAs($path . $model->name);

                    $media_info = getimagesize($path . $model->name);

                    if (!is_array($media_info) OR count($media_info) < 3)
                        $isMedia = false;
                    else {
                        $this->createMedia($model->name, $model->size, $_POST["batch_id"], $model->mime_type);
                        $thumbUrl = Yii::app()->getBaseUrl() . Yii::app()->fbvStorage->get('settings.app_upload_url') . "/thumbs/" . $model->name;
                        $isMedia = true;
                    }
                } elseif ($media_type == "video" || $media_type == "audio") {
                    $path = $this->path . "/";
                    $model->name = $this->checkFileName($path, $model->name);
                    $model->file->saveAs($path . $model->name);<commit_msg>Add Missing import<commit_after><?php
Yii::import("ext.xupload.models.XUploadForm");
Yii::import("ext.runner.BConsoleRunner");
Yii::import("application.commands.MediaParameters");

class ImportController extends GxController
{
    /**
     * Full path of the main uploading folder.
     * @var string
     */
    public $path;

    /**
     * Subfolder in which files will be stored
     * @var string
     */
    public $subfolder = "images";

    public function filters()
    {
        return array(
            'IPBlock',
            'accessControl',
        );
    }

    public function accessRules()
    {
        return array(
            array('allow',
                'actions' => array('view'),
                'roles' => array('*'),
            ),
            array('allow',
                'actions' => array('index', 'uploadfromlocal', 'queueprocess', 'uploadzip', 'uploadftp', 'uploadprocess', 'xuploadmedia', 'batch', 'delete'),
                'roles' => array('editor', 'dbmanager', 'admin'),
            ),
            array('deny',
                'users' => array('*'),
            ),
        );
    }

    public function actionIndex()
    {
        $this->layout = '//layouts/column1';

        if (Yii::app()->user->checkAccess('editor')) {
            $tools = array();

            $tools["import-local"] = array(
                "name" => Yii::t('app', "Import medias from your computer"),
                "description" => Yii::t('app', "Select media(s). This includes the ability to click and drag files to import <strong>(good for small media sets)</strong>."),
                "url" => $this->createUrl('/admin/import/uploadfromlocal'),
            );

            $tools["import-ftp"] = array(
                "name" => Yii::t('app', "Import medias that can be found in the server's '/uploads/ftp' folder"),
                "description" => Yii::t('app', "Place medias in this folder using a SFTP client and let the system do its work <strong>(recommended method for large media sets)</strong>."),
                "url" => $this->createUrl('/admin/import/uploadftp'),
            );

            $tools["import-zip"] = array(
                "name" => Yii::t('app', "Import medias in a ZIP file from your computer"),
                "description" => Yii::t('app', "Import .zip compressed archives of medias. Currently has a filesize limit of 32 MB."),
                "url" => $this->createUrl('/admin/import/uploadzip'),
            );

            $tools["process"] = array(
                "name" => Yii::t('app', "Process imported medias"),
                "description" => Yii::t('app', "Once you have imported medias into the system, use this to process them."),
                "url" => $this->createUrl('/admin/import/uploadprocess'),
            );

            $this->render('index',
                array(
                    'tools' => $tools
                )
            );
        } else {
            throw new CHttpException(403, Yii::t('app', 'Access Denied.'));
        }
    }

    public function actionImportSettings()
    {
        $this->layout = '//layouts/column1';
        $this->render('processimportedmedias', array());
    }

    public function actionUploadFromLocal()
    {
        $this->layout = '//layouts/column1';

        $model = new XUploadForm;
        $this->render('uploadfromlocal', array(
            'model' => $model,
        ));
    }

    public function actionUploadZip()
    {
        $this->layout = '//layouts/column1';
        $this->checkUploadFolder();

        $model = new ImportZipForm;

        if (isset($_POST['ImportZipForm'])) {
            $model->setAttributes($_POST['ImportZipForm']);

            if ($model->validate()) {

                $file_media = CUploadedFile::getInstance($model, 'zipfile');

                if ((is_object($file_media) && get_class($file_media) === 'CUploadedFile')) {
                    $pclzip = $this->module->zip;

                    $tmp_path = sys_get_temp_dir() . "/MG" . date('YmdHis');
                    if (!is_dir($tmp_path)) {
                        mkdir($tmp_path);
                        chmod($tmp_path, 0777);
                    }

                    if (is_dir($tmp_path)) {
                        $list = $pclzip->extractZip($file_media->tempName, $tmp_path);

                        if ($list) {
                            $cnt_added = 0;
                            $cnt_skipped = 0;

                            $path = $this->path . "/" . $this->subfolder . "/";
                            if (!is_dir($path)) {
                                mkdir($path);
                                chmod($path, 0777);
                            }

                            foreach ($list as $file) {
                                $file_info = pathinfo($file['stored_filename']);

                                if (!$file["folder"] && strpos($file['stored_filename'], "__MACOSX") === false) { // we don't want to process folder and MACOSX meta data file mirrors as the mirrored files also return the image/jpg mime type
                                    $mime_type = CFileHelper::getMimeType($file['filename']);
                                    $file_ok = $this->_checkMedia($file['filename'], $mime_type);
                                    //var_dump("TODO");
                                    list($media_type, $extention) = explode('/', $mime_type);
                                    //if ($media_type == "image" && $file_ok) {
                                    if (($media_type == "image" || $media_type == "video" || $media_type == "audio") && $file_ok) {
                                        $cnt_added++;

                                        $file['stored_filename'] = $this->checkFileName($path, $file_info["basename"]);
                                        rename($file['filename'], $path . $file['stored_filename']);
                                        $this->createMedia($file['stored_filename'], $file['size'], $_POST['ImportZipForm']["batch_id"], $mime_type);

                                    } else {
                                        if (!$file_ok)
                                            Flash::add('error', Yii::t('app', 'The file {file} is corrupt and could therefore not be imported.', array('{file}' => $file['filename'])), true);
                                        $cnt_skipped++;
                                    }
                                }
                            }
                            Flash::add("success", Yii::t('app', '{total} files found, {num} medias imported, {num_skipped} other files skipped', array("{num}" => $cnt_added, "{total}" => $cnt_added + $cnt_skipped, "{num_skipped}" => $cnt_skipped)));
                            $this->redirect("uploadprocess");
                        }
                    }

                } else {
                    $model->addError("zipfile", Yii::t('app', 'Please choose a zip file'));
                }
            }
        }

        if (!Yii::app()->getRequest()->getIsPostRequest())
            $model->batch_id = "B-" . date('Y-m-d-H:i:s');

        if (Yii::app()->getRequest()->getIsPostRequest() && !$model->hasErrors()) {
            $model->addError("zipfile", Yii::t('app', 'Please make sure to keep the file smaller than %dB', array('%d' => ini_get('upload_max_filesize'))));
            $model->batch_id = "B-" . date('Y-m-d-H:i:s');
            $model->addError("batch_id", Yii::t('app', 'Please check your upload batch id'));
        }

        $this->render('uploadzip', array(
            'model' => $model,
        ));
    }

    public function actionUploadFtp()
    {
        Yii::app()->clientScript->registerCoreScript('jquery');
        Yii::app()->clientScript->registerScriptFile(Yii::app()->baseUrl . '/js/mg.api.js', CClientScript::POS_END);
        Yii::app()->clientScript->registerCssFile(Yii::app()->baseUrl . '/css/jquery.fancybox-1.3.4.css');
        Yii::app()->clientScript->registerScriptFile(Yii::app()->baseUrl . '/js/jquery.fancybox-1.3.4.pack.js', CClientScript::POS_END);

        // add page resubmit if script time out is close.
        $this->layout = '//layouts/column1';
        $this->checkUploadFolder();

        $path = $this->path . "/" . $this->subfolder . "/";
        if (!is_dir($path)) {
            mkdir($path);
            chmod($path, 0777);
        }

        $model = new ImportFtpForm;
        $count_files = 0;

        $ftp_path = $this->path . "/ftp/";
        if (is_dir($ftp_path)) {
            $list = CFileHelper::findFiles($ftp_path);

            foreach ($list as $file) {
                $file_info = pathinfo($file);
                if ($file_info['basename'] != '.gitignore') {
                    $count_files++;
                }
            }

        }
        if (!Yii::app()->getRequest()->getIsPostRequest())
            $model->batch_id = "B-" . date('Y-m-d-H:i:s');

        $this->render('uploadftp', array(
            'model' => $model,
            'count_files' => $count_files,
        ));
    }

    private function _processFTPQueue()
    {
        $data = array();
        $data['status'] = 'ok';

        $this->checkUploadFolder();

        $path = $this->path . "/" . $this->subfolder . "/";
        if (!is_dir($path)) {
            mkdir($path);
            chmod($path, 0777);
        }

        $model = new ImportFtpForm;
        $count_files = 0;

        $ftp_path = $this->path . "/ftp/";
        if (is_dir($ftp_path)) {
            if (isset($_POST['ImportFtpForm'])) {
                $model->setAttributes($_POST['ImportFtpForm']);

                if ($model->validate()) {
                    $cnt_added = 0;
                    $cnt_skipped = 0;

                    $list = CFileHelper::findFiles($ftp_path);
                    foreach ($list as $file) {
                        $file_info = pathinfo($file);
                        if ($file_info['basename'] != '.gitignore') {
                            $count_files++;
                        }
                    }

                    if ($count_files > 0) {
                        $import_per_request = $model->import_per_request;
                        $model->import_skipped = 0;

                        foreach ($list as $file) {
                            if ($import_per_request > 0) {
                                $file_info = pathinfo($file);
                                $mime_type = CFileHelper::getMimeType($file);
                                $file_ok = $this->_checkMedia($file, $mime_type);

                                if ($file_info['basename'] != '.gitignore') {
                                    //var_dump("TODO");
                                    list($media_type, $extention) = explode('/', $mime_type);
                                    //if ($media_type == "image" && $file_ok) {
                                    if (($media_type == "image" || $media_type == "video" || $media_type == "audio") && $file_ok) {
                                        $model->import_processed++;
                                        $file_name = $this->checkFileName($path, $file_info["basename"]);
                                        rename(str_replace('//', '/', $file), $path . $file_name);
                                        $this->createMedia($file_name, filesize($path . $file_name), $_POST['ImportFtpForm']["batch_id"], $mime_type);
                                        $import_per_request--;
                                    } else {
                                        if (!$file_ok)
                                            Flash::add('error', Yii::t('app', 'The file {file} is corrupt and could therefore not be imported.', array('{file}' => $file)), true);
                                        $model->import_skipped++;
                                    }
                                    $count_files--;
                                }
                            }
                        }

                        if ($count_files == 0) {
                            $this->_finishFTPQueue($model->import_processed, $model->import_skipped);
                        } else {
                            $data['status'] = 'retry';
                            $data['files_left'] = $count_files;
                            $data['ImportFtpForm'] = $model;
                        }
                    } else {
                        $this->_finishFTPQueue($model->import_processed, $model->import_skipped);
                    }
                }
            }
        }
        $this->jsonResponse($data);
    }

    private function _finishFTPQueue($added, $skipped)
    {
        $data['status'] = 'done';
        $data['redirect'] = Yii::app()->createUrl('admin/import/uploadprocess');

        Flash::add("success", Yii::t('app', '{total} files found in \'/uploads/ftp\' folder, {num} medias imported, {num_skipped} other files skipped', array("{total}" => $added + $skipped, "{num}" => $added, "{num_skipped}" => $skipped)));
        if ($skipped > 0)
            Flash::add("warning", Yii::t('app', 'The {num_skipped} files that are still in the \'/uploads/ftp\' folder cannot be imported and should therfore be manually removed!', array("{total}" => $added + $skipped, "{num}" => $added, "{num_skipped}" => $skipped)), true);

        $this->jsonResponse($data);
    }

    public function actionQueueProcess($action)
    {
        switch ($action) {
            case 'ftp':
                $this->_processFTPQueue();
                break;
        }
    }

    public function actionUploadProcess()
    {
        $this->layout = '//layouts/column1';

        $model = new Media('search');
        $model->unsetAttributes();

        if (isset($_GET['Media']))
            $model->setAttributes($_GET['Media']);

        $this->render('uploadprocess', array(
            'model' => $model,
        ));
    }

    public function actionDelete($id)
    {
        if (Yii::app()->getRequest()->getIsPostRequest()) {
            $model = $this->loadModel($id, 'Media');
            if ($model->hasAttribute("locked") && $model->locked) {
                throw new CHttpException(400, Yii::t('app', 'Your request is invalid.'));
            } else {
                $model->delete();
                MGHelper::log('delete', 'Deleted Media with ID(' . $id . ')');

                Flash::add('success', Yii::t('app', "Media deleted"));

                if (!Yii::app()->getRequest()->getIsAjaxRequest())
                    $this->redirect(array('uploadprocess'));
            }
        } else
            throw new CHttpException(400, Yii::t('app', 'Your request is invalid.'));
    }

    public function actionBatch($op)
    {
        $this->layout = '//layouts/column1';
        if (Yii::app()->getRequest()->getIsPostRequest()) {
            switch ($op) {
                case "delete":
                    $this->_batchDelete();
                    break;

                case "process":
                    $this->_batchProcess();
                    break;
            }
            if (!Yii::app()->getRequest()->getIsAjaxRequest())
                $this->actionUploadProcess();
        } else
            throw new CHttpException(400, Yii::t('app', 'Your request is invalid.'));

    }

    private function _batchDelete()
    {
        if (isset($_POST['media-ids'])) {
            $medias = Media::model()->findAllByPk($_POST['media-ids']);

            if ($medias) {
                foreach ($medias as $media) {
                    $media->delete();
                }
            }
            MGHelper::log('batch-delete', 'Batch deleted Medias with IDs(' . implode(',', $_POST['media-ids']) . ')');
        }
    }

    private function _batchProcess()
    {
        $errors = array();
        $processedIDs = array();

        if (isset($_POST['media-ids']) || isset($_POST['massProcess'])) {
            if (isset($_POST['media-ids'])) {
                $medias = Media::model()->findAllByPk($_POST['media-ids']);
            } else {
                $condition = new CDbCriteria;
                $condition->limit = (int)$_POST['massProcess'];
                $condition->order = 'created DESC';
                $medias = Media::model()->findAllByAttributes(array('locked' => 0), $condition);
            }

            if ($medias) {
                $firstModel = $medias[0];

                $plugins = PluginsModule::getAccessiblePlugins("import");
                if (count($plugins) > 0) {
                    foreach ($plugins as $plugin) {
                        if (method_exists($plugin->component, "validate")) {
                            $plugin->component->validate($firstModel, $errors);
                        }
                    }
                }

                if (count($errors) == 0) {
                    if (count($plugins) > 0) {
                        foreach ($plugins as $plugin) {
                            if (method_exists($plugin->component, "process")) {
                                $plugin->component->process($medias);
                            }
                        }
                    }

                    foreach ($medias as $media) {
                        $media->locked = 1;
                        $media->save();
                        $processedIDs[] = $media->id;
                    }
                    MGHelper::log('batch-import-process', 'Batch processed Media with IDs(' . implode(',', $processedIDs) . ')');
                    Flash::add('success', Yii::t('app', 'Processed {count} medias with the IDs({ids})', array("{count}" => count($processedIDs), "{ids}" => implode(',', $processedIDs))));
                }
            }
        } else {
            $errors["noMedias"] = array(Yii::t('ui', 'Please check at least one media you would like to process!'));
        }

        if (count($errors) > 0) {
            if (Yii::app()->getRequest()->getIsAjaxRequest()) {
                $this->jsonResponse($errors);
            } else {
                $model = new Media('search');
                $model->unsetAttributes();

                if (isset($_GET['Media']))
                    $model->setAttributes($_GET['Media']);

                $model->addErrors($errors);

                $this->render('uploadprocess', array(
                    'model' => $model,
                ));
            }
            Yii::app()->end();
        }
    }

    public function actionXUploadMedia()
    {
        $info = array();

        $this->checkUploadFolder();

        $model = new XUploadForm;
        $model->file = CUploadedFile::getInstance($model, 'file');

        if (isset($model->file) && isset($_POST["batch_id"]) && trim($_POST["batch_id"]) != "") {
            $model->mime_type = $model->file->getType();
            $model->size = $model->file->getSize();

            // Remove path information and dots around the filename, to prevent uploading
            // into different directories or replacing hidden system files.
            // Also remove control characters and spaces (\x00..\x20) around the filename:
            $model->name = trim(basename(stripslashes($model->file->getName())), ".\x00..\x20");
            $isMedia = false;
            $thumbUrl = "";
            if ($model->validate()) {
                list($media_type, $extention) = explode('/', $model->mime_type);

                //var_dump("TODO");
                if ($media_type == 'image') {
                    $path = $this->path . "/" . $this->subfolder . "/";
                    if (!is_dir($path)) {
                        mkdir($path);
                        chmod($path, 0777);
                    }

                    $model->name = $this->checkFileName($path, $model->name);
                    $model->file->saveAs($path . $model->name);

                    $media_info = getimagesize($path . $model->name);

                    if (!is_array($media_info) OR count($media_info) < 3)
                        $isMedia = false;
                    else {
                        $this->createMedia($model->name, $model->size, $_POST["batch_id"], $model->mime_type);
                        $thumbUrl = Yii::app()->getBaseUrl() . Yii::app()->fbvStorage->get('settings.app_upload_url') . "/thumbs/" . $model->name;
                        $isMedia = true;
                    }
                }
                elseif ($media_type == "video" || $media_type == "audio") {
                    $path = $this->path . "/";
                    $model->name = $this->checkFileName($path, $model->name);
                    $model->file->saveAs($path . $model->name);
{'fraction_non_alphanumeric': 0.15946337926033358, 'fraction_numerical': 0.0035774715977761663, 'mean_word_length': 1.4555436847103513, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 425, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 76, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3313805', 'n_tokens_mistral': 11294, 'n_tokens_neox': 10573, 'n_words': 2112}