<commit_before><?php

$obj = new stdClass();
header('Content-Type: application/json');
require_once dirname(__FILE__) . '/../videos/configuration.php';
require_once $global['systemRootPath'] . 'objects/Encoder.php';
require_once $global['systemRootPath'] . 'objects/Login.php';
require_once $global['systemRootPath'] . 'objects/Streamer.php';

session_write_close();

function addVideo($link, $streamers_id, $title = "") {
    $obj = new stdClass();
    // remove list parameter from
    $link = preg_replace('~(\?|&)list=[^&]*~', '$1', $link);
    $link = str_replace("?&", "?", $link);
    if (substr($link, -1) == '&') {
        $link = substr($link, 0, -1);
    }
    if(empty($title)){
        $title = Encoder::getTitleFromLink($link);
    }
    if (!$title) {
        $obj->error = "youtube-dl --force-ipv4 get title ERROR** " . print_r($link, true);
        $obj->type = "warning";
        $obj->title = "Sorry!";
        $obj->text = sprintf("We could not get the title of your video (%s) go to %s to fix it", $link, "<a href='https://github.com/DanielnetoDotCom/YouPHPTube/wiki/youdtube-dl-failed-to-extract-signature' class='btn btn-xm btn-default'>https://github.com/DanielnetoDotCom/YouPHPTube/wiki/youdtube-dl-failed-to-extract-signature</a>");
    } else {
        $obj->type = "success";
        $obj->title = "Congratulations!";
        $obj->text = sprintf("Your video (%s) is downloading", $title);

        $filename = preg_replace("/[^A-Za-z0-9]+/", "_", cleanString($title));
        $filename = uniqid("{$filename}_YPTuniqid_", true) . ".mp4";

        $s = new Streamer($streamers_id);

        $e = new Encoder("");
        $e->setStreamers_id($streamers_id);
        $e->setTitle($title);
        $e->setFileURI($link);
        $e->setVideoDownloadedLink($link);
        $e->setFilename($filename);
        $e->setStatus('queue');
        $e->setPriority($s->getPriority());
        //$e->setNotifyURL($global['YouPHPTubeURL'] . "youPHPTubeEncoder.json");

        $encoders_ids = array();

        if (!empty($_POST['audioOnly']) && $_POST['audioOnly'] !== 'false') {
            if (!empty($_POST['spectrum']) && $_POST['spectrum'] !== 'false') {
                $e->setFormats_idFromOrder(70); // video to spectrum [(6)MP4 to MP3] -> [(5)MP3 to spectrum] -> [(2)MP4 to webm] 
            } else {
                $e->setFormats_idFromOrder(71);
            }
        } else {
            $e->setFormats_idFromOrder(decideFormatOrder());
        }
        $obj = new stdClass();
        $f = new Format($e->getFormats_id());
        $format = $f->getExtension();
        $response = Encoder::sendFile('', 0, $format, $e);
        //var_dump($response);exit;
        if (!empty($response->response->video_id)) {
            $obj->videos_id = $response->response->video_id;
        }
        $e->setReturn_vars(json_encode($obj));
        $encoders_ids[] = $e->save();
    }
    return $obj;
}

if (!Login::canUpload()) {
    $obj->msg = "This user can not upload files";
} else {
    if (!($streamers_id = Login::getStreamerId())) {
        $obj->msg = "There is no streamer site";
    } else {
        // if it is a channel
        $rexexp = "/^(http(s)?:\/\/)?((w){3}.)?youtu(be|.be)?(\.com)?\/(channel|user).+/";
        if (preg_match($rexexp, $_POST['videoURL'])) {
            if (!empty($global['disableBulkEncode'])){
                $obj->msg = "Channel Import is disabled";
                die(json_encode($obj));
            }
            $start = 0;
            $end = 100;
            if(!empty($_POST['startIndex'])){
                $start = $current = intval($_POST['startIndex']);                
            }
            if(!empty($_POST['endIndex'])){
                $end = intval($_POST['endIndex']);
            }            
            error_log("Processing Channel {$start} to {$end}");
            $list = Encoder::getReverseVideosJsonListFromLink($_POST['videoURL']);
            $i=$start;
            for(; $i<=$end;$i++){
                error_log(($i)." Process Video {$list[$i]->id}");
                $url = "https://www.youtube.com/watch?v={$list[$i]->url}";
                $obj = addVideo($url, $streamers_id, $list[$i]->title);
            }
            error_log("Process Done Total {$i}");
        } else {
            $obj = addVideo($_POST['videoURL'], $streamers_id);
        }
    }
}
die(json_encode($obj));<commit_msg>Improve Channel Import, much faster now<commit_after><?php

$obj = new stdClass();
header('Content-Type: application/json');
require_once dirname(__FILE__) . '/../videos/configuration.php';
require_once $global['systemRootPath'] . 'objects/Encoder.php';
require_once $global['systemRootPath'] . 'objects/Login.php';
require_once $global['systemRootPath'] . 'objects/Streamer.php';

session_write_close();

function addVideo($link, $streamers_id, $title = "") {
    $obj = new stdClass();
    // remove list parameter from
    $link = preg_replace('~(\?|&)list=[^&]*~', '$1', $link);
    $link = str_replace("?&", "?", $link);
    if (substr($link, -1) == '&') {
        $link = substr($link, 0, -1);
    }
    if(empty($title)){
        $title = Encoder::getTitleFromLink($link);
    }
    if (!$title) {
        $obj->error = "youtube-dl --force-ipv4 get title ERROR** " . print_r($link, true);
        $obj->type = "warning";
        $obj->title = "Sorry!";
        $obj->text = sprintf("We could not get the title of your video (%s) go to %s to fix it", $link, "<a href='https://github.com/DanielnetoDotCom/YouPHPTube/wiki/youdtube-dl-failed-to-extract-signature' class='btn btn-xm btn-default'>https://github.com/DanielnetoDotCom/YouPHPTube/wiki/youdtube-dl-failed-to-extract-signature</a>");
    } else {
        $obj->type = "success";
        $obj->title = "Congratulations!";
        $obj->text = sprintf("Your video (%s) is downloading", $title);

        $filename = preg_replace("/[^A-Za-z0-9]+/", "_", cleanString($title));
        $filename = uniqid("{$filename}_YPTuniqid_", true) . ".mp4";

        $s = new Streamer($streamers_id);

        $e = new Encoder("");
        $e->setStreamers_id($streamers_id);
        $e->setTitle($title);
        $e->setFileURI($link);
        $e->setVideoDownloadedLink($link);
        $e->setFilename($filename);
        $e->setStatus('queue');
        $e->setPriority($s->getPriority());
        //$e->setNotifyURL($global['YouPHPTubeURL'] . "youPHPTubeEncoder.json");

        $encoders_ids = array();

        if (!empty($_POST['audioOnly']) && $_POST['audioOnly'] !== 'false') {
            if (!empty($_POST['spectrum']) && $_POST['spectrum'] !== 'false') {
                $e->setFormats_idFromOrder(70); // video to spectrum [(6)MP4 to MP3] -> [(5)MP3 to spectrum] -> [(2)MP4 to webm] 
            } else {
                $e->setFormats_idFromOrder(71);
            }
        } else {
            $e->setFormats_idFromOrder(decideFormatOrder());
        }
        $obj = new stdClass();
        $f = new Format($e->getFormats_id());
        $format = $f->getExtension();
        $response = Encoder::sendFile('', 0, $format, $e);
        //var_dump($response);exit;
        if (!empty($response->response->video_id)) {
            $obj->videos_id = $response->response->video_id;
        }
        $e->setReturn_vars(json_encode($obj));
        $encoders_ids[] = $e->save();
    }
    return $obj;
}

if (!Login::canUpload()) {
    $obj->msg = "This user can not upload files";
} else {
    if (!($streamers_id = Login::getStreamerId())) {
        $obj->msg = "There is no streamer site";
    } else {
        // if it is a channel
        $rexexp = "/^(http(s)?:\/\/)?((w){3}.)?youtu(be|.be)?(\.com)?\/(channel|user).+/";
        if (preg_match($rexexp, $_POST['videoURL'])) {
            if (!empty($global['disableBulkEncode'])){
                $obj->msg = "Channel Import is disabled";
                die(json_encode($obj));
            }
            $start = 0;
            $end = 100;
            if(!empty($_POST['startIndex'])){
                $start = $current = intval($_POST['startIndex']);                
            }
            if(!empty($_POST['endIndex'])){
                $end = intval($_POST['endIndex']);
            }            
            error_log("Processing Channel {$start} to {$end}");
            $list = Encoder::getReverseVideosJsonListFromLink($_POST['videoURL']);
            $i=$start;
            for(; $i<=$end;$i++){
                if(is_object($list[$i]) && empty($list[$i]->id)){
                    continue;
                }
                error_log(($i)." Process Video {$list[$i]->id}");
                $url = "https://www.youtube.com/watch?v={$list[$i]->url}";
                $obj = addVideo($url, $streamers_id, $list[$i]->title);
            }
            error_log("Process Done Total {$i}");
        } else {
            $obj = addVideo($_POST['videoURL'], $streamers_id);
        }
    }
}
die(json_encode($obj));
{'fraction_non_alphanumeric': 0.20264514682806545, 'fraction_numerical': 0.005604124635731899, 'mean_word_length': 2.416156202143951, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 80, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '887243', 'n_tokens_mistral': 2928, 'n_tokens_neox': 2837, 'n_words': 499}