<commit_before><?php

function startsWith($haystack, $needle)
{
     $length = strlen($needle);
     return (substr($haystack, 0, $length) === $needle);
}

function endsWith($haystack, $needle)
{
    $length = strlen($needle);
    if ($length == 0) {
        return true;
    }

    return (substr($haystack, -$length) === $needle);
}

function analyze_url($url) {

    $description = NULL;
    $embed = NULL;

    $url_headers = get_headers($url, 1);
    if( isset($url_headers['Content-Type']) && !is_array($url_headers['Content-Type']) ){

        $type = strtolower($url_headers['Content-Type']);

        $valid_image_type = array();
        $valid_image_type['image/png'] = '';
        $valid_image_type['image/jpg'] = '';
        $valid_image_type['image/jpeg'] = '';
        $valid_image_type['image/jpe'] = '';
        $valid_image_type['image/gif'] = '';
        $valid_image_type['image/tif'] = '';
        $valid_image_type['image/tiff'] = '';
        $valid_image_type['image/svg'] = '';
        $valid_image_type['image/ico'] = '';
        $valid_image_type['image/icon'] = '';
        $valid_image_type['image/x-icon'] = '';

        if(isset($valid_image_type[$type])){
            $embed = '<img src="'.$url.'" style="max-height:315px;"/>';
            return [$url,$description,$embed];
        }
    }

    $parsed = parse_url($url);
    $segment = explode('/', $parsed['path']);
    if (!isset($segment[1])) {
        $segment[1] = '';
    }
    if (!isset($segment[2])) {
        $segment[2] = '';
    }

    $curl = curl_init();
    curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
    curl_setopt($curl, CURLOPT_HEADER, false);
    curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($curl, CURLOPT_URL, $url);
    curl_setopt($curl, CURLOPT_REFERER, $url);
    curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
    curl_setopt($curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4");
    $curl_result = curl_exec($curl);
    curl_close($curl);

    $CI =& get_instance();
    $CI->load->library("simple_html_dom");
    $html = new Simple_html_dom();
    //$html->load_file($url);
    $html->load($curl_result);

    if ($html->find('meta[property=og:description]')) {
        $description = trim(str_replace(array('&#039;','&#39;'),"'",$html->find('meta[property=og:description]',0)->content));
    }

    if ( endsWith($parsed['host'],'youtube.com') || ($parsed['host'] == 'youtu.be') ) {

        preg_match("/^.*((youtu.be\/)|(v\/)|(\/u\/\w\/)|(embed\/)|(watch\?))\??v?=?([^#\&\?]*).*/", $url, $id);
        $embed = '<iframe width="560" height="315" src="https://www.youtube.com/embed/'.$id[7].'" frameborder="0" allowfullscreen></iframe>';

        return [$html->find('link[itemprop=thumbnailUrl]',0)->href,$description,$embed];
    }

    if ( endsWith($parsed['host'],'twitter.com') && ($segment[2] == 'status') ) {
        if (startsWith($parsed['host'],'mobile')) {
            $json = file_get_contents("https://publish.twitter.com/oembed?url=".'https://twitter.com/'.$segment[1].'/'.$segment[2].'/'.$segment[3]);
        } else {
            $json = file_get_contents("https://publish.twitter.com/oembed?url=".$url);
        }
        $obj = json_decode($json);
        $embed = $obj->html;
    }

    $fb_post_criteria = ['posts','activity','photo.php','photos','permalink.php','media','questions','notes'];
    if ( endsWith($parsed['host'],'facebook.com') && (!empty(array_intersect($segment, $fb_post_criteria))) ) {
        $json_url = "https://www.facebook.com/plugins/post/oembed.json/?url=".$url;
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
        curl_setopt($curl, CURLOPT_HEADER, false);
        curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($curl, CURLOPT_URL, $json_url);
        curl_setopt($curl, CURLOPT_REFERER, $json_url);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
        curl_setopt($curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4");
        $json = curl_exec($curl);
        curl_close($curl);
        $obj = json_decode($json);
        $embed = $obj->html;
        return ['https://www.facebook.com/images/fb_icon_325x325.png',$description,$embed];
    }

    $fb_video_criteria = ['videos','video.php'];
    if ( endsWith($parsed['host'],'facebook.com') && (!empty(array_intersect($segment, $fb_video_criteria))) ) {
        $json_url = "https://www.facebook.com/plugins/video/oembed.json/?url=".$url;
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
        curl_setopt($curl, CURLOPT_HEADER, false);
        curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($curl, CURLOPT_URL, $json_url);
        curl_setopt($curl, CURLOPT_REFERER, $json_url);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
        curl_setopt($curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4");
        $json = curl_exec($curl);
        curl_close($curl);
        $obj = json_decode($json);
        $embed = $obj->html;
        return ['https://www.facebook.com/images/fb_icon_325x325.png',$description,$embed];
    }

    if ( ( endsWith($parsed['host'],'instagram.com') || ($parsed['host'] == 'instagr.am') ) && ($segment[1] == 'p') ) {
        $json = file_get_contents("https://api.instagram.com/oembed?url=".$url);
        $obj = json_decode($json);
        $embed = $obj->html;
    }

    if ($html->find('meta[property=og:image]')) {
        return [$html->find('meta[property=og:image]',0)->content,$description,$embed];
    }

    $biggestImage = ''; // Is returned when no images are found.
    $maxSize = 0;
    $visited = array();
    $getimagesize_counter = 0;
    $min_edge_length = 200;
    foreach($html->find('img') as $element) {
        //echo "\nNEW IMAGE:\n";
        $src = $element->src;
        if($src=='')continue;// it happens on your test url
        if (strpos($src, '://') !== false) {
            $imageurl = $src;
        } elseif (substr( $src, 0, 2 ) === "//") {
            $imageurl = 'http:'.$src;
        } else {
            $imageurl = $parsed['scheme'].'://'.$parsed['host'].'/'.$src;//get image absolute url
        }<commit_msg>Remove hex chars from Instagram embed<commit_after><?php

function startsWith($haystack, $needle)
{
     $length = strlen($needle);
     return (substr($haystack, 0, $length) === $needle);
}

function endsWith($haystack, $needle)
{
    $length = strlen($needle);
    if ($length == 0) {
        return true;
    }

    return (substr($haystack, -$length) === $needle);
}

function analyze_url($url) {

    $description = NULL;
    $embed = NULL;

    $url_headers = get_headers($url, 1);
    if( isset($url_headers['Content-Type']) && !is_array($url_headers['Content-Type']) ){

        $type = strtolower($url_headers['Content-Type']);

        $valid_image_type = array();
        $valid_image_type['image/png'] = '';
        $valid_image_type['image/jpg'] = '';
        $valid_image_type['image/jpeg'] = '';
        $valid_image_type['image/jpe'] = '';
        $valid_image_type['image/gif'] = '';
        $valid_image_type['image/tif'] = '';
        $valid_image_type['image/tiff'] = '';
        $valid_image_type['image/svg'] = '';
        $valid_image_type['image/ico'] = '';
        $valid_image_type['image/icon'] = '';
        $valid_image_type['image/x-icon'] = '';

        if(isset($valid_image_type[$type])){
            $embed = '<img src="'.$url.'" style="max-height:315px;"/>';
            return [$url,$description,$embed];
        }
    }

    $parsed = parse_url($url);
    $segment = explode('/', $parsed['path']);
    if (!isset($segment[1])) {
        $segment[1] = '';
    }
    if (!isset($segment[2])) {
        $segment[2] = '';
    }

    $curl = curl_init();
    curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
    curl_setopt($curl, CURLOPT_HEADER, false);
    curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($curl, CURLOPT_URL, $url);
    curl_setopt($curl, CURLOPT_REFERER, $url);
    curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
    curl_setopt($curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4");
    $curl_result = curl_exec($curl);
    curl_close($curl);

    $CI =& get_instance();
    $CI->load->library("simple_html_dom");
    $html = new Simple_html_dom();
    //$html->load_file($url);
    $html->load($curl_result);

    if ($html->find('meta[property=og:description]')) {
        $description = trim(str_replace(array('&#039;','&#39;'),"'",$html->find('meta[property=og:description]',0)->content));
    }

    if ( endsWith($parsed['host'],'youtube.com') || ($parsed['host'] == 'youtu.be') ) {

        preg_match("/^.*((youtu.be\/)|(v\/)|(\/u\/\w\/)|(embed\/)|(watch\?))\??v?=?([^#\&\?]*).*/", $url, $id);
        $embed = '<iframe width="560" height="315" src="https://www.youtube.com/embed/'.$id[7].'" frameborder="0" allowfullscreen></iframe>';

        return [$html->find('link[itemprop=thumbnailUrl]',0)->href,$description,$embed];
    }

    if ( endsWith($parsed['host'],'twitter.com') && ($segment[2] == 'status') ) {
        if (startsWith($parsed['host'],'mobile')) {
            $json = file_get_contents("https://publish.twitter.com/oembed?url=".'https://twitter.com/'.$segment[1].'/'.$segment[2].'/'.$segment[3]);
        } else {
            $json = file_get_contents("https://publish.twitter.com/oembed?url=".$url);
        }
        $obj = json_decode($json);
        $embed = $obj->html;
    }

    $fb_post_criteria = ['posts','activity','photo.php','photos','permalink.php','media','questions','notes'];
    if ( endsWith($parsed['host'],'facebook.com') && (!empty(array_intersect($segment, $fb_post_criteria))) ) {
        $json_url = "https://www.facebook.com/plugins/post/oembed.json/?url=".$url;
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
        curl_setopt($curl, CURLOPT_HEADER, false);
        curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($curl, CURLOPT_URL, $json_url);
        curl_setopt($curl, CURLOPT_REFERER, $json_url);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
        curl_setopt($curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4");
        $json = curl_exec($curl);
        curl_close($curl);
        $obj = json_decode($json);
        $embed = $obj->html;
        return ['https://www.facebook.com/images/fb_icon_325x325.png',$description,$embed];
    }

    $fb_video_criteria = ['videos','video.php'];
    if ( endsWith($parsed['host'],'facebook.com') && (!empty(array_intersect($segment, $fb_video_criteria))) ) {
        $json_url = "https://www.facebook.com/plugins/video/oembed.json/?url=".$url;
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
        curl_setopt($curl, CURLOPT_HEADER, false);
        curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($curl, CURLOPT_URL, $json_url);
        curl_setopt($curl, CURLOPT_REFERER, $json_url);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, TRUE);
        curl_setopt($curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.125 Safari/533.4");
        $json = curl_exec($curl);
        curl_close($curl);
        $obj = json_decode($json);
        $embed = $obj->html;
        return ['https://www.facebook.com/images/fb_icon_325x325.png',$description,$embed];
    }

    if ( ( endsWith($parsed['host'],'instagram.com') || ($parsed['host'] == 'instagr.am') ) && ($segment[1] == 'p') ) {
        $json = file_get_contents("https://api.instagram.com/oembed?url=".$url);
        $obj = json_decode($json);
        $embed = $obj->html;
        $embed = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $embed);
    }

    if ($html->find('meta[property=og:image]')) {
        return [$html->find('meta[property=og:image]',0)->content,$description,$embed];
    }

    $biggestImage = ''; // Is returned when no images are found.
    $maxSize = 0;
    $visited = array();
    $getimagesize_counter = 0;
    $min_edge_length = 200;
    foreach($html->find('img') as $element) {
        //echo "\nNEW IMAGE:\n";
        $src = $element->src;
        if($src=='')continue;// it happens on your test url
        if (strpos($src, '://') !== false) {
            $imageurl = $src;
        } elseif (substr( $src, 0, 2 ) === "//") {
            $imageurl = 'http:'.$src;
        } else {
            $imageurl = $parsed['scheme'].'://'.$parsed['host'].'/'.$src;//get image absolute url
        }
{'fraction_non_alphanumeric': 0.20973322057353733, 'fraction_numerical': 0.017298377796571077, 'mean_word_length': 3.249591636720026, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 45, 'https://': 18, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2131891', 'n_tokens_mistral': 4794, 'n_tokens_neox': 4495, 'n_words': 648}