<commit_before><?php
namespace php_active_record;

class ContentManager
{
    private $unique_key;
    private static $valid_image_extensions = array(
        'jpg', 'png', 'gif', 'tiff', 'tif', 'svg', 'bmp');
    private static $valid_video_extensions = array(
        'flv', 'mov', 'avi', 'mp4', 'wmv', 'flc', 'ogg', 'mpeg');
    private static $valid_sound_extensions = array(
        'wav', 'aif', 'mp3', 'ogg', 'flac', 'au', 'aac', 'wma');

    function __construct()
    {
        $this->unique_key = Functions::generate_guid();
    }

    // This function will download a file and place it on the content server
    //
    // Possible types for this function
    // image - this is for images
    // video - this is for videos - mpg, flv, mp4, etc...
    // audio - this is for sounds - mp3, wav, etc...
    // upload - what is this for?
    // partner - this type means we are downloading a logo for a content partner
    // resource - this means we are downloading an XML or zipped file of the EOL schema for processing

    function grab_file($file, $type, $options = array())
    {
        if(@!$options['timeout']) $options['timeout'] = DOWNLOAD_TIMEOUT_SECONDS;
        $options['type'] = $type;
        if($temp_file_path = self::download_temp_file_and_assign_extension($file, array_merge($options, array('unique_key' => $this->unique_key, 'is_resource' => ($type == "resource")))))
        {
            $suffix = null;
            if(preg_match("/\.(.*)$/", $temp_file_path, $arr)) $suffix = strtolower(trim($arr[1]));
            if(!$suffix && $type != 'resource') return false;

            switch($type) {
                case "resource":
                    if(!isset($options['resource_id']) || !$options['resource_id']) {
                        trigger_error("ContentManager: type is 'resource' but no resource id given", E_USER_NOTICE);
                        return false;
                    }
                    if(!$suffix)
                    {   // this would be a DwC-A resource
                        $resource_archive_directory = $this->new_resource_file_name($options['resource_id']);
                        // first delete the archive directory that currently exists
                        recursive_rmdir($resource_archive_directory);
                        // move the temp, uncompressed directory to its new home with the resources
                        rename($temp_file_path, $resource_archive_directory);
                        return $resource_archive_directory;
                    }

                    $new_file_prefix = $this->new_resource_file_name($options['resource_id']);
                    break;
                case "image":
                case "video":
                case "audio":
                case "upload":
                case "partner":
                    $new_file_prefix = $this->new_content_file_name();
                    break;
                case "dataset":
                    $new_file_prefix = $this->new_dataset_file_name($options);
                    break;
                default:
                    trigger_error("ContentManager: non-valid type (".$type.")", E_USER_NOTICE);
                    return false;
            }
            if(!$new_file_prefix) return false;
            $new_file_path = $new_file_prefix . "." . $suffix;

            // copy temporary file into its new home
            copy($temp_file_path, $new_file_path);
            // fail if for some reason there is still no file at the new path
            if(!file_exists($new_file_path))
            {
                if(file_exists($temp_file_path)) unlink($temp_file_path);
                trigger_error("ContentManager: Unable to download file $file", E_USER_NOTICE);
                return false;
            }

            // create thumbnails of website content and agent logos
            if($type=="image") $this->create_local_files_and_thumbnails($new_file_path, $new_file_prefix, $options);
            elseif($type=="partner") $this->create_agent_thumbnails($new_file_path, $new_file_prefix);
            elseif($type=="dataset")
            {
                $new_file_path = $this->zip_file($new_file_path);
                $this->delete_old_datasets();
            }

            if(in_array($type, array("image", "video", "audio", "upload", "partner"))) self::create_checksum($new_file_path);

            // Take the substring of the new file path to return via the webservice, so that /content/2009/05/19/23/85866.png -> 200905192385866
            if(($type=="image" || $type=="video" || $type=="audio" || $type=="partner" || $type=="upload") &&
              preg_match("/^".preg_quote(CONTENT_LOCAL_PATH, "/")."(.*)\.[^\.]+$/", $new_file_path, $arr))
            {
                $new_file_path = str_replace("/", "", $arr[1]);
            }
            elseif($type=="resource" &&
              preg_match("/^".preg_quote(CONTENT_RESOURCE_LOCAL_PATH, "/")."(.*)$/", $new_file_path, $arr))  $new_file_path = $arr[1];
            elseif($type=="dataset" &&
              preg_match("/^".preg_quote(CONTENT_DATASET_PATH, "/")."(.*)$/", $new_file_path, $arr))  $new_file_path = $arr[1];
        }

        if(file_exists($temp_file_path)) unlink($temp_file_path);
        if(isset($new_file_path) && $new_file_path) return $new_file_path;
        return null;
    }<commit_msg>Correct faulty logic in grab_file<commit_after><?php
namespace php_active_record;

class ContentManager
{
    private $unique_key;
    private static $valid_image_extensions = array(
        'jpg', 'png', 'gif', 'tiff', 'tif', 'svg', 'bmp');
    private static $valid_video_extensions = array(
        'flv', 'mov', 'avi', 'mp4', 'wmv', 'flc', 'ogg', 'mpeg');
    private static $valid_sound_extensions = array(
        'wav', 'aif', 'mp3', 'ogg', 'flac', 'au', 'aac', 'wma');

    function __construct()
    {
        $this->unique_key = Functions::generate_guid();
    }

    // This function will download a file and place it on the content server
    //
    // Possible types for this function
    // image - this is for images
    // video - this is for videos - mpg, flv, mp4, etc...
    // audio - this is for sounds - mp3, wav, etc...
    // upload - what is this for?
    // partner - this type means we are downloading a logo for a content partner
    // resource - this means we are downloading an XML or zipped file of the EOL schema for processing

    function grab_file($file, $type, $options = array())
    {
        if(@!$options['timeout']) $options['timeout'] = DOWNLOAD_TIMEOUT_SECONDS;
        $options['type'] = $type;
        //should check here if $file is local. If so, just make a hard link to save space
        // (note that the PHP docs wrongly claim link() requires extra privileges on Windows 
        if($temp_file_path = self::download_temp_file_and_assign_extension($file, array_merge($options, array('unique_key' => $this->unique_key, 'is_resource' => ($type == "resource")))))
        {
            $suffix = null;
            if(preg_match("/\.(.*)$/", $temp_file_path, $arr)) $suffix = strtolower(trim($arr[1]));
            if(!$suffix && $type != 'resource') return false;

            switch($type) {
                case "resource":
                    if(!isset($options['resource_id']) || !$options['resource_id']) {
                        trigger_error("ContentManager: type is 'resource' but no resource id given", E_USER_NOTICE);
                        return false;
                    }
                    if(!$suffix)
                    {   // this would be a DwC-A resource
                        $resource_archive_directory = $this->new_resource_file_name($options['resource_id']);
                        // first delete the archive directory that currently exists
                        recursive_rmdir($resource_archive_directory);
                        // move the temp, uncompressed directory to its new home with the resources
                        rename($temp_file_path, $resource_archive_directory);
                        return $resource_archive_directory;
                    }

                    $new_file_prefix = $this->new_resource_file_name($options['resource_id']);
                    break;
                case "image":
                case "video":
                case "audio":
                case "upload":
                case "partner":
                    $new_file_prefix = $this->new_content_file_name();
                    break;
                case "dataset":
                    $new_file_prefix = $this->new_dataset_file_name($options);
                    break;
                default:
                    trigger_error("ContentManager: non-valid type (".$type.")", E_USER_NOTICE);
                    return false;
            }
            if(!$new_file_prefix) return false;
            $new_file_path = $new_file_prefix . "." . $suffix;

            // copy temporary file into its new home
            copy($temp_file_path, $new_file_path);
            // fail if for some reason there is still no file at the new path
            if(!file_exists($new_file_path))
            {
                if(file_exists($temp_file_path)) unlink($temp_file_path);
                trigger_error("ContentManager: Unable to download file $file", E_USER_NOTICE);
                return false;
            }

            // create thumbnails of website content and agent logos
            if($type=="image") $this->create_local_files_and_thumbnails($new_file_path, $new_file_prefix, $options);
            elseif($type=="partner") $this->create_agent_thumbnails($new_file_path, $new_file_prefix);
            elseif($type=="dataset")
            {
                $new_file_path = $this->zip_file($new_file_path);
                $this->delete_old_datasets();
            }

            if(in_array($type, array("image", "video", "audio", "upload", "partner"))) self::create_checksum($new_file_path);

            // Take the substring of the new file path to return via the webservice, so that /content/2009/05/19/23/85866.png -> 200905192385866
            if(($type=="image" || $type=="video" || $type=="audio" || $type=="partner" || $type=="upload") &&
              preg_match("/^".preg_quote(CONTENT_LOCAL_PATH, "/")."(.*)\.[^\.]+$/", $new_file_path, $arr))
            {
                $new_file_path = str_replace("/", "", $arr[1]);
            }
            elseif($type=="resource" &&
              preg_match("/^".preg_quote(CONTENT_RESOURCE_LOCAL_PATH, "/")."(.*)$/", $new_file_path, $arr))  $new_file_path = $arr[1];
            elseif($type=="dataset" &&
              preg_match("/^".preg_quote(CONTENT_DATASET_PATH, "/")."(.*)$/", $new_file_path, $arr))  $new_file_path = $arr[1];

            if(file_exists($temp_file_path)) unlink($temp_file_path);
            if(isset($new_file_path) && $new_file_path) return $new_file_path;
        }
        return null;
    }
{'fraction_non_alphanumeric': 0.1372943327239488, 'fraction_numerical': 0.006946983546617916, 'mean_word_length': 2.076771653543307, 'pattern_counts': {'":': 14, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1269157', 'n_tokens_mistral': 3222, 'n_tokens_neox': 3068, 'n_words': 726}