<commit_before><?php

/**
 * Get temp dir for ocrstream, create one if none exists
 * 
 * @return string OCRStream plugin temp directory
 */
function get_ocr_temp_dir() {
    $temp_dir = get_temp_dir();
    if(!is_dir($temp_dir . "/ocrstream_plugin")){
        mkdir($temp_dir . "/ocrstream_plugin",0777);        
    }
    $ocr_temp_dir = $temp_dir . "/ocrstream_plugin";
    return ($ocr_temp_dir);
}

/**
 * Checks if OS is Windows
 * 
 * @return boolean
 */
function is_windows() {
    $os = php_uname('s');
    if (stristr($os, 'win')) {
        if (stristr($os, 'Darwin')) {
            $is_windows = false;
        } else {
            $is_windows = true;
        }
    } else {
        $is_windows = false;
    }
    return $is_windows;
}

/**
 * Get path to tesseract executable
 * 
 * @global string $tesseract_path Tesseract directory path
 * @return string
 */
function get_tesseract_fullpath() {
    global $tesseract_path;
    if (is_windows()) {
        $tesseract_fullpath = $tesseract_path . '\tesseract.exe';
    } else {
        $tesseract_fullpath = $tesseract_path . '/tesseract';
    }
    return $tesseract_fullpath;
}

/**
 * Checks if tesseract executable exists
 * 
 * @return boolean 
 */
function is_tesseract_installed() {
    $tesseract_fullpath = get_tesseract_fullpath();
    if (file_exists($tesseract_fullpath)) {
        $tesseract_installed = true;
    } else {
        $tesseract_installed = false;
    }
    return $tesseract_installed;
}

/**
 * Get tesseract and leptonica version information
 * 
 * @return array tesseract version, leptonica version, tesseract_is_old
 */
function get_tesseract_version() {
    // Check if version information is already in plugin_config
    $plugin_config = get_plugin_config('ocrstream');
    if (isset($plugin_config['tesseract_version']) && isset($plugin_config['leptonica_version'])) {
        // Read version from config and return array
        $tesseract_version = $plugin_config['tesseract_version'];
        $leptonica_version = $plugin_config['leptonica_version'];
    } else {
        $tesseract_fullpath = get_tesseract_fullpath();
        $tesseract_version_command = run_command(escapeshellarg($tesseract_fullpath) . ' -v', true);
        $tesseract_version_output = explode("\n", $tesseract_version_command);
        if (stristr($tesseract_version_output [0], 'libtiff.so.5')) { // Skipping error output in first line if libftiff/liblept version mismatch
            $tesseract_version = $tesseract_version_output[1];
            $leptonica_version = $tesseract_version_output[2];
        } else {
            $tesseract_version = $tesseract_version_output[0];
            $leptonica_version = $tesseract_version_output[1];
        }
        $version_regex = "/([0-9.]+)/mi"; 
        preg_match_all($version_regex, $tesseract_version, $tess_matches);
        preg_match_all($version_regex, $leptonica_version, $lept_matches);
        $tesseract_version = trim($tess_matches[0][0]);
        $leptonica_version = trim($lept_matches[0][0]);
        // Add version to plugin_config and write it back to db
        $plugin_config['tesseract_version'] = $tesseract_version;
        $plugin_config['leptonica_version'] = $leptonica_version;
        set_plugin_config('ocrstream', $plugin_config);
    }
    if (version_compare($tesseract_version, '3.03') >= 0) {
        $tesseract_version_is_old = false;
    } else {
        $tesseract_version_is_old = true;
    }       
    return array($tesseract_version, $leptonica_version, $tesseract_version_is_old);
}

/**
 * Get available languages for tesseract-ocr
 * 
 * Returns an array of languages that are currently installed into the /TESSDATA directory.
 * Language codes use ISO 639-2 standard.
 * 
 * @link https://code.google.com/p/tesseract-ocr/downloads/list tesseract language data downloads
 * @return array 
 */
function get_tesseract_languages() {
    // Check if languages are already in plugin_config
    $plugin_config = get_plugin_config('ocrstream');
    if (isset($plugin_config['tesseract_languages'])) {
        // Read languages from config and return array
        $tesseract_languages = $plugin_config['tesseract_languages'];
    } else {
        // Get languages via tesseract cli
        $tesseract_fullpath = get_tesseract_fullpath();
        $tesseract_language_command = run_command(escapeshellarg($tesseract_fullpath) . ' --list-langs', true);
        $tesseract_languages = explode("\n", $tesseract_language_command);
        if (stristr($tesseract_languages [0], 'libtiff.so.5')) { // Skipping additional line if libftiff version does not match liblept version
            array_shift($tesseract_languages);
            array_shift($tesseract_languages);
            array_pop($tesseract_languages);
        } else {
            array_shift($tesseract_languages); // Skipping first line output ("Available languages...")
            array_pop($tesseract_languages); // Skipping last line (empty)
        }
        array_walk($tesseract_languages, 'trim_value');
        // Add langauges to plugin_config and write it back to db
        $plugin_config['tesseract_languages'] = $tesseract_languages;
        set_plugin_config('ocrstream', $plugin_config);        
    }
    return $tesseract_languages;
}

/**
 * Trim value, use for arrays
 * 
 * @param string $value
 */
function trim_value(&$value) {
    $value = trim($value);
}

/**
 * Check if PHP Session is started
 * 
 * @return boolean
 */
function is_session_started() {
    if ( php_sapi_name() !== 'cli' ) {<commit_msg>undo skip last line of language output<commit_after><?php

/**
 * Get temp dir for ocrstream, create one if none exists
 * 
 * @return string OCRStream plugin temp directory
 */
function get_ocr_temp_dir() {
    $temp_dir = get_temp_dir();
    if(!is_dir($temp_dir . "/ocrstream_plugin")){
        mkdir($temp_dir . "/ocrstream_plugin",0777);        
    }
    $ocr_temp_dir = $temp_dir . "/ocrstream_plugin";
    return ($ocr_temp_dir);
}

/**
 * Checks if OS is Windows
 * 
 * @return boolean
 */
function is_windows() {
    $os = php_uname('s');
    if (stristr($os, 'win')) {
        if (stristr($os, 'Darwin')) {
            $is_windows = false;
        } else {
            $is_windows = true;
        }
    } else {
        $is_windows = false;
    }
    return $is_windows;
}

/**
 * Get path to tesseract executable
 * 
 * @global string $tesseract_path Tesseract directory path
 * @return string
 */
function get_tesseract_fullpath() {
    global $tesseract_path;
    if (is_windows()) {
        $tesseract_fullpath = $tesseract_path . '\tesseract.exe';
    } else {
        $tesseract_fullpath = $tesseract_path . '/tesseract';
    }
    return $tesseract_fullpath;
}

/**
 * Checks if tesseract executable exists
 * 
 * @return boolean 
 */
function is_tesseract_installed() {
    $tesseract_fullpath = get_tesseract_fullpath();
    if (file_exists($tesseract_fullpath)) {
        $tesseract_installed = true;
    } else {
        $tesseract_installed = false;
    }
    return $tesseract_installed;
}

/**
 * Get tesseract and leptonica version information
 * 
 * @return array tesseract version, leptonica version, tesseract_is_old
 */
function get_tesseract_version() {
    // Check if version information is already in plugin_config
    $plugin_config = get_plugin_config('ocrstream');
    if (isset($plugin_config['tesseract_version']) && isset($plugin_config['leptonica_version'])) {
        // Read version from config and return array
        $tesseract_version = $plugin_config['tesseract_version'];
        $leptonica_version = $plugin_config['leptonica_version'];
    } else {
        $tesseract_fullpath = get_tesseract_fullpath();
        $tesseract_version_command = run_command(escapeshellarg($tesseract_fullpath) . ' -v', true);
        $tesseract_version_output = explode("\n", $tesseract_version_command);
        if (stristr($tesseract_version_output [0], 'libtiff.so.5')) { // Skipping error output in first line if libftiff/liblept version mismatch
            $tesseract_version = $tesseract_version_output[1];
            $leptonica_version = $tesseract_version_output[2];
        } else {
            $tesseract_version = $tesseract_version_output[0];
            $leptonica_version = $tesseract_version_output[1];
        }
        $version_regex = "/([0-9.]+)/mi"; 
        preg_match_all($version_regex, $tesseract_version, $tess_matches);
        preg_match_all($version_regex, $leptonica_version, $lept_matches);
        $tesseract_version = trim($tess_matches[0][0]);
        $leptonica_version = trim($lept_matches[0][0]);
        // Add version to plugin_config and write it back to db
        $plugin_config['tesseract_version'] = $tesseract_version;
        $plugin_config['leptonica_version'] = $leptonica_version;
        set_plugin_config('ocrstream', $plugin_config);
    }
    if (version_compare($tesseract_version, '3.03') >= 0) {
        $tesseract_version_is_old = false;
    } else {
        $tesseract_version_is_old = true;
    }       
    return array($tesseract_version, $leptonica_version, $tesseract_version_is_old);
}

/**
 * Get available languages for tesseract-ocr
 * 
 * Returns an array of languages that are currently installed into the /TESSDATA directory.
 * Language codes use ISO 639-2 standard.
 * 
 * @link https://code.google.com/p/tesseract-ocr/downloads/list tesseract language data downloads
 * @return array 
 */
function get_tesseract_languages() {
    // Check if languages are already in plugin_config
    $plugin_config = get_plugin_config('ocrstream');
    if (isset($plugin_config['tesseract_languages'])) {
        // Read languages from config and return array
        $tesseract_languages = $plugin_config['tesseract_languages'];
    } else {
        // Get languages via tesseract cli
        $tesseract_fullpath = get_tesseract_fullpath();
        $tesseract_language_command = run_command(escapeshellarg($tesseract_fullpath) . ' --list-langs', true);
        $tesseract_languages = explode("\n", $tesseract_language_command);
        if (stristr($tesseract_languages [0], 'libtiff.so.5')) { // Skipping additional line if libftiff version does not match liblept version
            array_shift($tesseract_languages);
            array_shift($tesseract_languages);
        } else {
            array_shift($tesseract_languages); // Skipping first line output ("Available languages...")
        }
        array_walk($tesseract_languages, 'trim_value');
        // Add langauges to plugin_config and write it back to db
        $plugin_config['tesseract_languages'] = $tesseract_languages;
        set_plugin_config('ocrstream', $plugin_config);        
    }
    return $tesseract_languages;
}

/**
 * Trim value, use for arrays
 * 
 * @param string $value
 */
function trim_value(&$value) {
    $value = trim($value);
}

/**
 * Check if PHP Session is started
 * 
 * @return boolean
 */
function is_session_started() {
    if ( php_sapi_name() !== 'cli' ) {
{'fraction_non_alphanumeric': 0.11716546893477668, 'fraction_numerical': 0.004730282907304649, 'mean_word_length': 3.2778210116731517, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '102064', 'n_tokens_mistral': 3523, 'n_tokens_neox': 3610, 'n_words': 722}