<commit_before><?php

require_once __DIR__ . '/geoPHP/geoPHP.inc'; // geoPHP library

error_reporting(E_ALL);
ini_set("max_execution_time", 0);       // capture script want unlimited execution time

function pdo_connect() {
    global $dbuser, $dbpass, $database, $hostname;

    $dbh = new PDO("mysql:host=$hostname;dbname=$database;charset=utf8mb4", $dbuser, $dbpass, array(PDO::MYSQL_ATTR_INIT_COMMAND => "set sql_mode='ALLOW_INVALID_DATES'"));
    $dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);

    return $dbh;
}

function geophp_sane() {
    $sane = true;
    if (!geoPHP::geosInstalled()) {
        $msg = "geoPHP needs the GEOS and its PHP extension (please download it at: http://trac.osgeo.org/geos/)";
        $sane = false;
    } else {
        // Is the Digital Methods lab in Amsterdam? 
        $point_lng = 4.893346; $point_lat = 52.369042;
        $sw_lng = 4.768520; $sw_lat = 52.321629;
        $ne_lng = 5.017270; $ne_lat = 52.425129;
        $sane = coordinatesInsideBoundingBox($point_lng, $point_lat, $sw_lng, $sw_lat, $ne_lng, $ne_lat);
        if (!$sane) {
            $msg = "geoPHP/GEOS library seems broken. searching on area will not work";
        }
    }
    if (!$sane) {
        if (defined('CAPTURE')) {
            logit(CAPTURE . ".error.log", $msg);
        } else {
            logit("cli", $msg);
        }
    }
    return $sane;
}

function create_error_logs() {
    $dbh = pdo_connect();

    $sql = 'create table if not exists tcat_error_ratelimit ( id bigint auto_increment, type varchar(32), start datetime not null, end datetime not null, tweets bigint not null, primary key(id), index(type), index(start), index(end) )';
    $h = $dbh->prepare($sql);
    $h->execute();

    $sql = 'create table if not exists tcat_error_gap ( id bigint auto_increment, type varchar(32), start datetime not null, end datetime not null, primary key(id), index(type), index(start), index(end) )';
    $h = $dbh->prepare($sql);
    $h->execute();
}

// Enclose identifier in backticks; escape backticks inside by doubling them.
function quoteIdent($field) {
    return "`" . str_replace("`", "``", $field) . "`";
}

function create_bin($bin_name, $dbh = false) {
    try {

        $dbh = pdo_connect();

        $sql = "CREATE TABLE IF NOT EXISTS " . quoteIdent($bin_name . "_hashtags") . " (
            `id` int(11) NOT NULL AUTO_INCREMENT,
            `tweet_id` bigint(20) NOT NULL,
            `created_at` datetime,
            `from_user_name` varchar(255),
            `from_user_id` bigint,
            `text` varchar(255),
            PRIMARY KEY (`id`),
                    KEY `created_at` (`created_at`),
                    KEY `tweet_id` (`tweet_id`),
                    KEY `text` (`text`),
                    KEY `from_user_name` (`from_user_name`)
            ) ENGINE=MyISAM  DEFAULT CHARSET=utf8mb4";

        $create_hashtags = $dbh->prepare($sql);
        $create_hashtags->execute();

        $sql = "CREATE TABLE IF NOT EXISTS " . quoteIdent($bin_name . "_withheld") . " (
            `id` int(11) NOT NULL AUTO_INCREMENT,
            `tweet_id` bigint(20) NOT NULL,
            `user_id` bigint(20),
            `country` char(5),
            PRIMARY KEY (`id`),
                    KEY `user_id` (`user_id`),
                    KEY `tweet_id` (`user_id`),
                    KEY `country` (`country`)
            ) ENGINE=MyISAM  DEFAULT CHARSET=utf8mb4";

        $create_withheld = $dbh->prepare($sql);
        $create_withheld->execute();


        $sql = "CREATE TABLE IF NOT EXISTS " . quoteIdent($bin_name . "_places") . " (
            `id` varchar(32) NOT NULL,
            `tweet_id` bigint(20) NOT NULL,
            PRIMARY KEY (`id`, `tweet_id`)
            ) ENGINE=MyISAM  DEFAULT CHARSET=utf8mb4";

        $create_places = $dbh->prepare($sql);
        $create_places->execute();



        $sql = "CREATE TABLE IF NOT EXISTS " . quoteIdent($bin_name . "_mentions") . " (
            `id` int(11) NOT NULL AUTO_INCREMENT,
            `tweet_id` bigint(20) NOT NULL,
            `created_at` datetime,
            `from_user_name` varchar(255),<commit_msg>Fixed columns forming tweet_id index on *_withheld tables.<commit_after><?php

require_once __DIR__ . '/geoPHP/geoPHP.inc'; // geoPHP library

error_reporting(E_ALL);
ini_set("max_execution_time", 0);       // capture script want unlimited execution time

function pdo_connect() {
    global $dbuser, $dbpass, $database, $hostname;

    $dbh = new PDO("mysql:host=$hostname;dbname=$database;charset=utf8mb4", $dbuser, $dbpass, array(PDO::MYSQL_ATTR_INIT_COMMAND => "set sql_mode='ALLOW_INVALID_DATES'"));
    $dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);

    return $dbh;
}

function geophp_sane() {
    $sane = true;
    if (!geoPHP::geosInstalled()) {
        $msg = "geoPHP needs the GEOS and its PHP extension (please download it at: http://trac.osgeo.org/geos/)";
        $sane = false;
    } else {
        // Is the Digital Methods lab in Amsterdam? 
        $point_lng = 4.893346; $point_lat = 52.369042;
        $sw_lng = 4.768520; $sw_lat = 52.321629;
        $ne_lng = 5.017270; $ne_lat = 52.425129;
        $sane = coordinatesInsideBoundingBox($point_lng, $point_lat, $sw_lng, $sw_lat, $ne_lng, $ne_lat);
        if (!$sane) {
            $msg = "geoPHP/GEOS library seems broken. searching on area will not work";
        }
    }
    if (!$sane) {
        if (defined('CAPTURE')) {
            logit(CAPTURE . ".error.log", $msg);
        } else {
            logit("cli", $msg);
        }
    }
    return $sane;
}

function create_error_logs() {
    $dbh = pdo_connect();

    $sql = 'create table if not exists tcat_error_ratelimit ( id bigint auto_increment, type varchar(32), start datetime not null, end datetime not null, tweets bigint not null, primary key(id), index(type), index(start), index(end) )';
    $h = $dbh->prepare($sql);
    $h->execute();

    $sql = 'create table if not exists tcat_error_gap ( id bigint auto_increment, type varchar(32), start datetime not null, end datetime not null, primary key(id), index(type), index(start), index(end) )';
    $h = $dbh->prepare($sql);
    $h->execute();
}

// Enclose identifier in backticks; escape backticks inside by doubling them.
function quoteIdent($field) {
    return "`" . str_replace("`", "``", $field) . "`";
}

function create_bin($bin_name, $dbh = false) {
    try {

        $dbh = pdo_connect();

        $sql = "CREATE TABLE IF NOT EXISTS " . quoteIdent($bin_name . "_hashtags") . " (
            `id` int(11) NOT NULL AUTO_INCREMENT,
            `tweet_id` bigint(20) NOT NULL,
            `created_at` datetime,
            `from_user_name` varchar(255),
            `from_user_id` bigint,
            `text` varchar(255),
            PRIMARY KEY (`id`),
                    KEY `created_at` (`created_at`),
                    KEY `tweet_id` (`tweet_id`),
                    KEY `text` (`text`),
                    KEY `from_user_name` (`from_user_name`)
            ) ENGINE=MyISAM  DEFAULT CHARSET=utf8mb4";

        $create_hashtags = $dbh->prepare($sql);
        $create_hashtags->execute();

        $sql = "CREATE TABLE IF NOT EXISTS " . quoteIdent($bin_name . "_withheld") . " (
            `id` int(11) NOT NULL AUTO_INCREMENT,
            `tweet_id` bigint(20) NOT NULL,
            `user_id` bigint(20),
            `country` char(5),
            PRIMARY KEY (`id`),
                    KEY `user_id` (`user_id`),
                    KEY `tweet_id` (`tweet_id`),
                    KEY `country` (`country`)
            ) ENGINE=MyISAM  DEFAULT CHARSET=utf8mb4";

        $create_withheld = $dbh->prepare($sql);
        $create_withheld->execute();


        $sql = "CREATE TABLE IF NOT EXISTS " . quoteIdent($bin_name . "_places") . " (
            `id` varchar(32) NOT NULL,
            `tweet_id` bigint(20) NOT NULL,
            PRIMARY KEY (`id`, `tweet_id`)
            ) ENGINE=MyISAM  DEFAULT CHARSET=utf8mb4";

        $create_places = $dbh->prepare($sql);
        $create_places->execute();



        $sql = "CREATE TABLE IF NOT EXISTS " . quoteIdent($bin_name . "_mentions") . " (
            `id` int(11) NOT NULL AUTO_INCREMENT,
            `tweet_id` bigint(20) NOT NULL,
            `created_at` datetime,
            `from_user_name` varchar(255),
{'fraction_non_alphanumeric': 0.14251439539347407, 'fraction_numerical': 0.020633397312859885, 'mean_word_length': 2.440775897647544, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5081488', 'n_tokens_mistral': 2975, 'n_tokens_neox': 2752, 'n_words': 658}