<commit_before><?php

$connection = false;

db_connect($hostname, $dbuser, $dbpass, $database);
// catch parameters
if (isset($_GET['dataset']) && !empty($_GET['dataset']))
    $dataset = $_GET['dataset'];
else
    $dataset = "tibet";
if (isset($_GET['query']) && !empty($_GET['query']))
    $query = $_GET['query'];
else
    $query = "";
if (isset($_GET['exclude']) && !empty($_GET['exclude']))
    $exclude = $_GET['exclude'];
else
    $exclude = "";
if (isset($_GET['from_user_name']) && !empty($_GET['from_user_name']))
    $from_user_name = $_GET['from_user_name'];
else
    $from_user_name = "";
if (isset($_GET['samplesize']) && !empty($_GET['samplesize']))
    $samplesize = $_GET['samplesize'];
else
    $samplesize = "1000";
if (isset($_GET['minf']) && preg_match("/^\d+$/",$_GET['minf'])!==false)
    $minf = $_GET['minf'];
else
    $minf = 2;
if (isset($_GET['startdate']) && !empty($_GET['startdate']))
    $startdate = $_GET['startdate'];
else
    $startdate = strftime("%Y-%m-%d", date('U') - (86400 * 2));
if (isset($_GET['enddate']) && !empty($_GET['enddate']))
    $enddate = $_GET['enddate'];
else
    $enddate = strftime("%Y-%m-%d", date('U') - 86400);
$u_startdate = $u_enddate = 0;

if (isset($_GET['whattodo']) && !empty($_GET['whattodo']))
    $whattodo = $_GET['whattodo'];
else
    $whattodo = "";

if (isset($_GET['keywordToTrack']) && !empty($_GET['keywordToTrack']))
    $keywordToTrack = trim(strtolower($_GET['keywordToTrack']));
else
    $keywordToTrack = "";

if (isset($_GET['minimumCowordFrequencyOverall']))
    $minimumCowordFrequencyOverall = $_GET['minimumCowordFrequencyOverall'];
else
    $minimumCowordFrequencyOverall = 10;

if (isset($_GET['minimumCowordFrequencyOverall']))
    $minimumCowordFrequencyInterval = $_GET['minimumCowordFrequencyInterval'];
else
    $minimumCowordFrequencyInterval = 0;

if (isset($_GET['showvis']) && !empty($_GET['showvis']))
    $showvis = $_GET['showvis'];
else
    $showvis = "";

$interval = "daily";
if (isset($_REQUEST['interval'])) {
    if (in_array($_REQUEST['interval'], array('hourly', 'daily', 'weekly', 'monthly', 'yearly', 'overall', 'custom')))
        $interval = $_REQUEST['interval'];
}
// check custom interval
$intervalDates = array();
if ($interval == "custom" && isset($_REQUEST['customInterval'])) {
    $intervalDates = explode(';', $_REQUEST['customInterval']);
    $firstDate = $lastDate = false;
    foreach ($intervalDates as $k => $date) {
        $date = trim($date);
        if (empty($date))
            continue;
        $intervalDates[$k] = $date;
        if (!preg_match("/^\d{4}-\d{2}-\d{2}$/", $intervalDates[$k]))
            die("<font size='+1' color='red'>custom interval not in right format</font>: YYYY-MM-DD;YYYY-MM-DD;...;YYYY-MM-DD");
        if (!$firstDate)
            $firstDate = $date;
        $lastDate = $date;
    }

    if ($firstDate != $startdate)
        die("<font size='+1' color='red'>custom interval should have the same start date as the selection</font>");
    if ($lastDate > $enddate)
        die("<font size='+1' color='red'>custom interval should have the same end date as the selection</font>");
}


$keywords = array();
$esc = array();

// define punctuation symbols
$punctuation = array("\s", "\.", ",", "!", "\?", ":", ";", "\/", "\\", "#", "@", "&", "\^", "\$", "\|", "`", "~", "=", "\+", "\*", "\"", "'", "\(", "\)", "\]", "\[", "\{", "\}", "<", ">", "�");

// define the type of output
$tsv = array("hashtag", "mention", "retweet", "urls", "hosts", "user", "user-mention"); // these analyses will output tsv files
$network = array("coword", "interaction");   // these analyses will output network files

$titles = array(
    "hashtag" => "Hashtag frequency",
    "retweet" => "Retweet frequency",
    "user" => "User frequency",
    "mention" => "Mention (@username) frequency",
    "urls" => "URL frequency",
    "hosts" => "host frequency"
);


if (!empty($whattodo)) {
    if (in_array($whattodo, $tsv) !== false)
        get_file($whattodo);
}

// return the desired file
function get_file($what) {
    validate_all_variables();

    // get filename (this also validates the data)
    global $database;
    $filename = get_filename($what);

    generate($what, $filename);

    // redirect to file
    $location = str_replace("index.php", "", ANALYSIS_URL) . str_replace("#", "%23", $filename);
    if (defined('LOCATION'))
        $location = LOCATION . $location;
    header("Content-type: text/csv");
    header("Location: $location");
}

/*
 * @var $toget specifies fieldname
 * @var $table specifies table name
 */

function frequencyTable($table, $toget) {
    global $esc, $intervalDates;
    $results = array();
    $sql = "SELECT COUNT($table.$toget) AS count, $table.$toget AS toget, ";
    $sql .= sqlInterval();
    $sql .= "FROM " . $esc['mysql']['dataset'] . "_$table $table, " . $esc['mysql']['dataset'] . "_tweets t ";
    $sql .= "WHERE t.id = $table.tweet_id AND ";
    $sql .= sqlSubset();
    $sql .= " GROUP BY toget, datepart ORDER BY datepart ASC, count DESC";
    $rec = mysql_query($sql);
    $date = false;
    while ($res = mysql_fetch_assoc($rec)) {
        if ($res['count'] > $esc['shell']['minf']) {
            if (!empty($intervalDates))
                $date = groupByInterval($res['datepart']);
            else
                $date = $res['datepart'];
            $results[$date][$res['toget']] = $res['count'];
        }
    }
    return $results;
}

// here further sqlSubset selection is constructed
function sqlSubset($table = "t", $period = FALSE) {
    error_reporting(E_ALL);
    global $esc;
    $sql = "";
    if (!empty($esc['mysql']['from_user_name'])) {
        if (strstr($esc['mysql']['from_user_name'], "AND") !== false) {
            $subqueries = explode(" AND ", $esc['mysql']['from_user_name']);
            foreach ($subqueries as $subquery) {
                $sql .= "LOWER($table.from_user_name) LIKE '%" . $subquery . "%' AND ";
            }
        } elseif (strstr($esc['mysql']['from_user_name'], "OR") !== false) {
            $subqueries = explode(" OR ", $esc['mysql']['from_user_name']);
            $sql .= "(";
            foreach ($subqueries as $subquery) {
                $sql .= "LOWER($table.from_user_name) LIKE '%" . $subquery . "%' OR ";
            }
            $sql = substr($sql, 0, -3) . ") AND ";
        } else {
            $sql .= "LOWER($table.from_user_name) LIKE '%" . $esc['mysql']['from_user_name'] . "%' AND ";
        }
    }
    if (!empty($esc['mysql']['query'])) {
        if (strstr($esc['mysql']['query'], "AND") !== false) {
            $subqueries = explode(" AND ", $esc['mysql']['query']);
            foreach ($subqueries as $subquery) {
                $sql .= "LOWER($table.text) LIKE '%" . $subquery . "%' AND ";
            }
        } elseif (strstr($esc['mysql']['query'], "OR") !== false) {
            $subqueries = explode(" OR ", $esc['mysql']['query']);
            $sql .= "(";<commit_msg>usernames should not be wildcard queries<commit_after><?php

$connection = false;

db_connect($hostname, $dbuser, $dbpass, $database);
// catch parameters
if (isset($_GET['dataset']) && !empty($_GET['dataset']))
    $dataset = $_GET['dataset'];
else
    $dataset = "tibet";
if (isset($_GET['query']) && !empty($_GET['query']))
    $query = $_GET['query'];
else
    $query = "";
if (isset($_GET['exclude']) && !empty($_GET['exclude']))
    $exclude = $_GET['exclude'];
else
    $exclude = "";
if (isset($_GET['from_user_name']) && !empty($_GET['from_user_name']))
    $from_user_name = $_GET['from_user_name'];
else
    $from_user_name = "";
if (isset($_GET['samplesize']) && !empty($_GET['samplesize']))
    $samplesize = $_GET['samplesize'];
else
    $samplesize = "1000";
if (isset($_GET['minf']) && preg_match("/^\d+$/",$_GET['minf'])!==false)
    $minf = $_GET['minf'];
else
    $minf = 2;
if (isset($_GET['startdate']) && !empty($_GET['startdate']))
    $startdate = $_GET['startdate'];
else
    $startdate = strftime("%Y-%m-%d", date('U') - (86400 * 2));
if (isset($_GET['enddate']) && !empty($_GET['enddate']))
    $enddate = $_GET['enddate'];
else
    $enddate = strftime("%Y-%m-%d", date('U') - 86400);
$u_startdate = $u_enddate = 0;

if (isset($_GET['whattodo']) && !empty($_GET['whattodo']))
    $whattodo = $_GET['whattodo'];
else
    $whattodo = "";

if (isset($_GET['keywordToTrack']) && !empty($_GET['keywordToTrack']))
    $keywordToTrack = trim(strtolower($_GET['keywordToTrack']));
else
    $keywordToTrack = "";

if (isset($_GET['minimumCowordFrequencyOverall']))
    $minimumCowordFrequencyOverall = $_GET['minimumCowordFrequencyOverall'];
else
    $minimumCowordFrequencyOverall = 10;

if (isset($_GET['minimumCowordFrequencyOverall']))
    $minimumCowordFrequencyInterval = $_GET['minimumCowordFrequencyInterval'];
else
    $minimumCowordFrequencyInterval = 0;

if (isset($_GET['showvis']) && !empty($_GET['showvis']))
    $showvis = $_GET['showvis'];
else
    $showvis = "";

$interval = "daily";
if (isset($_REQUEST['interval'])) {
    if (in_array($_REQUEST['interval'], array('hourly', 'daily', 'weekly', 'monthly', 'yearly', 'overall', 'custom')))
        $interval = $_REQUEST['interval'];
}
// check custom interval
$intervalDates = array();
if ($interval == "custom" && isset($_REQUEST['customInterval'])) {
    $intervalDates = explode(';', $_REQUEST['customInterval']);
    $firstDate = $lastDate = false;
    foreach ($intervalDates as $k => $date) {
        $date = trim($date);
        if (empty($date))
            continue;
        $intervalDates[$k] = $date;
        if (!preg_match("/^\d{4}-\d{2}-\d{2}$/", $intervalDates[$k]))
            die("<font size='+1' color='red'>custom interval not in right format</font>: YYYY-MM-DD;YYYY-MM-DD;...;YYYY-MM-DD");
        if (!$firstDate)
            $firstDate = $date;
        $lastDate = $date;
    }

    if ($firstDate != $startdate)
        die("<font size='+1' color='red'>custom interval should have the same start date as the selection</font>");
    if ($lastDate > $enddate)
        die("<font size='+1' color='red'>custom interval should have the same end date as the selection</font>");
}


$keywords = array();
$esc = array();

// define punctuation symbols
$punctuation = array("\s", "\.", ",", "!", "\?", ":", ";", "\/", "\\", "#", "@", "&", "\^", "\$", "\|", "`", "~", "=", "\+", "\*", "\"", "'", "\(", "\)", "\]", "\[", "\{", "\}", "<", ">", "�");

// define the type of output
$tsv = array("hashtag", "mention", "retweet", "urls", "hosts", "user", "user-mention"); // these analyses will output tsv files
$network = array("coword", "interaction");   // these analyses will output network files

$titles = array(
    "hashtag" => "Hashtag frequency",
    "retweet" => "Retweet frequency",
    "user" => "User frequency",
    "mention" => "Mention (@username) frequency",
    "urls" => "URL frequency",
    "hosts" => "host frequency"
);


if (!empty($whattodo)) {
    if (in_array($whattodo, $tsv) !== false)
        get_file($whattodo);
}

// return the desired file
function get_file($what) {
    validate_all_variables();

    // get filename (this also validates the data)
    global $database;
    $filename = get_filename($what);

    generate($what, $filename);

    // redirect to file
    $location = str_replace("index.php", "", ANALYSIS_URL) . str_replace("#", "%23", $filename);
    if (defined('LOCATION'))
        $location = LOCATION . $location;
    header("Content-type: text/csv");
    header("Location: $location");
}

/*
 * @var $toget specifies fieldname
 * @var $table specifies table name
 */

function frequencyTable($table, $toget) {
    global $esc, $intervalDates;
    $results = array();
    $sql = "SELECT COUNT($table.$toget) AS count, $table.$toget AS toget, ";
    $sql .= sqlInterval();
    $sql .= "FROM " . $esc['mysql']['dataset'] . "_$table $table, " . $esc['mysql']['dataset'] . "_tweets t ";
    $sql .= "WHERE t.id = $table.tweet_id AND ";
    $sql .= sqlSubset();
    $sql .= " GROUP BY toget, datepart ORDER BY datepart ASC, count DESC";
    $rec = mysql_query($sql);
    $date = false;
    while ($res = mysql_fetch_assoc($rec)) {
        if ($res['count'] > $esc['shell']['minf']) {
            if (!empty($intervalDates))
                $date = groupByInterval($res['datepart']);
            else
                $date = $res['datepart'];
            $results[$date][$res['toget']] = $res['count'];
        }
    }
    return $results;
}

// here further sqlSubset selection is constructed
function sqlSubset($table = "t", $period = FALSE) {
    error_reporting(E_ALL);
    global $esc;
    $sql = "";
    if (!empty($esc['mysql']['from_user_name'])) {
        if (strstr($esc['mysql']['from_user_name'], "AND") !== false) {
            $subqueries = explode(" AND ", $esc['mysql']['from_user_name']);
            foreach ($subqueries as $subquery) {
                $sql .= "LOWER($table.from_user_name) = LOWER('" . $subquery . "') AND ";
            }
        } elseif (strstr($esc['mysql']['from_user_name'], "OR") !== false) {
            $subqueries = explode(" OR ", $esc['mysql']['from_user_name']);
            $sql .= "(";
            foreach ($subqueries as $subquery) {
                $sql .= "LOWER($table.from_user_name) = LOWER('" . $subquery . "') OR ";
            }
            $sql = substr($sql, 0, -3) . ") AND ";
        } else {
            $sql .= "LOWER($table.from_user_name) = LOWER('" . $esc['mysql']['from_user_name'] . "') AND ";
        }
    }
    if (!empty($esc['mysql']['query'])) {
        if (strstr($esc['mysql']['query'], "AND") !== false) {
            $subqueries = explode(" AND ", $esc['mysql']['query']);
            foreach ($subqueries as $subquery) {
                $sql .= "LOWER($table.text) LIKE '%" . $subquery . "%' AND ";
            }
        } elseif (strstr($esc['mysql']['query'], "OR") !== false) {
            $subqueries = explode(" OR ", $esc['mysql']['query']);
            $sql .= "(";
{'fraction_non_alphanumeric': 0.22547825463924911, 'fraction_numerical': 0.004298918105610088, 'mean_word_length': 3.3728070175438596, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6060954', 'n_tokens_mistral': 4666, 'n_tokens_neox': 4527, 'n_words': 953}