<commit_before><?php

require_once('core/twitter.php');

function array_get($array, $key, $default=null) {
  if (isset($array[$key])) return $array[$key];
  return $default;
}

function echo_global_var($varname) {
  return $GLOBALS[$varname];
}

function pluralise($word, $count, $show) {
  if ($show) $word = "{$count} {$word}";
  return $word . (($count > 1) ? 's' : '');
}

function local_date($format, $timestamp, $offset) {
  return gmdate($format, $timestamp + $offset);
}

function format_time($time, $offset) {
  $units = array(
    'sec'  => array(1, 60),
    'min'  => array(60, 3600),
    'hour' => array(3600, 86400),
  );
  $delta = time() - $time;
  $ret = '';
  foreach ($units as $key => $value) {
    if ($delta < $value[1]) {
      $ret = pluralise($key, floor($delta / $value[0]), true) . ' ago';
      break;
    }
  }
  if ($ret == '') {
    $ret = local_date("d F Y", $time, $offset);
  }
  return $ret;
}

function format_tweet($tweet) {
  global $settings;

  $list_pattern = '/@([a-zA-Z0-9_]+)\/([a-zA-Z0-9_]+)/';
  $list_replace = '@\1/<a href="'.make_path('/list').'/\1/\2">\2</a>';
  $user_pattern = '/@([a-zA-Z0-9_]+)/';
  $user_replace = '@<a href="'.make_path('/user/show').'/\1">\1</a>';
  $url_pattern = '/((http|https)\:\/\/[a-zA-Z0-9_\-\+\.\/\?\&\$\@\:\=]+)/';
  $url_replace = '<a target="_blank" href="\1">\1</a>';
  $tag_pattern = '/(#[a-zA-Z0-9_]+)/';

  $highlight_pattern = $settings->highlight;
  if (! empty($highlight_pattern))
    $tweet = keyword_highlight($highlight_pattern, $tweet);

  $tweet = preg_replace($list_pattern, $list_replace, $tweet);
  $tweet = preg_replace($user_pattern, $user_replace, $tweet);
  $tweet = preg_replace($url_pattern, $url_replace, $tweet);
  $tweet = preg_replace_callback($tag_pattern, 'hashtag_encode', $tweet);

  return $tweet;
}

function hashtag_encode($match) {
  return "<a href='".make_path("/search/query")."?q=" . urlencode($match[1]) . "'>". $match[1] . "</a>";
}

function keyword_highlight($pattern, $tweet) {
  preg_match_all("/$pattern/", $tweet, $match);
  $match = array_unique($match[0]);
  if (! empty($match)) {<commit_msg>update url pattern<commit_after><?php

require_once('core/twitter.php');

function array_get($array, $key, $default=null) {
  if (isset($array[$key])) return $array[$key];
  return $default;
}

function echo_global_var($varname) {
  return $GLOBALS[$varname];
}

function pluralise($word, $count, $show) {
  if ($show) $word = "{$count} {$word}";
  return $word . (($count > 1) ? 's' : '');
}

function local_date($format, $timestamp, $offset) {
  return gmdate($format, $timestamp + $offset);
}

function format_time($time, $offset) {
  $units = array(
    'sec'  => array(1, 60),
    'min'  => array(60, 3600),
    'hour' => array(3600, 86400),
  );
  $delta = time() - $time;
  $ret = '';
  foreach ($units as $key => $value) {
    if ($delta < $value[1]) {
      $ret = pluralise($key, floor($delta / $value[0]), true) . ' ago';
      break;
    }
  }
  if ($ret == '') {
    $ret = local_date("d F Y", $time, $offset);
  }
  return $ret;
}

function format_tweet($tweet) {
  global $settings;

  $list_pattern = '/@([a-zA-Z0-9_]+)\/([a-zA-Z0-9_]+)/';
  $list_replace = '@\1/<a href="'.make_path('/list').'/\1/\2">\2</a>';
  $user_pattern = '/@([a-zA-Z0-9_]+)/';
  $user_replace = '@<a href="'.make_path('/user/show').'/\1">\1</a>';
  $url_pattern = '/((http|https)\:\/\/[a-zA-Z0-9_\-\+\.\/\?\&\$\@\:\=\#\%]+)/';
  $url_replace = '<a target="_blank" href="\1">\1</a>';
  $tag_pattern = '/(#[a-zA-Z0-9_]+)/';

  $highlight_pattern = $settings->highlight;
  if (! empty($highlight_pattern))
    $tweet = keyword_highlight($highlight_pattern, $tweet);

  $tweet = preg_replace($list_pattern, $list_replace, $tweet);
  $tweet = preg_replace($user_pattern, $user_replace, $tweet);
  $tweet = preg_replace($url_pattern, $url_replace, $tweet);
  $tweet = preg_replace_callback($tag_pattern, 'hashtag_encode', $tweet);

  return $tweet;
}

function hashtag_encode($match) {
  return "<a href='".make_path("/search/query")."?q=" . urlencode($match[1]) . "'>". $match[1] . "</a>";
}

function keyword_highlight($pattern, $tweet) {
  preg_match_all("/$pattern/", $tweet, $match);
  $match = array_unique($match[0]);
  if (! empty($match)) {
{'fraction_non_alphanumeric': 0.2513507164669955, 'fraction_numerical': 0.019732205778717406, 'mean_word_length': 4.0630202140309155, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '295849', 'n_tokens_mistral': 1721, 'n_tokens_neox': 1670, 'n_words': 294}