<commit_before>"""
Add a few specific filters to Jinja2.
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import re
from functools import wraps
import datetime
from calendar import timegm

from flask import Flask
from flask.ext import babel
from jinja2 import Markup, escape, evalcontextfilter
from pytz import utc
from babel.dates import DateTimePattern, format_timedelta, parse_pattern
import bleach
from werkzeug.routing import BuildError

from ..core.util import local_dt, utc_dt, slugify
from .util import url_for


def autoescape(filter_func):
  """
  Decorator to autoescape result from filters.
  """
  @evalcontextfilter
  @wraps(filter_func)
  def _autoescape(eval_ctx, *args, **kwargs):
    result = filter_func(*args, **kwargs)
    if eval_ctx.autoescape:
      result = Markup(result)
    return result
  return _autoescape


@autoescape
def nl2br(value):
  """
  Replace newlines with <br />.
  """
  result = escape(value).replace(u'\n', Markup(u'<br />\n'))
  return result

_PARAGRAPH_RE = re.compile(r'(?:\r\n|\r|\n){2,}')


@autoescape
def paragraphs(value):
  """
  Blank lines delimitates paragraphs.
  """
  result = u'\n\n'.join(
    (u'<p>{}</p>'.format(p.strip().replace('\n', Markup('<br />\n')))
     for p in _PARAGRAPH_RE.split(escape(value))))
  return result


def labelize(s):
  return u" ".join([ w.capitalize() for w in s.split(u"_") ])


def filesize(d):
  if not isinstance(d, int):
    d = int(d)

  if d < 1000:
    s = "%d&nbsp;B" % d

  elif d < 1e4:
    s = "%.1f&nbsp;kB" % (d / 1e3)
  elif d < 1e6:
    s = "%.0f&nbsp;kB" % (d / 1e3)

  elif d < 1e7:
    s = "%.1f&nbsp;MB" % (d / 1e6)
  elif d < 1e9:
    s = "%.0f&nbsp;MB" % (d / 1e6)

  elif d < 1e10:
    s = "%.1f&nbsp;GB" % (d / 1e9)

  else:
    s = "%.0f&nbsp;GB" % (d / 1e9)

  return Markup(s)

def roughsize(size, above=20, mod=10):
  """
  6 -> '6'
  15 -> '15'
  134 -> '130+'
  """
  if size < above:
    return unicode(size)

  return u'{:d}+'.format(size - size % mod)


def age(dt, now=None, add_direction=True):
  # Fail silently for now XXX
  if not dt:
    return ""

  if not now:
    now = datetime.datetime.utcnow()

  dt = utc_dt(dt)
  now = utc_dt(now)

  # don't use (flask.ext.)babel.format_timedelta: as of Flask-Babel 0.9 it
  # doesn't support "threshold" arg.
  return format_timedelta((dt - now),
                          locale=babel.get_locale(),
                          granularity='minute',
                          threshold=0.9,<commit_msg>age filter: add param `date_threshold`<commit_after>"""
Add a few specific filters to Jinja2.
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import re
from functools import wraps
import datetime
from calendar import timegm

from flask import Flask
from flask.ext import babel
from jinja2 import Markup, escape, evalcontextfilter
from pytz import utc
from babel.dates import DateTimePattern, format_timedelta, parse_pattern
import bleach
from werkzeug.routing import BuildError

from ..core.util import local_dt, utc_dt, slugify
from .util import url_for


def autoescape(filter_func):
  """
  Decorator to autoescape result from filters.
  """
  @evalcontextfilter
  @wraps(filter_func)
  def _autoescape(eval_ctx, *args, **kwargs):
    result = filter_func(*args, **kwargs)
    if eval_ctx.autoescape:
      result = Markup(result)
    return result
  return _autoescape


@autoescape
def nl2br(value):
  """
  Replace newlines with <br />.
  """
  result = escape(value).replace(u'\n', Markup(u'<br />\n'))
  return result

_PARAGRAPH_RE = re.compile(r'(?:\r\n|\r|\n){2,}')


@autoescape
def paragraphs(value):
  """
  Blank lines delimitates paragraphs.
  """
  result = u'\n\n'.join(
    (u'<p>{}</p>'.format(p.strip().replace('\n', Markup('<br />\n')))
     for p in _PARAGRAPH_RE.split(escape(value))))
  return result


def labelize(s):
  return u" ".join([ w.capitalize() for w in s.split(u"_") ])


def filesize(d):
  if not isinstance(d, int):
    d = int(d)

  if d < 1000:
    s = "%d&nbsp;B" % d

  elif d < 1e4:
    s = "%.1f&nbsp;kB" % (d / 1e3)
  elif d < 1e6:
    s = "%.0f&nbsp;kB" % (d / 1e3)

  elif d < 1e7:
    s = "%.1f&nbsp;MB" % (d / 1e6)
  elif d < 1e9:
    s = "%.0f&nbsp;MB" % (d / 1e6)

  elif d < 1e10:
    s = "%.1f&nbsp;GB" % (d / 1e9)

  else:
    s = "%.0f&nbsp;GB" % (d / 1e9)

  return Markup(s)

def roughsize(size, above=20, mod=10):
  """
  6 -> '6'
  15 -> '15'
  134 -> '130+'
  """
  if size < above:
    return unicode(size)

  return u'{:d}+'.format(size - size % mod)


def age(dt, now=None, add_direction=True, date_threshold=None):
  """
  :param dt: :class:`datetime<datetime.datetime>` instance to format

  :param now: :class:`datetime<datetime.datetime>` instance to compare to `dt`

  :param add_direction: if `True`, will add "in" or "ago" (example for `en`
  locale) to time difference `dt - now`, i.e "in 9 min." or " 9min. ago"

  :param date_threshold: above threshold, will use a formated date instead of
  elapsed time indication. Supported values: "day".
  """
  # Fail silently for now XXX
  if not dt:
    return ""

  if not now:
    now = datetime.datetime.utcnow()

  locale = babel.get_locale()
  dt = utc_dt(dt)
  now = utc_dt(now)
  delta = dt - now

  if date_threshold is not None:
    dy, dw, dd = dt_cal = dt.isocalendar()
    ny, nw, nd = now_cal =now.isocalendar()

    if dt_cal != now_cal:
      # not same day
      remove_year = dy != dy
      date_fmt = locale.date_formats['long'].pattern
      time_fmt = locale.time_formats['short'].pattern
      fmt = locale.datetime_formats['medium']

      if remove_year:
        date_fmt.replace('y', '').strip()
        date_fmt.replace('y', '')
        # remove leading or trailing spaces, comma, etc...
        date_fmt = re.sub(u'^[^A-Za-z]*|[^A-Za-z]*$', u'', date_fmt)

      fmt = fmt.format(time_fmt, date_fmt)
      return babel.format_datetime(dt, format=fmt)

  # don't use (flask.ext.)babel.format_timedelta: as of Flask-Babel 0.9 it
  # doesn't support "threshold" arg.
  return format_timedelta(delta,
                          locale=locale,
                          granularity='minute',
                          threshold=0.9,
{'fraction_non_alphanumeric': 0.14634146341463414, 'fraction_numerical': 0.018491949625378607, 'mean_word_length': 3.029543994861914, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '882656', 'n_tokens_mistral': 2421, 'n_tokens_neox': 2313, 'n_words': 626}