<commit_before>import EDTF = require('edtf')
import edtfy = require('edtfy')
import debug = require('./debug.ts')

// import escapeStringRegexp = require('escape-string-regexp')

const months = require('../gen/dateparser-data.json')
const months_re = new RegExp(Object.keys(months).sort((a, b) => b.length - a.length).join('|'), 'i')

/*
regex = {
  My: new RegExp('^(' + months.english.join('|') + ')\\s([0-9]{3,})$', 'i'),
  Mdy: new RegExp('^(' + months.english.join('|') + ')\\s([0-9]{1,2})\\s*,\\s*([0-9]{3,})$', 'i'),
  dMy: new RegExp('^([0-9]{1,2})\\.?\\s+(' + months.english.join('|') + ')\\s+([0-9]{3,})$', 'i'),
}
*/

const SPRING = 21
const WINTER = 24

function seasonize(date) {
  if (date.type === 'date' && typeof date.month === 'number' && date.month >= SPRING && date.month <= WINTER && !date.day) {
    date.type = 'season'
    date.season = (date.month - SPRING) + 1
    delete date.month
  }
  return date
}

function doubt(date, state) {
  if (state.uncertain) date.uncertain = true
  if (state.approximate) date.approximate = true
  return date
}

function normalize_edtf(date) {
  let year, month, day

  switch (date.type) {
    case 'Date':
      [ year, month, day ] = date.values
      if (typeof month === 'number') month += 1
      return doubt({ type: 'date', year, month, day}, {approximate: date.approximate || date.unspecified, uncertain: date.uncertain })

    case 'Interval':
      // tslint:disable-next-line:no-magic-numbers
      if (date.values.length !== 2) throw new Error(JSON.stringify(date))
      const from = date.values[0] ? normalize_edtf(date.values[0]) : { type: 'open' }
      const to = date.values[1] ? normalize_edtf(date.values[1]) : { type: 'open' }
      return { type: 'interval', from, to }

    case 'Season':
      [ year, month ] = date.values
      if (month < SPRING || month > WINTER) throw new Error(`Unexpected season ${month}`)
      return seasonize({ type: 'date', year, month })

    case 'List':
      return { type: 'list', dates: date.values.map(normalize_edtf) }

    default:
      throw new Error(JSON.stringify(date))
  }
}

function upgrade_edtf(date) {
  return date
    .replace(/unknown/g, '*')
    .replace(/u/g, 'X')
    .replace(/(\?~)|(~\?)/g, '%')
    .replace(/open/g, '')
    .replace(/\.\./g, '')
    .replace(/y/g, 'Y')
}

function is_valid_month(month) {
  if (month >= 1 && month <= 12) return true // tslint:disable-line:no-magic-numbers
  if (month >= 21 && month <= 24) return true // tslint:disable-line:no-magic-numbers

  return false
}

function parse(value, mayrecurse = true) {
  value = value.trim()

  let parsed, m

  debug('dateparser: parsing', value, mayrecurse)

  if (mayrecurse && value === '') return { type: 'open' }

  if (mayrecurse && (m = /^\[(.+)\]\s*(.+)$/.exec(value))) {
    const [ , _orig, _year ] = m
    const year = parse(_year, false)
    const orig = parse(_orig, false)
    if (year.type === 'date' && orig.type === 'date') return {...year, ...{ orig } }
  }

  if (mayrecurse && (m = /^(-?[0-9]+)\s*\[(-?[0-9]+)\]$/.exec(value))) {
    const [ , _year, _orig ] = m
    const year = parse(_year, false)
    const orig = parse(_orig, false)
    if (year.type === 'date' && orig.type === 'date') return {...year, ...{ orig } }
  }

  if (mayrecurse && (m = /^\[(-?[0-9]+)\]$/.exec(value))) {
    const [ , _orig ] = m
    const orig = parse(_orig, false)
    if (orig.type === 'date') return { ...{ orig } }
  }

  // 747
  if (mayrecurse && (m = /^([a-zA-Z]+)\s+([0-9]+)(?:--|-|–)([0-9]+)[, ]\s*([0-9]+)$/.exec(value))) {
    const [ , month, day1, day2, year ] = m

    const from = parse(`${month} ${day1} ${year}`, false)
    const to = parse(`${month} ${day2} ${year}`, false)

    if (from.type === 'date' && to.type === 'date') return { type: 'interval', from, to }
  }

  // 747, January 30–February 3, 1989
  if (mayrecurse && (m = /^([a-zA-Z]+\s+[0-9]+)(?:--|-|–)([a-zA-Z]+\s+[0-9]+)[, ]\s*([0-9]+)$/.exec(value))) {
    const [ , date1, date2, year ] = m

    const from = parse(`${date1} ${year}`, false)
    const to = parse(`${date2} ${year}`, false)

    if (from.type === 'date' && to.type === 'date') return { type: 'interval', from, to }
  }

  // 746
  if (mayrecurse && (m = /^([0-9]+)(?:--|-|–)([0-9]+)\s+([a-zA-Z]+)\s+([0-9]+)$/.exec(value))) {
    const [ , day1, day2, month, year ] = m

    const from = parse(`${month} ${day1} ${year}`, false)
    const to = parse(`${month} ${day2} ${year}`, false)

    if (from.type === 'date' && to.type === 'date') return { type: 'interval', from, to }
  }

  const state = {approximate: false, uncertain: false}
  const exactish = value.replace(/[~?]+$/, match => {
    state.approximate = match.indexOf('~') >= 0
    state.uncertain = match.indexOf('?') >= 0
    return ''
  })

  // these assume a sensible d/m/y format by default. There's no sane way to guess between m/d/y and d/m/y, and m/d/y is
  // just wrong. https://en.wikipedia.org/wiki/Date_format_by_country
  if (m = /^(-?[0-9]{3,})([-\/\.])([0-9]{1,2})(\2([0-9]{1,2}))?$/.exec(exactish)) {
    const [ , _year, , _month, , _day ] = m
    const year = parseInt(_year)
    let month = parseInt(_month)
    let day = _day ? parseInt(_day) : undefined

    // swap day/month for our American brethren
    if (is_valid_month(day) && !is_valid_month(month)) [day, month] = [month, day]

    if (is_valid_month(month)) return seasonize(doubt({ type: 'date', year, month, day }, state))
  }

  if (m = /^([0-9]{1,2})([-\/\.])([0-9]{1,2})(\2([0-9]{3,}))$/.exec(exactish)) {
    const [ , _day, , _month, , _year ] = m
    const year = parseInt(_year)
    let month = parseInt(_month)
    let day = parseInt(_day)

    // swap day/month for our American brethren
    if (is_valid_month(day) && !is_valid_month(month)) [day, month] = [month, day]

    if (is_valid_month(month)) return seasonize(doubt({ type: 'date', year, month, day }, state))
  }

  if (exactish.match(/^-?[0-9]+$/)) {
    return doubt({ type: 'date', year: parseInt(exactish) }, state)
  }

  try {
    // https://github.com/inukshuk/edtf.js/issues/5
    parsed = normalize_edtf(EDTF.parse(upgrade_edtf(value
      .replace(/_|--/, '/')
      .replace(/(\s+|T)[0-9]{2}:[0-9]{2}:[0-9]{2}(Z|\+[0-9]{2}:?[0-9]{2})?$/, '')
    )))
  } catch (err) {
    parsed = null
  }

  if (!parsed) {
    try {
      parsed = normalize_edtf(EDTF.parse(edtfy(value
        .normalize('NFC')
        .replace(/\. /, ' ') // 8. july 2011
        .replace(months_re, _ => months[_.toLowerCase()] || _)
      )))
    } catch (err) {
      parsed = null<commit_msg>really, really nuts<commit_after>import EDTF = require('edtf')
import edtfy = require('edtfy')
import debug = require('./debug.ts')

// import escapeStringRegexp = require('escape-string-regexp')

const months = require('../gen/dateparser-data.json')
const months_re = new RegExp(Object.keys(months).sort((a, b) => b.length - a.length).join('|'), 'i')

/*
regex = {
  My: new RegExp('^(' + months.english.join('|') + ')\\s([0-9]{3,})$', 'i'),
  Mdy: new RegExp('^(' + months.english.join('|') + ')\\s([0-9]{1,2})\\s*,\\s*([0-9]{3,})$', 'i'),
  dMy: new RegExp('^([0-9]{1,2})\\.?\\s+(' + months.english.join('|') + ')\\s+([0-9]{3,})$', 'i'),
}
*/

const SPRING = 21
const WINTER = 24

function seasonize(date) {
  if (date.type === 'date' && typeof date.month === 'number' && date.month >= SPRING && date.month <= WINTER && !date.day) {
    date.type = 'season'
    date.season = (date.month - SPRING) + 1
    delete date.month
  }
  return date
}

function doubt(date, state) {
  if (state.uncertain) date.uncertain = true
  if (state.approximate) date.approximate = true
  return date
}

function normalize_edtf(date) {
  let year, month, day

  switch (date.type) {
    case 'Date':
      [ year, month, day ] = date.values
      if (typeof month === 'number') month += 1
      return doubt({ type: 'date', year, month, day}, {approximate: date.approximate || date.unspecified, uncertain: date.uncertain })

    case 'Interval':
      // tslint:disable-next-line:no-magic-numbers
      if (date.values.length !== 2) throw new Error(JSON.stringify(date))
      const from = date.values[0] ? normalize_edtf(date.values[0]) : { type: 'open' }
      const to = date.values[1] ? normalize_edtf(date.values[1]) : { type: 'open' }
      return { type: 'interval', from, to }

    case 'Season':
      [ year, month ] = date.values
      if (month < SPRING || month > WINTER) throw new Error(`Unexpected season ${month}`)
      return seasonize({ type: 'date', year, month })

    case 'List':
      return { type: 'list', dates: date.values.map(normalize_edtf) }

    default:
      throw new Error(JSON.stringify(date))
  }
}

function upgrade_edtf(date) {
  return date
    .replace(/unknown/g, '*')
    .replace(/u/g, 'X')
    .replace(/(\?~)|(~\?)/g, '%')
    .replace(/open/g, '')
    .replace(/\.\./g, '')
    .replace(/y/g, 'Y')
}

function is_valid_month(month, allowseason) {
  if (month >= 1 && month <= 12) return true // tslint:disable-line:no-magic-numbers
  if (allowseason && month >= 21 && month <= 24) return true // tslint:disable-line:no-magic-numbers

  return false
}

function parse(value, mayrecurse = true) {
  value = value.trim()

  let parsed, m

  debug('dateparser: parsing', value, mayrecurse)

  if (mayrecurse && value === '') return { type: 'open' }

  if (mayrecurse && (m = /^\[(.+)\]\s*(.+)$/.exec(value))) {
    const [ , _orig, _year ] = m
    const year = parse(_year, false)
    const orig = parse(_orig, false)
    if (year.type === 'date' && orig.type === 'date') return {...year, ...{ orig } }
  }

  if (mayrecurse && (m = /^(-?[0-9]+)\s*\[(-?[0-9]+)\]$/.exec(value))) {
    const [ , _year, _orig ] = m
    const year = parse(_year, false)
    const orig = parse(_orig, false)
    if (year.type === 'date' && orig.type === 'date') return {...year, ...{ orig } }
  }

  if (mayrecurse && (m = /^\[(-?[0-9]+)\]$/.exec(value))) {
    const [ , _orig ] = m
    const orig = parse(_orig, false)
    if (orig.type === 'date') return { ...{ orig } }
  }

  // 747
  if (mayrecurse && (m = /^([a-zA-Z]+)\s+([0-9]+)(?:--|-|–)([0-9]+)[, ]\s*([0-9]+)$/.exec(value))) {
    const [ , month, day1, day2, year ] = m

    const from = parse(`${month} ${day1} ${year}`, false)
    const to = parse(`${month} ${day2} ${year}`, false)

    if (from.type === 'date' && to.type === 'date') return { type: 'interval', from, to }
  }

  // 747, January 30–February 3, 1989
  if (mayrecurse && (m = /^([a-zA-Z]+\s+[0-9]+)(?:--|-|–)([a-zA-Z]+\s+[0-9]+)[, ]\s*([0-9]+)$/.exec(value))) {
    const [ , date1, date2, year ] = m

    const from = parse(`${date1} ${year}`, false)
    const to = parse(`${date2} ${year}`, false)

    if (from.type === 'date' && to.type === 'date') return { type: 'interval', from, to }
  }

  // 746
  if (mayrecurse && (m = /^([0-9]+)(?:--|-|–)([0-9]+)\s+([a-zA-Z]+)\s+([0-9]+)$/.exec(value))) {
    const [ , day1, day2, month, year ] = m

    const from = parse(`${month} ${day1} ${year}`, false)
    const to = parse(`${month} ${day2} ${year}`, false)

    if (from.type === 'date' && to.type === 'date') return { type: 'interval', from, to }
  }

  const state = {approximate: false, uncertain: false}
  const exactish = value.replace(/[~?]+$/, match => {
    state.approximate = match.indexOf('~') >= 0
    state.uncertain = match.indexOf('?') >= 0
    return ''
  })

  // these assume a sensible d/m/y format by default. There's no sane way to guess between m/d/y and d/m/y, and m/d/y is
  // just wrong. https://en.wikipedia.org/wiki/Date_format_by_country
  if (m = /^(-?[0-9]{3,})([-\/\.])([0-9]{1,2})(\2([0-9]{1,2}))?$/.exec(exactish)) {
    const [ , _year, , _month, , _day ] = m
    const year = parseInt(_year)
    let month = parseInt(_month)
    let day = _day ? parseInt(_day) : undefined

    // swap day/month for our American brethren
    if (day && is_valid_month(day, false) && !is_valid_month(month, false)) [day, month] = [month, day]

    if (is_valid_month(month, !day)) return seasonize(doubt({ type: 'date', year, month, day }, state))
  }

  if (m = /^([0-9]{1,2})([-\/\.])([0-9]{1,2})(\2([0-9]{3,}))$/.exec(exactish)) {
    const [ , _day, , _month, , _year ] = m
    const year = parseInt(_year)
    let month = parseInt(_month)
    let day = parseInt(_day)

    // swap day/month for our American brethren
    if (is_valid_month(day, false) && !is_valid_month(month, false)) [day, month] = [month, day]

    if (is_valid_month(month, false)) return seasonize(doubt({ type: 'date', year, month, day }, state))
  }

  if (exactish.match(/^-?[0-9]+$/)) {
    return doubt({ type: 'date', year: parseInt(exactish) }, state)
  }

  try {
    // https://github.com/inukshuk/edtf.js/issues/5
    parsed = normalize_edtf(EDTF.parse(upgrade_edtf(value
      .replace(/_|--/, '/')
      .replace(/(\s+|T)[0-9]{2}:[0-9]{2}:[0-9]{2}(Z|\+[0-9]{2}:?[0-9]{2})?$/, '')
    )))
  } catch (err) {
    parsed = null
  }

  if (!parsed) {
    try {
      parsed = normalize_edtf(EDTF.parse(edtfy(value
        .normalize('NFC')
        .replace(/\. /, ' ') // 8. july 2011
        .replace(months_re, _ => months[_.toLowerCase()] || _)
      )))
    } catch (err) {
      parsed = null
{'fraction_non_alphanumeric': 0.23799787975162806, 'fraction_numerical': 0.020596698470392246, 'mean_word_length': 2.9804098854731764, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5285566', 'n_tokens_mistral': 5170, 'n_tokens_neox': 4982, 'n_words': 1108}