<commit_before>cssParse = require 'css-parse'
langCodes = require './lang_codes.js'

# from http://phpjs.org/functions/strip_tags/
`function strip_tags(input, allowed) {
  // http://kevin.vanzonneveld.net
  // +   original by: <NAME> (http://kevin.vanzonneveld.net)
  // +   improved by: <NAME>
  // +      input by: Pul
  // +   bugfixed by: <NAME> (http://kevin.vanzonneveld.net)
  // +   bugfixed by: <NAME>
  // +      input by: Alex
  // +   bugfixed by: <NAME> (http://kevin.vanzonneveld.net)
  // +      input by: <NAME>
  // +   improved by: <NAME> (http://kevin.vanzonneveld.net)
  // +      input by: <NAME> (http://brett-zamir.me)
  // +   bugfixed by: <NAME> (http://kevin.vanzonneveld.net)
  // +   bugfixed by: <NAME>
  // +      input by: <NAME>
  // +   bugfixed by: <NAME> (http://kevin.vanzonneveld.net)
  // +   bugfixed by: <NAME>
  // +      input by: <NAME>
  // +    revised by: <NAME> (http://blog.kukawski.pl/)
  // *     example 1: strip_tags('<p>Kevin</p> <br /><b>van</b> <i>Zonneveld</i>', '<i><b>');
  // *     returns 1: 'Kevin <b>van</b> <i>Zonneveld</i>'
  // *     example 2: strip_tags('<p>Kevin <img src="someimage.png" onmouseover="someFunction()">van <i>Zonneveld</i></p>', '<p>');
  // *     returns 2: '<p><NAME></p>'
  // *     example 3: strip_tags("<a href='http://kevin.vanzonneveld.net'><NAME></a>", "<a>");
  // *     returns 3: '<a href='http://kevin.vanzonneveld.net'><NAME></a>'
  // *     example 4: strip_tags('1 < 5 5 > 1');
  // *     returns 4: '1 < 5 5 > 1'
  // *     example 5: strip_tags('1 <br/> 1');
  // *     returns 5: '1  1'
  // *     example 6: strip_tags('1 <br/> 1', '<br>');
  // *     returns 6: '1  1'
  // *     example 7: strip_tags('1 <br/> 1', '<br><br/>');
  // *     returns 7: '1 <br/> 1'
  allowed = (((allowed || "") + "").toLowerCase().match(/<[a-z][a-z0-9]*>/g) || []).join(''); // making sure the allowed arg is a string containing only tags in lowercase (<a><b><c>)
  var tags = /<\/?([a-z][a-z0-9]*)\b[^>]*>/gi,
    commentsAndPhpTags = /<!--[\s\S]*?-->|<\?(?:php)?[\s\S]*?\?>/gi;
  return input.replace(commentsAndPhpTags, '').replace(tags, function($0, $1) {
    return allowed.indexOf('<' + $1.toLowerCase() + '>') > -1 ? $0 : '';
  });
}`

reOpenSync = /<sync/i
reLineEnding = /\r\n?|\n/g
reBrokenTag = /<[a-z]*[^>]*<[a-z]*/g
reStartTime = /<sync[^>]+?start[^=]*=[^0-9]*([0-9]*)["^0-9"]*/i
reBr = /<br[^>]*>/ig
reStyle = /<style[^>]*>([\s\S]*?)<\/style[^>]*>/i
reComment = /(<!--|-->)/g

_sort = (langItem) ->
  langItem.sort((a, b) ->
    if (res = a.startTime - b.startTime) is 0
      return a.endTime - b.endTime
    else
      return res
  )

_makeEndTime = (langItem) ->
  i = langItem.length
  while i--
    item = langItem[i]
    langItem[i-1]?.endTime = item.startTime
    if !item.contents or item.contents is '&nbsp;'
      langItem.splice i, 1
    else
      delete langItem[i].contents
  langItem

_mergeMultiLanguages = (arr) ->
  dict = {}
  i = arr.length 
  ret = []

  for val, i in arr
    key = val.startTime+','+val.endTime
    if dict[key] isnt undefined
      idx = dict[key]
      for lang, content of val.languages
        ret[idx].languages[lang] = content
    else
      ret.push val
      dict[key] = ret.length-1

  return ret



class Parser
  @defaultLanguage: {className: 'KRCC', lang: 'ko', reClassName: /class[^=]*?=["']?(KRCC)["']?/i}
  @defaultLanguageCode: 'ko' 
  availableLanguages: null
  errors: null

  constructor: () ->
    @errors = []
    @availableLanguages = []

  _parse: (str) ->
    lineNum = 1
    ret = []
    tempRet = {}

    while true
      startTagIdx = str.search(reOpenSync)
      break if nextStartTagIdx <= 0 || startTagIdx < 0
      nextStartTagIdx = str.slice(startTagIdx+1).search(reOpenSync)+1
      if nextStartTagIdx > 0
        element = str.slice(startTagIdx, startTagIdx+nextStartTagIdx)
      else
        element = str.slice(startTagIdx)

      lineNum += str.slice(0, startTagIdx).match(reLineEnding)?.length or 0 
      if isBroken = reBrokenTag.test(element)
        e = new Error('ERROR_BROKEN_TAGS')
        e.line = lineNum
        e.context = element
        @errors.push(e)

      str = str.slice(startTagIdx+nextStartTagIdx)

      startTime = +element.match(reStartTime)?[1] or -1

      if startTime < 0
        e = new Error('ERROR_INVALID_TIME')
        e.line = lineNum
        e.context = element
        @errors.push(e)

      lineNum += element.match(reLineEnding)?.length or 0

      for lang in @availableLanguages when lang.reClassName.test element
        lang = lang.lang
        break;

      lang or= @defaultLanguageCode
      element = element.replace(reLineEnding, '')
      element = element.replace(reBr, "\n")
      innerText = strip_tags(element).trim()
      lang = @getLanguage(element)
      item = {startTime, languages: {}, contents: innerText}
      item.languages[lang] = innerText

      tempRet[lang] or= []
      tempRet[lang].push(item)

    for lang, langItem of tempRet
      langItem = _sort(langItem)
      langItem = _makeEndTime(langItem)

      ret = ret.concat(langItem)

    ret = _mergeMultiLanguages(ret)
    ret = _sort(ret)
    return ret

  getLanguage: (element) ->
    for lang in @availableLanguages when lang.reClassName.test element
      return lang.lang
    return Parser.defaultLanguage.lang

  getAvailableLanguages: (str) ->
    try
      matched = str.match(reStyle)?[1] or ''
      matched = matched.replace(reComment, '')
      parsed = cssParse matched

      for rule in parsed.stylesheet.rules
        # currently support single language, class selectors only
        selector = rule.selectors[0]
        if selector?[0] is '.'
          for declaration in rule.declarations
            if declaration.property.toLowerCase() is 'lang'
              className = selector.slice(1) # pass dot (.ENCC -> ENCC)
              lang = declaration.value.slice(0,2)
              if ~langCodes.indexOf lang
                language = {
                  className: className
                  lang: lang
                  reClassName: new RegExp("class[^=]*?=[\"']?(#{className})['\"]?", 'i')
                }
                @availableLanguages.push language
              else
                throw Error()
    catch e
      @errors.push error = new Error('ERROR_INVALID_LANGUAGE')
      @availableLanguages.push Parser.defaultLanguage # parse failed
      return

  parse: (str) ->
    @getAvailableLanguages(str)
    result = @_parse(str)
    return {result, errors: @errors}

module.exports = Parser<commit_msg>minor change<commit_after>cssParse = require 'css-parse'
langCodes = require './lang_codes.js'

# from http://phpjs.org/functions/strip_tags/
`function strip_tags(input, allowed) {
  // http://kevin.vanzonneveld.net
  // +   original by: <NAME> (http://kevin.vanzonneveld.net)
  // +   improved by: <NAME>
  // +      input by: Pul
  // +   bugfixed by: <NAME> (http://kevin.vanzonneveld.net)
  // +   bugfixed by: <NAME>
  // +      input by: Alex
  // +   bugfixed by: <NAME> (http://kevin.vanzonneveld.net)
  // +      input by: <NAME>
  // +   improved by: <NAME> (http://kevin.vanzonneveld.net)
  // +      input by: <NAME> (http://brett-zamir.me)
  // +   bugfixed by: Kevin <NAME>veld (http://kevin.vanzonneveld.net)
  // +   bugfixed by: <NAME>
  // +      input by: <NAME>
  // +   bugfixed by: <NAME> (http://kevin.vanzonneveld.net)
  // +   bugfixed by: <NAME>
  // +      input by: <NAME>
  // +    revised by: <NAME> (http://blog.kukawski.pl/)
  // *     example 1: strip_tags('<p>Kevin</p> <br /><b>van</b> <i>Zonneveld</i>', '<i><b>');
  // *     returns 1: 'Kevin <b>van</b> <i>Zonneveld</i>'
  // *     example 2: strip_tags('<p>Kevin <img src="someimage.png" onmouseover="someFunction()">van <i>Zonneveld</i></p>', '<p>');
  // *     returns 2: '<p><NAME></p>'
  // *     example 3: strip_tags("<a href='http://kevin.vanzonneveld.net'><NAME></a>", "<a>");
  // *     returns 3: '<a href='http://kevin.vanzonneveld.net'><NAME></a>'
  // *     example 4: strip_tags('1 < 5 5 > 1');
  // *     returns 4: '1 < 5 5 > 1'
  // *     example 5: strip_tags('1 <br/> 1');
  // *     returns 5: '1  1'
  // *     example 6: strip_tags('1 <br/> 1', '<br>');
  // *     returns 6: '1  1'
  // *     example 7: strip_tags('1 <br/> 1', '<br><br/>');
  // *     returns 7: '1 <br/> 1'
  allowed = (((allowed || "") + "").toLowerCase().match(/<[a-z][a-z0-9]*>/g) || []).join(''); // making sure the allowed arg is a string containing only tags in lowercase (<a><b><c>)
  var tags = /<\/?([a-z][a-z0-9]*)\b[^>]*>/gi,
    commentsAndPhpTags = /<!--[\s\S]*?-->|<\?(?:php)?[\s\S]*?\?>/gi;
  return input.replace(commentsAndPhpTags, '').replace(tags, function($0, $1) {
    return allowed.indexOf('<' + $1.toLowerCase() + '>') > -1 ? $0 : '';
  });
}`

reOpenSync = /<sync/i
reLineEnding = /\r\n?|\n/g
reBrokenTag = /<[a-z]*[^>]*<[a-z]*/g
reStartTime = /<sync[^>]+?start[^=]*=[^0-9]*([0-9]*)["^0-9"]*/i
reBr = /<br[^>]*>/ig
reStyle = /<style[^>]*>([\s\S]*?)<\/style[^>]*>/i
reComment = /(<!--|-->)/g

_sort = (langItem) ->
  langItem.sort((a, b) ->
    if (res = a.startTime - b.startTime) is 0
      return a.endTime - b.endTime
    else
      return res
  )

_makeEndTime = (langItem) ->
  i = langItem.length
  while i--
    item = langItem[i]
    langItem[i-1]?.endTime = item.startTime
    if !item.contents or item.contents is '&nbsp;'
      langItem.splice i, 1
    else
      delete langItem[i].contents
  langItem

_mergeMultiLanguages = (arr) ->
  dict = {}
  i = arr.length 
  ret = []

  for val, i in arr
    key = val.startTime+','+val.endTime
    if (idx = dict[key]) isnt undefined
      for lang, content of val.languages
        ret[idx].languages[lang] = content
    else
      ret.push val
      dict[key] = ret.length-1

  return ret



class Parser
  @defaultLanguage: {className: 'KRCC', lang: 'ko', reClassName: /class[^=]*?=["']?(KRCC)["']?/i}
  @defaultLanguageCode: 'ko' 
  availableLanguages: null
  errors: null

  constructor: () ->
    @errors = []
    @availableLanguages = []

  _parse: (str) ->
    lineNum = 1
    ret = []
    tempRet = {}

    while true
      startTagIdx = str.search(reOpenSync)
      break if nextStartTagIdx <= 0 || startTagIdx < 0
      nextStartTagIdx = str.slice(startTagIdx+1).search(reOpenSync)+1
      if nextStartTagIdx > 0
        element = str.slice(startTagIdx, startTagIdx+nextStartTagIdx)
      else
        element = str.slice(startTagIdx)

      lineNum += str.slice(0, startTagIdx).match(reLineEnding)?.length or 0 
      if isBroken = reBrokenTag.test(element)
        e = new Error('ERROR_BROKEN_TAGS')
        e.line = lineNum
        e.context = element
        @errors.push(e)

      str = str.slice(startTagIdx+nextStartTagIdx)

      startTime = +element.match(reStartTime)?[1] or -1

      if startTime < 0
        e = new Error('ERROR_INVALID_TIME')
        e.line = lineNum
        e.context = element
        @errors.push(e)

      lineNum += element.match(reLineEnding)?.length or 0

      for lang in @availableLanguages when lang.reClassName.test element
        lang = lang.lang
        break;

      lang or= @defaultLanguageCode
      element = element.replace(reLineEnding, '')
      element = element.replace(reBr, "\n")
      innerText = strip_tags(element).trim()
      lang = @getLanguage(element)
      item = {startTime, languages: {}, contents: innerText}
      item.languages[lang] = innerText

      tempRet[lang] or= []
      tempRet[lang].push(item)

    for lang, langItem of tempRet
      langItem = _sort(langItem)
      langItem = _makeEndTime(langItem)

      ret = ret.concat(langItem)

    ret = _mergeMultiLanguages(ret)
    ret = _sort(ret)
    return ret

  getLanguage: (element) ->
    for lang in @availableLanguages when lang.reClassName.test element
      return lang.lang
    return Parser.defaultLanguage.lang

  getAvailableLanguages: (str) ->
    try
      matched = str.match(reStyle)?[1] or ''
      matched = matched.replace(reComment, '')
      parsed = cssParse matched

      for rule in parsed.stylesheet.rules
        # currently support single language, class selectors only
        selector = rule.selectors[0]
        if selector?[0] is '.'
          for declaration in rule.declarations
            if declaration.property.toLowerCase() is 'lang'
              className = selector.slice(1) # pass dot (.ENCC -> ENCC)
              lang = declaration.value.slice(0,2)
              if ~langCodes.indexOf lang
                language = {
                  className: className
                  lang: lang
                  reClassName: new RegExp("class[^=]*?=[\"']?(#{className})['\"]?", 'i')
                }
                @availableLanguages.push language
              else
                throw Error()
    catch e
      @errors.push error = new Error('ERROR_INVALID_LANGUAGE')
      @availableLanguages.push Parser.defaultLanguage # parse failed
      return

  parse: (str) ->
    @getAvailableLanguages(str)
    result = @_parse(str)
    return {result, errors: @errors}

module.exports = Parser
{'fraction_non_alphanumeric': 0.1775636613902271, 'fraction_numerical': 0.010858759654354974, 'mean_word_length': 2.4452054794520546, 'pattern_counts': {'":': 0, '<': 147, '<?xml version=': 0, '>': 169, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2530541', 'n_tokens_mistral': 4689, 'n_tokens_neox': 4549, 'n_words': 980}