<commit_before># a View consists of Data and a cursor
# it also renders

renderLine = (lineData, onto, options = {}) ->
  options.cursors ?= {}
  options.highlights ?= {}
  defaultStyle = options.defaultStyle || ''

  # ideally this takes up space but is unselectable (uncopyable)
  cursorChar = ' '

  # array of dicts:
  # {
  #   text: text
  #   column: column
  #   cursor: true/false
  #   highlighted: true/false
  # }
  line = []

  # add cursor if at end
  if lineData.length of options.cursors
    lineData.push cursorChar

  # if still empty, put a newline
  if lineData.length == 0
    lineData.push '\n'

  for char, i in lineData
    info = {
      column: i
    }

    x = char

    if char == '\n'
      x = '<br/>'
      if i of options.cursors
        x = cursorChar + x

    if i of options.cursors
      info.cursor = true
    if i of options.highlights
      info.highlighted = true

    info.text = x
    line.push info

  do onto.empty

  acc = ''
  style = defaultStyle
  for x in line
    mystyle = defaultStyle
    if x.cursor
      mystyle = 'cursor'
    if x.highlighted
      mystyle = 'highlight'
    if mystyle != style
      onto.append $('<span>').html(acc).addClass(style)
      style = mystyle
      acc = ''
    acc += x.text

  if acc.length
    onto.append $('<span>').html(acc).addClass(style)

  # NOTE: the following renders each character separately
  # it makes it so we can click individual characters to move the cursor
  # however, it seems to make rendering laggy

  # for x in line
  #   mystyle = defaultStyle
  #   if x.cursor
  #     mystyle = 'cursor'
  #   if x.highlighted
  #     mystyle = 'highlight'

  #   charDiv = $('<span>').html(x.text).addClass(mystyle)
  #   if options.onclick?
  #     charDiv.on('click', options.onclick.bind @, x)
  #   onto.append charDiv
<commit_msg>convoluted code for displaying links...<commit_after># a View consists of Data and a cursor
# it also renders

renderLine = (lineData, onto, options = {}) ->
  options.cursors ?= {}
  options.highlights ?= {}
  defaultStyle = options.defaultStyle || ''

  # ideally this takes up space but is unselectable (uncopyable)
  cursorChar = ' '

  # array of dicts:
  # {
  #   text: text
  #   column: column
  #   cursor: true/false
  #   highlighted: true/false
  # }
  line = []

  # add cursor if at end
  if lineData.length of options.cursors
    lineData.push cursorChar

  # if still empty, put a newline
  if lineData.length == 0
    lineData.push '\n'

  for char, i in lineData
    info = {
      column: i
    }

    x = char

    if char == '\n'
      x = '<br/>'
      if i of options.cursors
        x = cursorChar + x

    if i of options.cursors
      info.cursor = true
    if i of options.highlights
      info.highlighted = true

    info.text = x
    line.push info

  # collect set of words, { word: word, start: start, end: end }
  words = []
  word = ''
  word_start = 0

  lineData.push ' ' # to make end condition easier
  for char, i in lineData
    if char == '\n' or char == ' '
      if i != word_start
        words.push {
          word: word
          start: word_start
          end: i - 1
        }
      word_start = i + 1
      word = ''
    else
      word += char

  urlRegex = /^https?:\/\/[^\s]+\.[^\s]+$/
  url_words = words.filter (w) ->
    return urlRegex.test w.word

  for url_word in url_words
    line[url_word.start].url_start = url_word.word
    line[url_word.end].url_end = url_word.word
    # console.log 'url word', url_word

  do onto.empty

  acc = ''
  style = defaultStyle
  url_onto = null

  for x in line
    mystyle = defaultStyle
    if x.cursor
      mystyle = 'cursor'
    if x.highlighted
      mystyle = 'highlight'

    if x.url_start
      if acc.length
        onto.append $('<span>').html(acc).addClass(style)
        acc = ''
      url_onto = $('<a>').attr('href', x.url_start)

    if mystyle != style
      if acc.length
        if url_onto
          url_onto.append $('<span>').html(acc).addClass(style)
        else
          onto.append $('<span>').html(acc).addClass(style)
        acc = ''
      style = mystyle

    acc += x.text

    if x.url_end
      url_onto.append $('<span>').html(acc).addClass(style)
      acc = ''

      onto.append url_onto
      url_onto = null

  if acc.length
    onto.append $('<span>').html(acc).addClass(style)

  # NOTE: the following renders each character separately
  # it makes it so we can click individual characters to move the cursor
  # however, it seems to make rendering laggy

  # for x in line
  #   mystyle = defaultStyle
  #   if x.cursor
  #     mystyle = 'cursor'
  #   if x.highlighted
  #     mystyle = 'highlight'

  #   charDiv = $('<span>').html(x.text).addClass(mystyle)
  #   if options.onclick?
  #     charDiv.on('click', options.onclick.bind @, x)
  #   onto.append charDiv

{'fraction_non_alphanumeric': 0.11787778695293147, 'fraction_numerical': 0.001032204789430223, 'mean_word_length': 2.225699067909454, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3585317', 'n_tokens_mistral': 1637, 'n_tokens_neox': 1566, 'n_words': 470}