<commit_before>define [
  'underscore'
  'jquery'
  'backbone'
  'jsSHA'
  'cs!gh-book/uuid'
  'cs!gh-book/binary-file'
  'cs!models/content/module'
  'cs!collections/content'
  'cs!models/utils'
], (_, $, Backbone, jsSHA, uuid, BinaryFile, ModuleModel, allContent, Utils) ->


  # An `<img src>` has the form `data:image/png;base64,.....`.
  # This method splits the src into a usable form and generates a resource path
  # using the hash of the bits.
  #
  # This is called twice:
  #
  # 1. When the body text is changed
  # 2. When the model is serialized
  #
  # The reason it is called twice is because Aloha strips the `data-src` attribute.
  # Since Aloha strips the data-src attribute the only data in the HTML is the
  # Base64-encoded string that represents the image.
  #
  # So, when the body text is changed a new BinaryFile is added to allContent
  # with the path `resources/[HASH][.EXTENSION] which will be PUT on save.
  #
  # And when serializing the XHTML file all `<img src="data:..."/>` elements are translated
  # to `<img src="../resources/[HASH][.EXTENSION]"/>`.
  imgDataSrcToAttributes = (src) ->
    [info, bits] = src.split(',')
    mediaType = info.split(';')[0].split(':')[1]

    # Github will render an image based on the file extension so add it to the id
    switch mediaType
      when 'image/png' then extension = '.png'
      when 'image/jpeg' then extension = '.jpg'
      when 'image/svg' then extension = '.svg'
      else extension = ''

    shaObj = new jsSHA(bits, 'TEXT')
    hash = shaObj.getHash('SHA-1', 'HEX')
    id = "resources/#{hash}#{extension}"
    return {
      id: id
      mediaType: mediaType
      base64Encoded: bits
    }

  xhtmlToDocument = (html) ->
    parser = new DOMParser()

    # The html should be html5 coded xhtml. A doctype is not required, but
    # if the document has xhtml entities it won't parse correctly.
    # Add a doctype only if needed. Only consider first 1000 characters or so
    # of the document, since doctype should be early on, to avoid performance
    # hit when scanning large documents.
    if html.slice(0, 1000).toLowerCase().indexOf('<!doctype') < 0
      # If any entities beyond the allowed html5 ones are used, assume
      # legacy xhtml. The allowed entities are amp, lt, gt, quot and apos.
      # Here we use a negative lookahead assertion. You may have to call a perl
      # coder to explain it.
      if /&(?!(amp|lt|gt|quot|apos))\w+;/.test(html)
        # If the document has an xml declaration, insert right after that.
        # Otherwise insert the DOCTYPE declaration at the beginning.
        html = html.replace(
          /^(\s*<\?xml [^>]*>)?/,
          "$&\n<!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.1//EN' 'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd'>\n"
        )

    doc = parser.parseFromString(html, 'text/xml')
    # Chrome litters the entire dom with duplicate ns attributes
    $(doc).find('html *[xmlns="http://www.w3.org/1999/xhtml"]').removeAttr('xmlns')
    return doc
<commit_msg>If contents isn't xhtml, retry as html, and convert.<commit_after>define [
  'underscore'
  'jquery'
  'backbone'
  'jsSHA'
  'cs!gh-book/uuid'
  'cs!gh-book/binary-file'
  'cs!models/content/module'
  'cs!collections/content'
  'cs!models/utils'
], (_, $, Backbone, jsSHA, uuid, BinaryFile, ModuleModel, allContent, Utils) ->


  # An `<img src>` has the form `data:image/png;base64,.....`.
  # This method splits the src into a usable form and generates a resource path
  # using the hash of the bits.
  #
  # This is called twice:
  #
  # 1. When the body text is changed
  # 2. When the model is serialized
  #
  # The reason it is called twice is because Aloha strips the `data-src` attribute.
  # Since Aloha strips the data-src attribute the only data in the HTML is the
  # Base64-encoded string that represents the image.
  #
  # So, when the body text is changed a new BinaryFile is added to allContent
  # with the path `resources/[HASH][.EXTENSION] which will be PUT on save.
  #
  # And when serializing the XHTML file all `<img src="data:..."/>` elements are translated
  # to `<img src="../resources/[HASH][.EXTENSION]"/>`.
  imgDataSrcToAttributes = (src) ->
    [info, bits] = src.split(',')
    mediaType = info.split(';')[0].split(':')[1]

    # Github will render an image based on the file extension so add it to the id
    switch mediaType
      when 'image/png' then extension = '.png'
      when 'image/jpeg' then extension = '.jpg'
      when 'image/svg' then extension = '.svg'
      else extension = ''

    shaObj = new jsSHA(bits, 'TEXT')
    hash = shaObj.getHash('SHA-1', 'HEX')
    id = "resources/#{hash}#{extension}"
    return {
      id: id
      mediaType: mediaType
      base64Encoded: bits
    }

  xhtmlToDocument = (html) ->
    original = html # Keep a copy before modification
    parser = new DOMParser()

    # The html should be html5 coded xhtml. A doctype is not required, but
    # if the document has xhtml entities it won't parse correctly.
    # Add a doctype only if needed. Only consider first 1000 characters or so
    # of the document, since doctype should be early on, to avoid performance
    # hit when scanning large documents.
    if html.slice(0, 1000).toLowerCase().indexOf('<!doctype') < 0
      # If any entities beyond the allowed html5 ones are used, assume
      # legacy xhtml. The allowed entities are amp, lt, gt, quot and apos.
      # Here we use a negative lookahead assertion. You may have to call a perl
      # coder to explain it.
      if /&(?!(amp|lt|gt|quot|apos))\w+;/.test(html)
        # If the document has an xml declaration, insert right after that.
        # Otherwise insert the DOCTYPE declaration at the beginning.
        html = html.replace(
          /^(\s*<\?xml [^>]*>)?/,
          "$&\n<!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.1//EN' 'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd'>\n"
        )

    doc = parser.parseFromString(html, 'text/xml')

    if $(doc).find('parsererror').length
      # It's possible that the document wasn't valid xhtml. In that case, retry
      # as html. This isn't legal in ebooks, but we provide it here as a
      # convenience.

      # This works in Firefox and maybe a couple of others.
      doc = (new DOMParser).parseFromString(original, "text/html")

      # Chrome (as of version 28) does not support text/html, so we need
      # further hackery
      if doc == null
        # Remove any xml declarations, if this really was XML, we would
        # not be in this situation.
        original = original.replace(/^(\s*<\?xml [^>]*>)?/, '')
        doc = document.implementation.createHTMLDocument ""
        doc.documentElement.innerHTML = original

        # Chrome's XMLSerializer will serialize to HTML if you pass it an
        # HTMLDocument, so instead we clone the contents of the html document
        # and construct an xhtml document.
        xmldoc = document.implementation.createDocument(
          'http://www.w3.org/1999/xhtml', 'html', null)
        $(doc).find('html > *').clone().each (idx, node) ->
          xmldoc.documentElement.appendChild(node)
        return xmldoc
      else
        return doc
    else
      # Chrome litters the entire dom with duplicate ns attributes
      $(doc).find('html *[xmlns="http://www.w3.org/1999/xhtml"]').removeAttr('xmlns')
      return doc

{'fraction_non_alphanumeric': 0.12167300380228137, 'fraction_numerical': 0.010727865290602934, 'mean_word_length': 2.8042355371900825, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 5, 'xml': 15}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3290471', 'n_tokens_mistral': 2372, 'n_tokens_neox': 2221, 'n_words': 822}