<commit_before>define [
  'underscore'
  'jquery'
  'backbone'
  'jsSHA'
  'cs!gh-book/uuid'
  'cs!gh-book/binary-file'
  'cs!models/content/module'
  'cs!collections/content'
  'cs!gh-book/utils'
], (_, $, Backbone, jsSHA, uuid, BinaryFile, ModuleModel, allContent, Utils) ->


  # An `<img src>` has the form `data:image/png;base64,.....`.
  # This method splits the src into a usable form and generates a resource path
  # using the hash of the bits.
  #
  # This is called twice:
  #
  # 1. When the body text is changed
  # 2. When the model is serialized
  #
  # The reason it is called twice is because Aloha strips the `data-src` attribute.
  # Since Aloha strips the data-src attribute the only data in the HTML is the
  # Base64-encoded string that represents the image.
  #
  # So, when the body text is changed a new BinaryFile is added to allContent
  # with the path `resources/[HASH][.EXTENSION] which will be PUT on save.
  #
  # And when serializing the XHTML file all `<img src="data:..."/>` elements are translated
  # to `<img src="../resources/[HASH][.EXTENSION]"/>`.
  imgDataSrcToAttributes = (src) ->
    [info, bits] = src.split(',')
    mediaType = info.split(';')[0].split(':')[1]

    # Github will render an image based on the file extension so add it to the id
    switch mediaType
      when 'image/png' then extension = '.png'
      when 'image/jpeg' then extension = '.jpg'
      when 'image/svg' then extension = '.svg'
      else extension = ''

    shaObj = new jsSHA(bits, 'TEXT')
    hash = shaObj.getHash('SHA-1', 'HEX')
    id = "resources/#{hash}#{extension}"
    return {
      id: id
      mediaType: mediaType
      base64Encoded: bits
    }

  # The `Content` model contains the following members:
  #
  # * `title` - an HTML title of the content
  # * `language` - the main language (eg `en-us`)
  # * `subjects` - an array of strings (eg `['Mathematics', 'Business']`)
  # * `keywords` - an array of keywords (eg `['constant', 'boltzmann constant']`)
  # * `authors` - an `Collection` of `User`s that are attributed as authors
  return class XhtmlModel extends ModuleModel
    mediaType: 'application/xhtml+xml'

    defaults:
      title: 'Untitled'

    initialize: (options) ->
      super(options)

      # Give the content an id if it does not already have one
      if not @id
        @setNew()
        @id = "content/#{uuid(@get('title'))}#{options.extension || '.html'}"

      # Clear that the title on the model has changed
      # so it does not get saved unnecessarily.
      # The title of the XhtmlFile is not stored inside the file;
      # it is stored in the navigation file

      # Ignore Title changes for now. The canonical title is in the ToC
      # TODO: Re-enable after the sprint
      # @on 'change:title', (model, value, options) =>
      #   head = @get 'head'
      #   $head = jQuery("<div class='unwrap-me'>#{head}</div>")

      #   $head.children('title').text(value)
      #   @set 'head', $head[0].innerHTML, options

      @on 'change:body', (model, value, options) =>
        $body = jQuery("<div class='unwrap-me'></div>")
        $body.append(value)

        # "1. When the body text is changed" (see above)
        # -------------
        # For newly added images (ones without a `data-src` attribute)
        # Create a new BinaryFile
        $body.find('img[src^="data:"]:not([data-src])').each (i, img) =>
          $img = $(img)

          # 'data:image/png;base64,.....'
          src = $img.attr('src')
          attrs = imgDataSrcToAttributes(src)

          # If the resource is not already in allContent then add it
          if not allContent.get(attrs.id)
            imageModel = new BinaryFile(attrs)
            # Make sure the mediaType is set (This may be redundant)
            imageModel.mediaType = attrs.mediaType
            # Set the bits here so isDirty is set to true
            imageModel._markDirty({}, true)

            allContent.add(imageModel)

          # $img.attr('data-src', attrs.id) TODO: Aloha keeps stripping this attribute off.

        @set('body', $body[0].innerHTML.trim())


    # Since the titles are purely cosmetic do not mark the model as dirty
    # TODO: Remove this after the sprint
    _markDirty: (options, force=false) ->
      changed = _.omit @changedAttributes(), 'title'
      super(options, force) if not _.isEmpty(changed)

    # This promise is resolved once the file is parsed so we know which images to load
    _imagesLoaded: new $.Deferred()
    _loadComplex: (originalPromise) ->
      return @_imagesLoaded

    parse: (json) ->
      html = json.content

      # If the parse is a result of a write then update the sha.
      # The parse is a result of a GitHub.write if there is no `.content`
      return {} if not json.content

      # Rename elements before jQuery parses and removes them
      # (because they are not valid children of a div)

      html = "<body>#{html}</body>" if not /<body/.test html
      html = "<html>#{html}</html>" if not /<html/.test html
<commit_msg>wait for parse to finish before resolving load promise for xhtml files<commit_after>define [
  'underscore'
  'jquery'
  'backbone'
  'jsSHA'
  'cs!gh-book/uuid'
  'cs!gh-book/binary-file'
  'cs!models/content/module'
  'cs!collections/content'
  'cs!gh-book/utils'
], (_, $, Backbone, jsSHA, uuid, BinaryFile, ModuleModel, allContent, Utils) ->


  # An `<img src>` has the form `data:image/png;base64,.....`.
  # This method splits the src into a usable form and generates a resource path
  # using the hash of the bits.
  #
  # This is called twice:
  #
  # 1. When the body text is changed
  # 2. When the model is serialized
  #
  # The reason it is called twice is because Aloha strips the `data-src` attribute.
  # Since Aloha strips the data-src attribute the only data in the HTML is the
  # Base64-encoded string that represents the image.
  #
  # So, when the body text is changed a new BinaryFile is added to allContent
  # with the path `resources/[HASH][.EXTENSION] which will be PUT on save.
  #
  # And when serializing the XHTML file all `<img src="data:..."/>` elements are translated
  # to `<img src="../resources/[HASH][.EXTENSION]"/>`.
  imgDataSrcToAttributes = (src) ->
    [info, bits] = src.split(',')
    mediaType = info.split(';')[0].split(':')[1]

    # Github will render an image based on the file extension so add it to the id
    switch mediaType
      when 'image/png' then extension = '.png'
      when 'image/jpeg' then extension = '.jpg'
      when 'image/svg' then extension = '.svg'
      else extension = ''

    shaObj = new jsSHA(bits, 'TEXT')
    hash = shaObj.getHash('SHA-1', 'HEX')
    id = "resources/#{hash}#{extension}"
    return {
      id: id
      mediaType: mediaType
      base64Encoded: bits
    }

  # The `Content` model contains the following members:
  #
  # * `title` - an HTML title of the content
  # * `language` - the main language (eg `en-us`)
  # * `subjects` - an array of strings (eg `['Mathematics', 'Business']`)
  # * `keywords` - an array of keywords (eg `['constant', 'boltzmann constant']`)
  # * `authors` - an `Collection` of `User`s that are attributed as authors
  return class XhtmlModel extends ModuleModel
    mediaType: 'application/xhtml+xml'

    defaults:
      title: 'Untitled'

    initialize: (options) ->
      super(options)

      # Give the content an id if it does not already have one
      if not @id
        @setNew()
        @id = "content/#{uuid(@get('title'))}#{options.extension || '.html'}"

      # Clear that the title on the model has changed
      # so it does not get saved unnecessarily.
      # The title of the XhtmlFile is not stored inside the file;
      # it is stored in the navigation file

      # Ignore Title changes for now. The canonical title is in the ToC
      # TODO: Re-enable after the sprint
      # @on 'change:title', (model, value, options) =>
      #   head = @get 'head'
      #   $head = jQuery("<div class='unwrap-me'>#{head}</div>")

      #   $head.children('title').text(value)
      #   @set 'head', $head[0].innerHTML, options

      @on 'change:body', (model, value, options) =>
        $body = jQuery("<div class='unwrap-me'></div>")
        $body.append(value)

        # "1. When the body text is changed" (see above)
        # -------------
        # For newly added images (ones without a `data-src` attribute)
        # Create a new BinaryFile
        $body.find('img[src^="data:"]:not([data-src])').each (i, img) =>
          $img = $(img)

          # 'data:image/png;base64,.....'
          src = $img.attr('src')
          attrs = imgDataSrcToAttributes(src)

          # If the resource is not already in allContent then add it
          if not allContent.get(attrs.id)
            imageModel = new BinaryFile(attrs)
            # Make sure the mediaType is set (This may be redundant)
            imageModel.mediaType = attrs.mediaType
            # Set the bits here so isDirty is set to true
            imageModel._markDirty({}, true)

            allContent.add(imageModel)

          # $img.attr('data-src', attrs.id) TODO: Aloha keeps stripping this attribute off.

        @set('body', $body[0].innerHTML.trim())


    # Since the titles are purely cosmetic do not mark the model as dirty
    # TODO: Remove this after the sprint
    _markDirty: (options, force=false) ->
      changed = _.omit @changedAttributes(), 'title'
      super(options, force) if not _.isEmpty(changed)

    # This promise is resolved once the file is parsed so we know which images to load
    _imagesLoaded: new $.Deferred()
    _loadComplex: (originalPromise) ->
      return $.when(@_imagesLoaded, originalPromise)

    parse: (json) ->
      html = json.content

      # If the parse is a result of a write then update the sha.
      # The parse is a result of a GitHub.write if there is no `.content`
      return {} if not json.content

      # Rename elements before jQuery parses and removes them
      # (because they are not valid children of a div)

      html = "<body>#{html}</body>" if not /<body/.test html
      html = "<html>#{html}</html>" if not /<html/.test html

{'fraction_non_alphanumeric': 0.12684510923046644, 'fraction_numerical': 0.003148986419996064, 'mean_word_length': 2.6322373123659757, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1344560', 'n_tokens_mistral': 3118, 'n_tokens_neox': 3016, 'n_words': 1108}