<commit_before>class Spree::CmsImage < ActiveRecord::Base

  # dragonfly
  image_accessor :file

  belongs_to :imageable, polymorphic: true

  attr_accessible :retained_file, :file_url, :remove_file, :alt, :name, :file

  validates :name, :file, presence: true
  validates :name, uniqueness: true
  validates_size_of :file, maximum: 3.megabytes
  validates_property :format, of: :file, in: [:jpeg, :jpg, :png, :gif]

  class << self
    def replace_tokens(str)
      str ||= ''
      # 1. check body text for any image tags containing 'data-cms-image=""'
      # 2. process the files
      # 2. cache the results
      matches = str.scan(/(<img(.*?)data-cms-image="(\d+)\|([\d+x\d+].*?+)?"(.*?)>)/)

      cached = str
      if matches.any?
        matches.each do |token|
          tok = token[0] # the original <img...> tag

          pre_data = token[1]
          img_id = token[2].to_i
          resize_str = (token[3] || '').gsub('&gt;', '>')
          post_data = token[4]

          # strip out the original src="..." width="..." and height="...",
          # leaving all other attributes in there
          reg = / src=".*?"| width=".*?"| height=".*?"/
          pre_data.gsub!(reg, '')
          post_data.gsub!(reg, '')

          # and recreate the tag
          if thumb = Spree::CmsImage.find_by_id(img_id)
            if resize_str.present?
              # thumbnail
              processed = thumb.file.thumb(resize_str)
            else
              # original
              processed = thumb.file
            end
            cached.gsub!(tok, %(<img#{pre_data} src="#{processed.url}" width="#{processed.width}" height="#{processed.height}"#{post_data}>))
          else
            # TODO: display a 'missing' image or something?
            # log an error?
            cached.gsub!(tok, '')
          end
        end
      end
      cached
    end<commit_msg>adding data-orig-src to all cms images<commit_after>class Spree::CmsImage < ActiveRecord::Base

  # dragonfly
  image_accessor :file

  belongs_to :imageable, polymorphic: true

  attr_accessible :retained_file, :file_url, :remove_file, :alt, :name, :file

  validates :name, :file, presence: true
  validates :name, uniqueness: true
  validates_size_of :file, maximum: 3.megabytes
  validates_property :format, of: :file, in: [:jpeg, :jpg, :png, :gif]

  class << self
    def replace_tokens(str)
      str ||= ''
      # 1. check body text for any image tags containing 'data-cms-image=""'
      # 2. process the files
      # 2. cache the results
      matches = str.scan(/(<img(.*?)data-cms-image="(\d+)\|([\d+x\d+].*?+)?"(.*?)>)/)

      cached = str
      if matches.any?
        matches.each do |token|
          tok = token[0] # the original <img...> tag

          pre_data = token[1]
          img_id = token[2].to_i
          resize_str = (token[3] || '').gsub('&gt;', '>')
          post_data = token[4]

          # strip out the original src="..." width="..." and height="...",
          # leaving all other attributes in there
          reg = / src=".*?"| width=".*?"| height=".*?"/
          pre_data.gsub!(reg, '')
          post_data.gsub!(reg, '')

          # and recreate the tag
          if thumb = Spree::CmsImage.find_by_id(img_id)
            original = thumb.file
            if resize_str.present?
              # thumbnail
              processed = thumb.file.thumb(resize_str)
            else
              # original
              processed = original
            end
            cached.gsub!(tok, %(<img#{pre_data} data-orig-src="#{original.url}" src="#{processed.url}" width="#{processed.width}" height="#{processed.height}"#{post_data}>))
          else
            # TODO: display a 'missing' image or something?
            # log an error?
            cached.gsub!(tok, '')
          end
        end
      end
      cached
    end
{'fraction_non_alphanumeric': 0.15344603381014305, 'fraction_numerical': 0.0046814044213263975, 'mean_word_length': 2.0694333599361534, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2462664', 'n_tokens_mistral': 1240, 'n_tokens_neox': 1186, 'n_words': 297}