<commit_before>require 'linguist/language'
require 'linguist/mime'
require 'linguist/pathname'

require 'escape_utils'

module Linguist
  module BlobHelper
    def pathname
      Pathname.new(name || "")
    end

    def mime_type
      @mime_type ||= pathname.mime_type
    end

    def content_type
      pathname.content_type
    end

    def disposition
      case content_type
      when 'application/octet-stream', 'application/java-archive'
        "attachment; filename=#{EscapeUtils.escape_url(pathname.basename)}"
      else
        'inline'
      end
    end

    def lines
      @lines ||= data ? data.split("\n", -1) : []
    end

    def loc
      lines.size
    end

    def sloc
      lines.grep(/\S/).size
    end

    def binary?
      content_type.include?('octet') || !(text? || image?)
    end

    def text?
      content_type[/(text|json)/]
    end

    def image?
      ['.png', '.jpg', '.jpeg', '.gif'].include?(pathname.extname)
    end

    MEGABYTE = 1024 * 1024

    def large?
      size.to_i > MEGABYTE
    end

    def viewable?
      !image? && !binary? && !large?
    end

    def generated?
      ['.xib', '.nib', '.pbxproj'].include?(pathname.extname)
    end

    def language
      if text?
        shebang_language || pathname.language
      else
        Language['Text']
      end
    end

    def lexer_name
      language.lexer_name
    end

    def lexer
      language.lexer
    end

    def shebang_script
      return if !text? || large?

      if (match = data.match(/(.+)\n?/)) && (bang = match[0]) =~ /^#!/
        bang.sub!(/^#! /, '#!')
        tokens = bang.split(' ')
        pieces = tokens.first.split('/')
        if pieces.size > 1
          script = pieces.last
        else
          script = pieces.first.sub('#!', '')
        end

        script = script == 'env' ? tokens[1] : script

        # python2.4 => python
        if script =~ /((?:\d+\.?)+)/
          script.sub! $1, ''
        end

        script
      end
    end

    def shebang_language
      if script = shebang_script
        case script
        when 'bash'
          Language['Shell']
        when 'groovy'
          Language['Java']
        when 'macruby'
          Language['Ruby']
        when 'node'
          Language['JavaScript']
        when 'rake'
          Language['Ruby']
        when 'sh'
          Language['Shell']
        when 'zsh'
          Language['Shell']
        else
          lang = Language.find_by_lexer(shebang_script)
          lang != Language['Text'] ? lang : nil
        end
      end
    end

    def colorize
      return if !text? || large?
      lexer.colorize(data)
    end

    def colorize_without_wrapper
      return if !text? || large?
      lexer.colorize_without_wrapper(data)
    end
  end
end
<commit_msg>Allow blob data to be nil<commit_after>require 'linguist/language'
require 'linguist/mime'
require 'linguist/pathname'

require 'escape_utils'

module Linguist
  module BlobHelper
    def pathname
      Pathname.new(name || "")
    end

    def mime_type
      @mime_type ||= pathname.mime_type
    end

    def content_type
      pathname.content_type
    end

    def disposition
      case content_type
      when 'application/octet-stream', 'application/java-archive'
        "attachment; filename=#{EscapeUtils.escape_url(pathname.basename)}"
      else
        'inline'
      end
    end

    def lines
      @lines ||= data ? data.split("\n", -1) : []
    end

    def loc
      lines.size
    end

    def sloc
      lines.grep(/\S/).size
    end

    def binary?
      content_type.include?('octet') || !(text? || image?)
    end

    def text?
      content_type[/(text|json)/]
    end

    def image?
      ['.png', '.jpg', '.jpeg', '.gif'].include?(pathname.extname)
    end

    MEGABYTE = 1024 * 1024

    def large?
      size.to_i > MEGABYTE
    end

    def viewable?
      !image? && !binary? && !large?
    end

    def generated?
      ['.xib', '.nib', '.pbxproj'].include?(pathname.extname)
    end

    def language
      if text?
        shebang_language || pathname.language
      else
        Language['Text']
      end
    end

    def lexer_name
      language.lexer_name
    end

    def lexer
      language.lexer
    end

    def shebang_script
      return if !text? || large?

      if data && (match = data.match(/(.+)\n?/)) && (bang = match[0]) =~ /^#!/
        bang.sub!(/^#! /, '#!')
        tokens = bang.split(' ')
        pieces = tokens.first.split('/')
        if pieces.size > 1
          script = pieces.last
        else
          script = pieces.first.sub('#!', '')
        end

        script = script == 'env' ? tokens[1] : script

        # python2.4 => python
        if script =~ /((?:\d+\.?)+)/
          script.sub! $1, ''
        end

        script
      end
    end

    def shebang_language
      if script = shebang_script
        case script
        when 'bash'
          Language['Shell']
        when 'groovy'
          Language['Java']
        when 'macruby'
          Language['Ruby']
        when 'node'
          Language['JavaScript']
        when 'rake'
          Language['Ruby']
        when 'sh'
          Language['Shell']
        when 'zsh'
          Language['Shell']
        else
          lang = Language.find_by_lexer(shebang_script)
          lang != Language['Text'] ? lang : nil
        end
      end
    end

    def colorize
      return if !text? || large?
      lexer.colorize(data)
    end

    def colorize_without_wrapper
      return if !text? || large?
      lexer.colorize_without_wrapper(data)
    end
  end
end

{'fraction_non_alphanumeric': 0.12504478681476172, 'fraction_numerical': 0.005374417771408097, 'mean_word_length': 1.7502463054187192, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3597934', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1759, 'n_words': 416}