<commit_before>#!/usr/bin/env ruby
# -*- encoding: us-ascii -*-

require 'lib/bun/file/descriptor'

module Bun

  class File < ::File
  
    class << self

      EOF_MARKER = 0x00000f000 # Octal 000000170000 or 0b000000000000000000001111000000000000
  
      def eof_marker
        EOF_MARKER
      end
    
      def characters_per_word
        Bun::Word.character.count
      end

      def packed_characters_per_word
        Bun::Word.packed_character.count
      end

      def decode(data)
        Bun::Words.import(data)
      end

      # Convert an eight-character Bun date "mm/dd/yy" to a Ruby Date
      def date(date)
        Date.strptime(date,"%m/%d/%y")
      end

      # Convert a Bun timestamp to the time of day in hours (24 hour clock)
      # Returns a three item array: hour, minutes, seconds (with embedded fractional seconds)
      TIME_SUM = 1620000 # additive offset for converting Bun times
      TIME_DIV = 64000.0 # division factor for converting Bun ticks to seconds
      def time_of_day(timestamp)
        timestamp = timestamp.integer.value
        seconds = (timestamp + TIME_SUM) / TIME_DIV
        minutes, seconds = seconds.divmod(60.0)
        hours, minutes = minutes.divmod(60.0)
        [hours, minutes, seconds.to_int]
      end

      # Convert a Bun date and time into a Ruby Time
      def time(date, timestamp)
        hours, minutes, seconds = time_of_day(timestamp)
        seconds, frac = seconds.divmod(1.0)
        micro_seconds = (frac*1000000).to_i
        date = self.date(date)
        Time.local(date.year, date.month, date.day, hours, minutes, seconds, micro_seconds)
      end
  
      def relative_path(*f)
        File.expand_path(File.join(*f), ENV['HOME']).sub(/^#{Regexp.escape(ENV['HOME'])}\//,'')
      end

      VALID_CONTROL_CHARACTERS = '\n\r\x8\x9\xb\xc' # \x8 is a backspace, \x9 is a tab, \xb is a VT, \xc is a form-feed
      VALID_CONTROL_CHARACTER_STRING = eval("\"#{VALID_CONTROL_CHARACTERS}\"")
      VALID_CONTROL_CHARACTER_REGEXP = /[#{VALID_CONTROL_CHARACTERS}]/
      INVALID_CHARACTER_REGEXP = /(?!(?>#{VALID_CONTROL_CHARACTER_REGEXP}))[[:cntrl:]]/
      VALID_CHARACTER_REGEXP = /(?!(?>#{INVALID_CHARACTER_REGEXP}))./

      def valid_control_character_regexp
        VALID_CONTROL_CHARACTER_REGEXP
      end

      def invalid_character_regexp
        INVALID_CHARACTER_REGEXP
      end

      def valid_character_regexp
        VALID_CHARACTER_REGEXP
      end
    
      def clean?(text)
        if RUBY_VERSION =~ /^1\.8/
          text !~ INVALID_CHARACTER_REGEXP
        else
          text !~ INVALID_CHARACTER_REGEXP
        end
      end
    
      def descriptor(options={})
        Header.new(options).descriptor
      end
    
      def frozen?(file_name)
        raise "File #{file_name} doesn't exist" unless File.exists?(file_name)
        new(:file=>file_name).frozen?
      end
    
      def create(options={}, &blk)
        preamble = nil
        if options[:type]
          ftype = options[:type]
        else
          preamble = get_preamble(options)
          ftype = preamble.file_type
        end
        klass = const_get(ftype.to_s.sub(/^./){|m| m.upcase}) unless ftype.is_a?(Class)
        if options[:header]
          if ftype == :frozen
            limit = Frozen.send(:new, :words=>preamble.words, :header=>true).header_size
          else
            limit = preamble.header_size
          end
        else
          limit = nil
        end
        f = klass.send(:new, options.merge(:n=>limit))
        if block_given?
          yield(f)
        else
          f
        end
      end

      def open(fname, options={}, &blk)
        create(options.merge(:file=>fname), &blk)
      end
      
      def read(fname, size=nil)
        if RUBY_VERSION =~ /^1\.8\.7/
          super
        else
          ::File.open(fname, "r:ascii-8bit") {|f| f.read(size) }
        end
      end
    
      def header(options={}, &blk)
        create(options.merge(:header=>true), &blk)
      end
    
      INITIAL_FETCH_SIZE = 30
      def get_preamble(options)<commit_msg>Clean up some redundant code<commit_after>#!/usr/bin/env ruby
# -*- encoding: us-ascii -*-

require 'lib/bun/file/descriptor'

module Bun

  class File < ::File
  
    class << self

      EOF_MARKER = 0x00000f000 # Octal 000000170000 or 0b000000000000000000001111000000000000
  
      def eof_marker
        EOF_MARKER
      end
    
      def characters_per_word
        Bun::Word.character.count
      end

      def packed_characters_per_word
        Bun::Word.packed_character.count
      end

      def decode(data)
        Bun::Words.import(data)
      end

      # Convert an eight-character Bun date "mm/dd/yy" to a Ruby Date
      def date(date)
        Date.strptime(date,"%m/%d/%y")
      end

      # Convert a Bun timestamp to the time of day in hours (24 hour clock)
      # Returns a three item array: hour, minutes, seconds (with embedded fractional seconds)
      TIME_SUM = 1620000 # additive offset for converting Bun times
      TIME_DIV = 64000.0 # division factor for converting Bun ticks to seconds
      def time_of_day(timestamp)
        timestamp = timestamp.integer.value
        seconds = (timestamp + TIME_SUM) / TIME_DIV
        minutes, seconds = seconds.divmod(60.0)
        hours, minutes = minutes.divmod(60.0)
        [hours, minutes, seconds.to_int]
      end

      # Convert a Bun date and time into a Ruby Time
      def time(date, timestamp)
        hours, minutes, seconds = time_of_day(timestamp)
        seconds, frac = seconds.divmod(1.0)
        micro_seconds = (frac*1000000).to_i
        date = self.date(date)
        Time.local(date.year, date.month, date.day, hours, minutes, seconds, micro_seconds)
      end
  
      def relative_path(*f)
        File.expand_path(File.join(*f), ENV['HOME']).sub(/^#{Regexp.escape(ENV['HOME'])}\//,'')
      end

      VALID_CONTROL_CHARACTERS = '\n\r\x8\x9\xb\xc' # \x8 is a backspace, \x9 is a tab, \xb is a VT, \xc is a form-feed
      VALID_CONTROL_CHARACTER_STRING = eval("\"#{VALID_CONTROL_CHARACTERS}\"")
      VALID_CONTROL_CHARACTER_REGEXP = /[#{VALID_CONTROL_CHARACTERS}]/
      INVALID_CHARACTER_REGEXP = /(?!(?>#{VALID_CONTROL_CHARACTER_REGEXP}))[[:cntrl:]]/
      VALID_CHARACTER_REGEXP = /(?!(?>#{INVALID_CHARACTER_REGEXP}))./

      def valid_control_character_regexp
        VALID_CONTROL_CHARACTER_REGEXP
      end

      def invalid_character_regexp
        INVALID_CHARACTER_REGEXP
      end

      def valid_character_regexp
        VALID_CHARACTER_REGEXP
      end
    
      def clean?(text)
        text !~ INVALID_CHARACTER_REGEXP
      end
    
      def descriptor(options={})
        Header.new(options).descriptor
      end
    
      def frozen?(file_name)
        raise "File #{file_name} doesn't exist" unless File.exists?(file_name)
        new(:file=>file_name).frozen?
      end
    
      def create(options={}, &blk)
        preamble = nil
        if options[:type]
          ftype = options[:type]
        else
          preamble = get_preamble(options)
          ftype = preamble.file_type
        end
        klass = const_get(ftype.to_s.sub(/^./){|m| m.upcase}) unless ftype.is_a?(Class)
        if options[:header]
          if ftype == :frozen
            limit = Frozen.send(:new, :words=>preamble.words, :header=>true).header_size
          else
            limit = preamble.header_size
          end
        else
          limit = nil
        end
        f = klass.send(:new, options.merge(:n=>limit))
        if block_given?
          yield(f)
        else
          f
        end
      end

      def open(fname, options={}, &blk)
        create(options.merge(:file=>fname), &blk)
      end
      
      def read(fname, size=nil)
        if RUBY_VERSION =~ /^1\.8/
          super
        else
          ::File.open(fname, "r:ascii-8bit") {|f| f.read(size) }
        end
      end
    
      def header(options={}, &blk)
        create(options.merge(:header=>true), &blk)
      end
    
      INITIAL_FETCH_SIZE = 30
      def get_preamble(options)
{'fraction_non_alphanumeric': 0.10763500931098696, 'fraction_numerical': 0.0244568590937306, 'mean_word_length': 2.1592156862745098, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5132724', 'n_tokens_mistral': 2765, 'n_tokens_neox': 2568, 'n_words': 578}