<commit_before>require File.dirname(__FILE__) + '/smart_asset/gems'

SmartAsset::Gems.require(:lib)

require 'digest'
require 'fileutils'
require 'time'
require 'yaml'

$:.unshift File.dirname(__FILE__)

require 'smart_asset/helper'
require 'smart_asset/version'

class SmartAsset
  class <<self
    
    attr_accessor :append_random, :asset_host, :asset_counter, :cache, :config, :dest, :env, :envs, :pub, :root, :sources
    
    BIN = File.expand_path(File.dirname(__FILE__) + '/../bin')
    CLOSURE_COMPILER = BIN + '/closure_compiler.jar'
    YUI_COMPRESSOR = BIN + '/yui_compressor.jar'
    
    def binary(root, relative_config=nil)
      load_config root, relative_config
      compress 'javascripts'
      compress 'stylesheets'
    end
    
    def compress(type)
      dest = @dest[type]
      dir = "#{@pub}/#{@sources[type]}"
      ext = ext_from_type(type)
      packages = []
      time_cache = {}
      
      FileUtils.mkdir_p dest
      
      (@config[type] || {}).each do |package, files|
        next if ENV['PACKAGE'] && ENV['PACKAGE'] != package
        if files
          # Retrieve list of Git modified timestamps
          timestamps = []
          files.each do |file|
            if File.exists?("#{dir}/#{file}.#{ext}")
              if time_cache[file]
                time = time_cache[file]
              else
                time = `cd #{@root} && git log --pretty=format:%cd -n 1 --date=iso #{@config['public']}/#{@sources[type]}/#{file}.#{ext}`
                if time.strip.empty? || time.include?('command not found')
                  time = ENV['MODIFIED'] ? Time.parse(ENV['MODIFIED']) : Time.now
                else
                  time = Time.parse(time)
                end
                time = time.utc.strftime("%Y%m%d%H%M%S")
                time += file.to_s
                time_cache[file] = time
              end
              timestamps << time
            end
          end
          next if timestamps.empty?
          
          # Modified hash
          hash = Digest::SHA1.hexdigest(timestamps.join)[0..7]
          
          # Package path
          package = "#{dest}/#{hash}_#{package}.#{ext}"
          packages << package
          
          # If package file does not exist
          unless File.exists?(package)
            data = []
            
            # Join files in package
            files.each do |file|
              if File.exists?(source = "#{dir}/#{file}.#{ext}")
                data << File.read(source)
              end
            end
            
            # Don't create new compressed file if no data
            data = data.join("\n")
            next if data.strip.empty?
            
            # Compress joined files
            tmp = "#{dest}/tmp.#{ext}"
            File.open(tmp, 'w') { |f| f.write(data) }
            puts "\nCreating #{package}..."
            if ext == 'js'
              warning = ENV['WARN'] ? nil : " --warning_level QUIET"
              cmd = "java -jar #{CLOSURE_COMPILER} --js #{tmp} --js_output_file #{package}#{warning}"
            elsif ext == 'css'
              warning = ENV['WARN'] ? " -v" : nil
              cmd = "java -jar #{YUI_COMPRESSOR} #{tmp} -o #{package}#{warning}"
            end
            puts cmd if ENV['DEBUG']
            `#{cmd}`
            FileUtils.rm(tmp) unless ENV['DEBUG']
            
            # Fix YUI compression issue
            if ext == 'css'
              if RUBY_PLATFORM.downcase.include?('darwin')
                `sed -i '' 's/ and(/ and (/g' #{package}`
              else
                `sed -i 's/ and(/ and (/g' #{package}`
              end
            end
          end
        end
      end
      
      # Remove old/unused packages
      (Dir["#{dest}/????????_*.#{ext}"] - packages).each do |path|
        FileUtils.rm path
      end
      
      # Delete legacy yml files
      Dir["#{dest}/*.yml"].each do |path|
        FileUtils.rm path
      end<commit_msg>Remove packages skipped due to no data, remove legacy assets<commit_after>require File.dirname(__FILE__) + '/smart_asset/gems'

SmartAsset::Gems.require(:lib)

require 'digest'
require 'fileutils'
require 'time'
require 'yaml'

$:.unshift File.dirname(__FILE__)

require 'smart_asset/helper'
require 'smart_asset/version'

class SmartAsset
  class <<self
    
    attr_accessor :append_random, :asset_host, :asset_counter, :cache, :config, :dest, :env, :envs, :pub, :root, :sources
    
    BIN = File.expand_path(File.dirname(__FILE__) + '/../bin')
    CLOSURE_COMPILER = BIN + '/closure_compiler.jar'
    YUI_COMPRESSOR = BIN + '/yui_compressor.jar'
    
    def binary(root, relative_config=nil)
      load_config root, relative_config
      compress 'javascripts'
      compress 'stylesheets'
    end
    
    def compress(type)
      dest = @dest[type]
      dir = "#{@pub}/#{@sources[type]}"
      ext = ext_from_type(type)
      packages = []
      time_cache = {}
      
      FileUtils.mkdir_p dest
      
      (@config[type] || {}).each do |package, files|
        next if ENV['PACKAGE'] && ENV['PACKAGE'] != package
        if files
          # Retrieve list of Git modified timestamps
          timestamps = []
          files.each do |file|
            if File.exists?("#{dir}/#{file}.#{ext}")
              if time_cache[file]
                time = time_cache[file]
              else
                time = `cd #{@root} && git log --pretty=format:%cd -n 1 --date=iso #{@config['public']}/#{@sources[type]}/#{file}.#{ext}`
                if time.strip.empty? || time.include?('command not found')
                  time = ENV['MODIFIED'] ? Time.parse(ENV['MODIFIED']) : Time.now
                else
                  time = Time.parse(time)
                end
                time = time.utc.strftime("%Y%m%d%H%M%S")
                time += file.to_s
                time_cache[file] = time
              end
              timestamps << time
            end
          end
          next if timestamps.empty?
          
          # Modified hash
          hash = Digest::SHA1.hexdigest(timestamps.join)[0..7]
          
          # Package path
          package = "#{dest}/#{hash}_#{package}.#{ext}"
          packages << package
          
          # If package file exists
          if File.exists?(package)
            packages << package
          else
            data = []
            
            # Join files in package
            files.each do |file|
              if File.exists?(source = "#{dir}/#{file}.#{ext}")
                data << File.read(source)
              end
            end
            
            # Don't create new compressed file if no data
            data = data.join("\n")
            next if data.strip.empty?
            
            # Compress joined files
            tmp = "#{dest}/tmp.#{ext}"
            File.open(tmp, 'w') { |f| f.write(data) }
            puts "\nCreating #{package}..."
            if ext == 'js'
              warning = ENV['WARN'] ? nil : " --warning_level QUIET"
              cmd = "java -jar #{CLOSURE_COMPILER} --js #{tmp} --js_output_file #{package}#{warning}"
            elsif ext == 'css'
              warning = ENV['WARN'] ? " -v" : nil
              cmd = "java -jar #{YUI_COMPRESSOR} #{tmp} -o #{package}#{warning}"
            end
            puts cmd if ENV['DEBUG']
            `#{cmd}`
            FileUtils.rm(tmp) unless ENV['DEBUG']
            
            # Fix YUI compression issue
            if ext == 'css'
              if RUBY_PLATFORM.downcase.include?('darwin')
                `sed -i '' 's/ and(/ and (/g' #{package}`
              else
                `sed -i 's/ and(/ and (/g' #{package}`
              end
            end
            
            # Package created
            packages << package
          end
        end
      end
      
      # Remove old/unused packages
      (Dir["#{dest}/????????_*.#{ext}"] - packages).each do |path|
        FileUtils.rm path
      end
      
      # Delete legacy files
      Dir["#{dest}/*.yml", "#{dest}/??????????????_*.{css,js}"].each do |path|
        FileUtils.rm path
      end
{'fraction_non_alphanumeric': 0.1404591458899037, 'fraction_numerical': 0.0009874105159219946, 'mean_word_length': 1.5650522317188984, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3555098', 'n_tokens_mistral': 2410, 'n_tokens_neox': 2398, 'n_words': 569}