<commit_before># -*- coding: utf-8 -*-

module Icosmith
  class Generator
    MANIFEST_FILENAME = "manifest.json"
    SVG_ZIPFILENAME = "svg.zip"
    FONTS_ZIPFILENAME = "fonts.zip"

    def initialize
      config_filename = File.join(Rails.root, "config", "icosmith", Icosmith::Config::FILENAME)
      @config = Icosmith::Config.load(config_filename)

      @manifest_full_path = File.join(Rails.root, @config.manifest_dir, MANIFEST_FILENAME)
      @src_dir = File.join(Rails.root, @config.svg_dir)

      @temp_dir = File.join(Rails.root, "tmp", "icosmith")
      @svg_zipfile = File.join(@temp_dir, SVG_ZIPFILENAME)
      @fonts_zipfile = File.join(@temp_dir, FONTS_ZIPFILENAME)
      @css_dir = File.join(Rails.root, @config.css_dir)
      @font_dir = File.join(Rails.root, @config.font_dir)

      FileUtils.mkdir_p(@temp_dir)
      FileUtils.mkdir_p(@css_dir)
      FileUtils.mkdir_p(@font_dir)
    end

    def create_svg_zipfile
      FileUtils.rm_f(@svg_zipfile)

      Zip::ZipFile.open(@svg_zipfile, Zip::ZipFile::CREATE) do |zipfile|
        Dir.glob("#{@src_dir}/*.svg").each do |filename|
          zipfile.add(filename.split("/").last, filename)
        end

        zipfile.add(MANIFEST_FILENAME, @manifest_full_path) if File.exists?(@manifest_full_path)
      end
    end

    def generate_font
      fontfile_contents = RestClient.post(@config.generate_fonts_url, file: File.new(@svg_zipfile))

      File.open(@fonts_zipfile, "w:binary") do |f|
        f.write(fontfile_contents)
      end

      extract_fonts
    end

    private
    def extract_fonts
      Zip::ZipFile.open(@fonts_zipfile) do |zip_file|
        zip_file.each do |file|
          dest_path = File.join(@temp_dir, file.name)
          FileUtils.rm_f(dest_path)
          zip_file.extract(file, dest_path)
        end
      end

      manifest_tempfile = File.join(@temp_dir, MANIFEST_FILENAME)
      manifest_contents = JSON.parse(File.read(manifest_tempfile))
      family_name = manifest_contents["family"].gsub(" ", "")
      weight = manifest_contents["weight"] || "Regular"
      font_basename = "#{family_name}-#{weight}"

      FileUtils.mkdir_p(@font_dir)
      FileUtils.mkdir_p(@css_dir)
      FileUtils.mv("#{@temp_dir}/#{font_basename}.css", @css_dir)
      Dir.glob("#{@temp_dir}/#{font_basename}.{ttf,woff,svg,eot,afm}").each do |file|
        FileUtils.mv(file, @font_dir)
      end
      FileUtils.mv(manifest_tempfile, @manifest_full_path)
      FileUtils.remove_dir(@temp_dir)
    end
  end
end<commit_msg>Updating file separator<commit_after># -*- coding: utf-8 -*-

module Icosmith
  class Generator
    MANIFEST_FILENAME = "manifest.json"
    SVG_ZIPFILENAME = "svg.zip"
    FONTS_ZIPFILENAME = "fonts.zip"

    def initialize
      config_filename = File.join(Rails.root, "config", "icosmith", Icosmith::Config::FILENAME)
      @config = Icosmith::Config.load(config_filename)

      @manifest_full_path = File.join(Rails.root, @config.manifest_dir, MANIFEST_FILENAME)
      @src_dir = File.join(Rails.root, @config.svg_dir)

      @temp_dir = File.join(Rails.root, "tmp", "icosmith")
      @svg_zipfile = File.join(@temp_dir, SVG_ZIPFILENAME)
      @fonts_zipfile = File.join(@temp_dir, FONTS_ZIPFILENAME)
      @css_dir = File.join(Rails.root, @config.css_dir)
      @font_dir = File.join(Rails.root, @config.font_dir)

      FileUtils.mkdir_p(@temp_dir)
      FileUtils.mkdir_p(@css_dir)
      FileUtils.mkdir_p(@font_dir)
    end

    def create_svg_zipfile
      FileUtils.rm_f(@svg_zipfile)

      Zip::ZipFile.open(@svg_zipfile, Zip::ZipFile::CREATE) do |zipfile|
        Dir.glob("#{@src_dir}#{File::SEPARATOR}*.svg").each do |filename|
          zipfile.add(filename.split(File::SEPARATOR).last, filename)
        end

        zipfile.add(MANIFEST_FILENAME, @manifest_full_path) if File.exists?(@manifest_full_path)
      end
    end

    def generate_font
      fontfile_contents = RestClient.post(@config.generate_fonts_url, file: File.new(@svg_zipfile))

      File.open(@fonts_zipfile, "w:binary") do |f|
        f.write(fontfile_contents)
      end

      extract_fonts
    end

    private
    def extract_fonts
      Zip::ZipFile.open(@fonts_zipfile) do |zip_file|
        zip_file.each do |file|
          dest_path = File.join(@temp_dir, file.name)
          FileUtils.rm_f(dest_path)
          zip_file.extract(file, dest_path)
        end
      end

      manifest_tempfile = File.join(@temp_dir, MANIFEST_FILENAME)
      manifest_contents = JSON.parse(File.read(manifest_tempfile))
      family_name = manifest_contents["family"].gsub(" ", "")
      weight = manifest_contents["weight"] || "Regular"
      font_basename = "#{family_name}-#{weight}"

      FileUtils.mkdir_p(@font_dir)
      FileUtils.mkdir_p(@css_dir)
      FileUtils.mv("#{@temp_dir}#{File::SEPARATOR}#{font_basename}.css", @css_dir)
      Dir.glob("#{@temp_dir}#{File::SEPARATOR}#{font_basename}.{ttf,woff,svg,eot,afm}").each do |file|
        FileUtils.mv(file, @font_dir)
      end
      FileUtils.mv(manifest_tempfile, @manifest_full_path)
      FileUtils.remove_dir(@temp_dir)
    end
  end
end
{'fraction_non_alphanumeric': 0.13198666405177487, 'fraction_numerical': 0.0003922337713277113, 'mean_word_length': 3.400345125107852, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3689936', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1783, 'n_words': 255}