<commit_before># Author: <NAME> <<EMAIL>>
require 'fileutils'
require 'rubygems'
require 'sequel'
require 'safe_yaml'

module JekyllImport
  module Typo
    # This SQL *should* work for both MySQL and PostgreSQL.
    SQL = <<-EOS
    SELECT c.id id,
           c.title title,
           c.permalink slug,
           c.body body,
           c.extended extended,
           c.published_at date,
           c.state state,
           COALESCE(tf.name, 'html') filter
      FROM contents c
           LEFT OUTER JOIN text_filters tf
                        ON c.text_filter_id = tf.id
    EOS

    def self.process server, dbname, user, pass, host='localhost'
      FileUtils.mkdir_p '_posts'
      case server.intern
      when :postgres
        db = Sequel.postgres(dbname, :user => user, :password => <PASSWORD>, :host => host, :encoding => 'utf8')
      when :mysql
        db = Sequel.mysql(dbname, :user => user, :password => <PASSWORD>, :host => host, :encoding => 'utf8')
      else
        raise "Unknown database server '#{server}'"
      end
      db[SQL].each do |post|
        next unless post[:state] =~ /published/

        if post[:slug] == nil
          post[:slug] = "no slug"
        end

        if post[:extended]
          post[:body] << "\n<!-- more -->\n"
          post[:body] << post[:extended]
        end

        name = [ sprintf("%.04d", post[:date].year),
                 sprintf("%.02d", post[:date].month),
                 sprintf("%.02d", post[:date].day),
                 post[:slug].strip ].join('-')

        # Can have more than one text filter in this field, but we just want
        # the first one for this.
        name += '.' + post[:filter].split(' ')[0]

        File.open("_posts/#{name}", 'w') do |f|
          f.puts({ 'layout'   => 'post',
                   'title'    => post[:title].to_s,
                   'typo_id'  => post[:id]
                 }.delete_if { |k, v| v.nil? || v == '' }.to_yaml)
          f.puts '---'
          f.puts post[:body].delete("\r")
        end
      end
    end

  end
end<commit_msg>Adding tags to Typo and forcing their encoding to UTF-8<commit_after># Author: <NAME> <<EMAIL>>
require 'fileutils'
require 'rubygems'
require 'sequel'
require 'safe_yaml'

module JekyllImport
  module Typo
    # This SQL *should* work for both MySQL and PostgreSQL.
    SQL = <<-EOS
    SELECT c.id id,
           c.title title,
           c.permalink slug,
           c.body body,
           c.extended extended,
           c.published_at date,
           c.state state,
           c.keywords keywords,
           COALESCE(tf.name, 'html') filter
      FROM contents c
           LEFT OUTER JOIN text_filters tf
                        ON c.text_filter_id = tf.id
    EOS

    def self.process server, dbname, user, pass, host='localhost'
      FileUtils.mkdir_p '_posts'
      case server.intern
      when :postgres
        db = Sequel.postgres(dbname, :user => user, :password => <PASSWORD>, :host => host, :encoding => 'utf8')
      when :mysql
        db = Sequel.mysql(dbname, :user => user, :password => <PASSWORD>, :host => host, :encoding => 'utf8')
      else
        raise "Unknown database server '#{server}'"
      end
      db[SQL].each do |post|
        next unless post[:state] =~ /published/i

        if post[:slug] == nil
          post[:slug] = "no slug"
        end

        if post[:extended]
          post[:body] << "\n<!-- more -->\n"
          post[:body] << post[:extended]
        end

        name = [ sprintf("%.04d", post[:date].year),
                 sprintf("%.02d", post[:date].month),
                 sprintf("%.02d", post[:date].day),
                 post[:slug].strip ].join('-')

        # Can have more than one text filter in this field, but we just want
        # the first one for this.
        name += '.' + post[:filter].split(' ')[0]

        File.open("_posts/#{name}", 'w') do |f|
          f.puts({ 'layout'   => 'post',
                   'title'    => (post[:title] and post[:title].to_s.force_encoding('UTF-8')),
                   'tags'     => (post[:keywords] and post[:keywords].to_s.force_encoding('UTF-8')),
                   'typo_id'  => post[:id]
                 }.delete_if { |k, v| v.nil? || v == '' }.to_yaml)
          f.puts '---'
          f.puts post[:body].delete("\r")
        end
      end
    end

  end
end
{'fraction_non_alphanumeric': 0.15076993794529994, 'fraction_numerical': 0.004826476672029419, 'mean_word_length': 1.7969151670951158, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4801418', 'n_tokens_mistral': 1384, 'n_tokens_neox': 1341, 'n_words': 372}