<commit_before>require "metaog/version"
require 'open-uri'
require 'nokogiri'

module Metaog
  # Your code goes here...
  def self.new url
    Core.new url
  end

  class Core
    attr_accessor :title, :url, :type, :image, :site_name, :description

    def initialize url
      charset = nil
      html = open(url) do |f|
        charset = f.charset
        f.read
      end
      @doc = Nokogiri::HTML.parse(html, nil, charset)

      @doc.xpath('//head/meta').each do |node|
        property = node.attr('property')

        # title
        if property == "og:title"
          @title = node.attr('content')
        end

        # url
        if property == "og:url"
          @url = node.attr('content')
        end

        # type
        if property == "og:type"
          @type = node.attr('content')
        end

        # image
        if property == "og:image"
          @image = node.attr('content')
        end

        # site_name
        if property == "og:site_name"
          @site_name = node.attr('content')
        end<commit_msg>encode utf8 and add ua.<commit_after># coding: utf-8
require "metaog/version"
require 'open-uri'
require 'nokogiri'

module Metaog
  # Your code goes here...
  def self.new url
    Core.new url
  end

  class Core
    attr_accessor :title, :url, :type, :image, :site_name, :description

    def initialize url
      html = open(url, 'User-Agent' => "metaog-bot/1.0.0") do |f|
        f.read
      end
      @doc = Nokogiri::HTML.parse(html, nil, 'utf-8')

      @doc.xpath('//head/meta').each do |node|
        property = node.attr('property')

        # title
        if property == "og:title"
          @title = node.attr('content')
        end

        # url
        if property == "og:url"
          @url = node.attr('content')
        end

        # type
        if property == "og:type"
          @type = node.attr('content')
        end

        # image
        if property == "og:image"
          @image = node.attr('content')
        end

        # site_name
        if property == "og:site_name"
          @site_name = node.attr('content')
        end
{'fraction_non_alphanumeric': 0.12998102466793168, 'fraction_numerical': 0.0028462998102466793, 'mean_word_length': 1.797082228116711, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2585943', 'n_tokens_mistral': 681, 'n_tokens_neox': 660, 'n_words': 172}