<commit_before>require 'oga'
require 'open-uri'

class VideoInfo
  module Providers
    module YoutubeScraper
      def date
        if available?
          date_published_node = meta_nodes.detect do |m|
            itemprop_attr = m.attr('itemprop')

            itemprop_attr.value == 'datePublished' unless !itemprop_attr
          end

          Time.parse(date_published_node.attr('content').value)
        end
      end

      def description
        if available?
          description_node = meta_nodes.detect do |m|
            m.attr('name').value == 'description'
          end

          description_node.attr('content').value
        end
      end

      def duration
        if available?
          duration_node = meta_nodes.detect do |m|
            itemprop_attr = m.attr('itemprop')

            itemprop_attr.value == 'duration' unless !itemprop_attr
          end

          duration = duration_node.attr('content').value

          ISO8601::Duration.new(duration).to_seconds.to_i
        else
          0
        end
      end

      def keywords
        value = meta_node_value('keywords')

        if value
          value.split(', ')
        end
      end

      def title
        meta_node_value('title')
      end

      def thumbnail_small
        "https://i.ytimg.com/vi/#{video_id}/default.jpg"
      end

      def thumbnail_medium
        "https://i.ytimg.com/vi/#{video_id}/mqdefault.jpg"
      end

      def thumbnail_large
        "https://i.ytimg.com/vi/#{video_id}/hqdefault.jpg"
      end

      def view_count
        data.css('div.watch-view-count').text.gsub(',', '').to_i
      end

      private

      def meta_nodes
        @meta_nodes ||= data.css('meta')
      end

      def meta_node_value(name)
        if available?
          node = meta_nodes.detect { |n| n.attr('name').value == name }

          node.attr('content').value
        end
      end

      def available?
        data.css('div#unavailable-submessage').text.strip.empty?
      end

      def _set_data_from_api(api_url = _api_url)
        Oga.parse_html(open(api_url, allow_redirections: :safe))
      end

      def _api_url
        @url
      end
    end
  end
end
<commit_msg>more cleanup<commit_after>require 'oga'
require 'open-uri'

class VideoInfo
  module Providers
    module YoutubeScraper
      def date
        if available?
          date_published_node = meta_nodes.detect do |m|
            itemprop_attr = m.attr('itemprop')

            itemprop_attr.value == 'datePublished' unless !itemprop_attr
          end

          Time.parse(date_published_node.attr('content').value)
        end
      end

      def description
        meta_node_value('description')
      end

      def duration
        if available?
          duration_node = meta_nodes.detect do |m|
            itemprop_attr = m.attr('itemprop')

            itemprop_attr.value == 'duration' unless !itemprop_attr
          end

          duration = duration_node.attr('content').value

          ISO8601::Duration.new(duration).to_seconds.to_i
        else
          0
        end
      end

      def keywords
        value = meta_node_value('keywords')

        value.split(', ') unless !value
      end

      def title
        meta_node_value('title')
      end

      def thumbnail_small
        "https://i.ytimg.com/vi/#{video_id}/default.jpg"
      end

      def thumbnail_medium
        "https://i.ytimg.com/vi/#{video_id}/mqdefault.jpg"
      end

      def thumbnail_large
        "https://i.ytimg.com/vi/#{video_id}/hqdefault.jpg"
      end

      def view_count
        data.css('div.watch-view-count').text.gsub(',', '').to_i
      end

      private

      def meta_nodes
        @meta_nodes ||= data.css('meta')
      end

      def meta_node_value(name)
        if available?
          node = meta_nodes.detect { |n| n.attr('name').value == name }

          node.attr('content').value
        end
      end

      def available?
        data.css('div#unavailable-submessage').text.strip.empty?
      end

      def _set_data_from_api(api_url = _api_url)
        Oga.parse_html(open(api_url, allow_redirections: :safe))
      end

      def _api_url
        @url
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.09983361064891846, 'fraction_numerical': 0.002376990729736154, 'mean_word_length': 1.8222669349429912, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '813899', 'n_tokens_mistral': 1311, 'n_tokens_neox': 1253, 'n_words': 250}