<commit_before>require 'purdie'
require 'cgi'
require 'google/api_client'

module Purdie
  module Services
    class YouTube
      include Ingester

      def configure
        @matcher = 'youtube.com'

        super
      end

      def client
        @client ||= Google::APIClient.new(
            key: ENV['YOUTUBE_API_KEY'],
            authorization: nil,
            application_name: self.class.name.split('::').first,
            application_version: Purdie::VERSION
          )
      end

      def yt_service
        @yt_service ||= client.discovered_api(
          ENV['YOUTUBE_API_SERVICE_NAME'],
          ENV['YOUTUBE_API_VERSION']
        )
      end

      def get_data id, part
        client.execute!(
          api_method: yt_service.videos.list,
          parameters: {
            id: id,
            part: part
          }
        )
      end

      def get url
        data = get_data YouTube.get_id(url), 'status,snippet'
        JSON.parse data.body
      end

      def distill url
        video = get url
        results = {}

        results['title'] = video['items'][0]['snippet']['localized']['title']
        results['id'] = YouTube.get_id(url)
        results['license'] = @config['license_lookups'][video['items'][0]['status']['license']]['full_name']
        results['license_url'] = @config['license_lookups'][video['items'][0]['status']['license']]['url']

        results
      end

      def self.get_id url
        params = CGI.parse(URI.parse(url).query)
        params['v'].first
      end
    end
  end
end
<commit_msg>These are not ENV things<commit_after>require 'purdie'
require 'cgi'
require 'google/api_client'

module Purdie
  module Services
    class YouTube
      include Ingester

      def configure
        @matcher = 'youtube.com'
        @api_service_name = 'youtube'
        @api_version = 'v3'

        super
      end

      def client
        @client ||= Google::APIClient.new(
            key: ENV['YOUTUBE_API_KEY'],
            authorization: nil,
            application_name: self.class.name.split('::').first,
            application_version: Purdie::VERSION
          )
      end

      def yt_service
        @yt_service ||= client.discovered_api(
          @api_service_name,
          @api_version
        )
      end

      def get_data id, part
        client.execute!(
          api_method: yt_service.videos.list,
          parameters: {
            id: id,
            part: part
          }
        )
      end

      def get url
        data = get_data YouTube.get_id(url), 'status,snippet'
        JSON.parse data.body
      end

      def distill url
        video = get url
        results = {}

        results['title'] = video['items'][0]['snippet']['localized']['title']
        results['id'] = YouTube.get_id(url)
        results['license'] = @config['license_lookups'][video['items'][0]['status']['license']]['full_name']
        results['license_url'] = @config['license_lookups'][video['items'][0]['status']['license']]['url']

        results
      end

      def self.get_id url
        params = CGI.parse(URI.parse(url).query)
        params['v'].first
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.12488174077578051, 'fraction_numerical': 0.002207505518763797, 'mean_word_length': 1.995278564683664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '642591', 'n_tokens_mistral': 956, 'n_tokens_neox': 942, 'n_words': 188}