<commit_before>require 'set'
require 'bundler'
Bundler.require

# Set environment variable TWINGLY_ANALYTICS_KEY
client = Twingly::Analytics::Client.new

finished   = nil
start_time = Time.new(2013, 01, 01)
blogs      = Set.new

until finished
  query = client.query
  query.pattern    = "sort-order:asc sort:published github"
  query.start_time = start_time
  query.language   = 'sv'
  result = query.execute

  result.posts.each do |post|
    blogs.add(post.blog_url)
  end

  finished = start_time == result.posts.last.published
  start_time = result.posts.last.published
end

blogs.each do |url|
  puts url
end<commit_msg>Rewrite github example<commit_after>Bundler.require

class AnalyticsPostStream
  def initialize(keyword, language: nil)
    # Set environment variable TWINGLY_ANALYTICS_KEY
    client = Twingly::Analytics::Client.new
    @query = client.query
    @query.language = language
    @query.pattern = "sort-order:asc sort:published #{keyword}"
  end

  def each
    loop do
      result = @query.execute
      result.posts.each do |post|
        yield post
      end

      break if result.all_results_returned?

      @query.start_time = result.posts.last.published
    end
  end
end

stream = AnalyticsPostStream.new("(github) AND (hipchat OR slack)")
stream.each do |post|
  puts post.url
end
{'fraction_non_alphanumeric': 0.0935361216730038, 'fraction_numerical': 0.006083650190114068, 'mean_word_length': 3.5853658536585367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4219658', 'n_tokens_mistral': 441, 'n_tokens_neox': 417, 'n_words': 105}