<commit_before>#!/usr/bin/env ruby
require "pathname"
ENV["BUNDLE_GEMFILE"] ||= File.expand_path "#{__FILE__}/../Gemfile"
require "rubygems"
require "bundler/setup"

require "twitter"
require "trollop"
require "csv"
require "dotenv"

MAX_API_TWEETS = 3200
MAX_TWEETS_PER_PAGE = 250
MAX_TWEETS_PER_REQUEST = 100

@options = Trollop::options do
  opt :force, "Actually delete/unfavourite/unretweet tweets"
  opt :user, "The Twitter username to purge", type: :string
  opt :csv, "Twitter archive tweets.csv file", type: :string
  opt :days, "Keep tweets under this many days old", default: 28
  opt :rts, "Keep tweet with this many retweets", default: 5
  opt :favs, "Keep tweets with this many favourites", default: 5
end

Trollop::die :user, "must be set" unless @options[:user_given]
if @options[:csv_given] && !File.exist?(@options[:csv])
  Trollop::die :csv, "must be a file that exists"
end

Dotenv.load
[ "TWITTER_CONSUMER_KEY", "TWITTER_CONSUMER_SECRET",
  "TWITTER_ACCESS_TOKEN", "TWITTER_ACCESS_TOKEN_SECRET"].each do |env|
  Trollop::die "#{env} environment variable must be set" unless ENV[env]
end

@client = Twitter::REST::Client.new do |config|
  config.consumer_key = ENV["TWITTER_CONSUMER_KEY"]
  config.consumer_secret = ENV["TWITTER_CONSUMER_SECRET"]
  config.access_token = ENV["TWITTER_ACCESS_TOKEN"]
  config.access_token_secret = ENV["TWITTER_ACCESS_TOKEN_SECRET"]
end

@oldest_tweet_time_to_keep = Time.now - @options[:days]*24*60*60<commit_msg>twitter_delete: try to get username from environment too.<commit_after>#!/usr/bin/env ruby
require "pathname"
ENV["BUNDLE_GEMFILE"] ||= File.expand_path "#{__FILE__}/../Gemfile"
require "rubygems"
require "bundler/setup"

require "twitter"
require "trollop"
require "csv"
require "dotenv"

MAX_API_TWEETS = 3200
MAX_TWEETS_PER_PAGE = 250
MAX_TWEETS_PER_REQUEST = 100

Dotenv.load

@options = Trollop::options do
  opt :force, "Actually delete/unfavourite/unretweet tweets"
  opt :user, "The Twitter username to purge", type: :string, default: ENV["TWITTER_USER"]
  opt :csv, "Twitter archive tweets.csv file", type: :string
  opt :days, "Keep tweets under this many days old", default: 28
  opt :rts, "Keep tweet with this many retweets", default: 5
  opt :favs, "Keep tweets with this many favourites", default: 5
end

Trollop::die :user, "must be set" if @options[:user].empty?
if @options[:csv_given] && !File.exist?(@options[:csv])
  Trollop::die :csv, "must be a file that exists"
end

[ "TWITTER_CONSUMER_KEY", "TWITTER_CONSUMER_SECRET",
  "TWITTER_ACCESS_TOKEN", "TWITTER_ACCESS_TOKEN_SECRET"].each do |env|
  Trollop::die "#{env} environment variable must be set" unless ENV[env]
end

@client = Twitter::REST::Client.new do |config|
  config.consumer_key = ENV["TWITTER_CONSUMER_KEY"]
  config.consumer_secret = ENV["TWITTER_CONSUMER_SECRET"]
  config.access_token = ENV["TWITTER_ACCESS_TOKEN"]
  config.access_token_secret = ENV["TWITTER_ACCESS_TOKEN_SECRET"]
end

@oldest_tweet_time_to_keep = Time.now - @options[:days]*24*60*60
{'fraction_non_alphanumeric': 0.1278295605858855, 'fraction_numerical': 0.013315579227696404, 'mean_word_length': 6.206235011990407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1257500', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1103, 'n_words': 276}