<commit_before>using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Sloader.Types;

namespace Sloader.Crawler.Twitter
{
    public class TwitterTimelineCrawler : ICrawler<List<TwitterTimelineCrawlerResult>>
    {
        private readonly ITwitterOAuthTokenLoader _oAuthTokenLoader;

        public TwitterTimelineCrawler() : this(new TwitterOAuthTokenLoader())
        {
        }

        public TwitterTimelineCrawler(ITwitterOAuthTokenLoader oAuthTokenLoader)
        {
            _oAuthTokenLoader = oAuthTokenLoader;
            Config = new TwitterTimelineCrawlerConfig();
        }

        public TwitterTimelineCrawlerConfig Config { get; set; }

        public async Task<List<TwitterTimelineCrawlerResult>> DoWorkAsync()
        {
            var results = new List<TwitterTimelineCrawlerResult>();

            var oauth = await _oAuthTokenLoader.GetAsync(Config.ConsumerKey, Config.ConsumerSecret);
            if (oauth == string.Empty)
                return results;

            foreach (var handle in Config.Handles.Split(';'))
            {
                var result = new TwitterTimelineCrawlerResult();

                result.Key = handle;
                result.Type = KnownCrawler.TwitterTimeline;
                result.Tweets = new List<TwitterTimelineCrawlerResult.Tweet>();

                var twitterResult = await GetTwitterTimeline(oauth, handle);
                result.Tweets.AddRange(new List<TwitterTimelineCrawlerResult.Tweet>(twitterResult));

                results.Add(result);
            }

            return results;
        }
    
        private static async Task<List<TwitterTimelineCrawlerResult.Tweet>> GetTwitterTimeline(string oauthToken, string screenname)
        {
            Trace.TraceInformation("GetTwitterTimeline invoked with screenname:" + screenname);

            var client = new HttpClient();

            var timelineFormat = "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name={0}&include_rts=1&exclude_replies=1&count=5";
            var timelineUrl = string.Format(timelineFormat, screenname);

            client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", oauthToken);
            var response = await client.GetAsync(timelineUrl);

            response.EnsureSuccessStatusCode();

            string timeline = await response.Content.ReadAsStringAsync();

            var resultForThisHandle = JsonConvert.DeserializeObject<List<TwitterTimelineCrawlerResult.Tweet>>(timeline);

            return resultForThisHandle;
        }
    }
}<commit_msg>unittest friendly twitertimelinecrawler<commit_after>using System.Collections.Generic;
using System.Diagnostics;
using System.Net.Http.Headers;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Sloader.Types;
using WorldDomination.Net.Http;

namespace Sloader.Crawler.Twitter
{
    public class TwitterTimelineCrawler : ICrawler<List<TwitterTimelineCrawlerResult>>
    {
        private readonly ITwitterOAuthTokenLoader _oAuthTokenLoader;

        public TwitterTimelineCrawler() : this(new TwitterOAuthTokenLoader())
        {
        }

        public TwitterTimelineCrawler(ITwitterOAuthTokenLoader oAuthTokenLoader)
        {
            _oAuthTokenLoader = oAuthTokenLoader;
            Config = new TwitterTimelineCrawlerConfig();
        }

        public TwitterTimelineCrawlerConfig Config { get; set; }

        public async Task<List<TwitterTimelineCrawlerResult>> DoWorkAsync()
        {
            var results = new List<TwitterTimelineCrawlerResult>();

            var oauth = await _oAuthTokenLoader.GetAsync(Config.ConsumerKey, Config.ConsumerSecret);
            if (oauth == string.Empty)
                return results;

            foreach (var handle in Config.Handles.Split(';'))
            {
                var result = new TwitterTimelineCrawlerResult();

                result.Key = handle;
                result.Type = KnownCrawler.TwitterTimeline;
                result.Tweets = new List<TwitterTimelineCrawlerResult.Tweet>();

                var twitterResult = await GetTwitterTimeline(oauth, handle);
                result.Tweets.AddRange(new List<TwitterTimelineCrawlerResult.Tweet>(twitterResult));

                results.Add(result);
            }

            return results;
        }
    
        private static async Task<List<TwitterTimelineCrawlerResult.Tweet>> GetTwitterTimeline(string oauthToken, string screenname)
        {
            Trace.TraceInformation("GetTwitterTimeline invoked with screenname:" + screenname);

            using (var httpClient = HttpClientFactory.GetHttpClient())
            {
                var timelineFormat =
                    "https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name={0}&include_rts=1&exclude_replies=1&count=5";
                var timelineUrl = string.Format(timelineFormat, screenname);

                httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", oauthToken);
                var response = await httpClient.GetAsync(timelineUrl);

                response.EnsureSuccessStatusCode();

                string timeline = await response.Content.ReadAsStringAsync();

                var resultForThisHandle =
                    JsonConvert.DeserializeObject<List<TwitterTimelineCrawlerResult.Tweet>>(timeline);

                return resultForThisHandle;
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.08079729489232959, 'fraction_numerical': 0.002135611318739989, 'mean_word_length': 2.6540962288686605, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6609622', 'n_tokens_mistral': 1522, 'n_tokens_neox': 1436, 'n_words': 262}