<commit_before>﻿using RedditSharp;
using RedditSharp.Things;
using SubredditPostGrabber.Holders;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SubredditPostGrabber.Utils;

namespace SubredditPostGrabber.Processors
{
    public class GetPostsFromReddit
    {
        public List<MattPost> AllPosts { get; set; }
        public string SaveDirectory { get; set; }
        public string Subreddit { get; set; }
        public string TemporarySaveFileFormatString { get { return Subreddit + "-{0}--{1}--{2}.xml"; } }
        public string SaveFileFormatString { get { return Subreddit + "_{0}_{1}.xml"; } }
        public string DateTimeFormatString = "yyyy-MM-dd_HH-mm-ss";

        public GetPostsFromReddit(string saveDirectory, string subreddit)
        {
            AllPosts = new List<MattPost>();
            SaveDirectory = saveDirectory;
            Subreddit = subreddit;
        }

        public void LoadPostsFromRedditAndSave()
        {
            LoadPostsFromRedditViaEnum();
            LoadPostsFromRedditViaSearching();
            LoadPostsByAuthor();
        }
        
        #region Utility
        /// <summary>
        /// Converts the reddit sharp post into a MattPost
        /// </summary>
        public MattPost ConvertRedditSharpPostIntoMattPost(Post post)
        {
            return new MattPost
            {
                Description = post.SelfText,
                URL = post.Url.ToString(),
                Title = post.Title,
                Votes = post.Upvotes,
                Id = post.Id,
                CommentCount = post.CommentCount,
                IsSelfPost = post.IsSelfPost,
                PostDate = post.CreatedUTC,
                Author = post.AuthorName,
                Tag = post.LinkFlairText,
            };
        }
        #endregion

        #region Get Posts Via Search
        public void LoadPostsFromRedditViaSearching()
        {
            var chars = "abcdefghijklmnopqrstuvwxyz".ToCharArray();
            foreach (var letter in chars)
            {
                var posts = new List<MattPost>();
                var started = DateTime.Now.ToString(DateTimeFormatString);
                int counter = 0;
                //1000 is all we can get :(
                var postCount = 1000;
                var lastCounterValue = counter;
                var numTimesStayedSame = 0;
                //postCount.Dump("Total number of Posts we are going to try to get!");
                while (counter < postCount)
                {
                    lastCounterValue = counter;

                    var reddit = new Reddit();
                    var subreddit = reddit.GetSubreddit(Subreddit);

                    foreach (var post in subreddit.GetAllViaSearch().Skip(counter).Take(100))
                    {
                        try
                        {
                            counter++;
                            var mattPost = ConvertRedditSharpPostIntoMattPost(post);
                            posts.Add(mattPost);
                            Console.WriteLine("{0} - {1}", counter, mattPost);
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine(ex.ToString());
                        }
                    }
                    SaveLoadData.SavePosts(Path.Combine(SaveDirectory, string.Format(TemporarySaveFileFormatString, started, letter, counter)), posts);
                    if (lastCounterValue == counter)
                    {
                        numTimesStayedSame++;
                    }
                    //If we've not changed the amount we're up to 4 times in a row, break out of here.
                    if (numTimesStayedSame > 4)
                    {
                        break;
                    }
                }
                SaveLoadData.SavePosts(Path.Combine(SaveDirectory, string.Format(SaveFileFormatString, letter, started)), posts);
            }
        }
        #endregion

        #region Get Posts Via Enum
        /// <summary>
        /// Loads the posts from reddit via enum.
        /// It gets posts by iterating through the enum and getting posts based off that
        /// </summary>
        public void LoadPostsFromRedditViaEnum()
        {
            foreach (GetBy getBy in (GetBy[])Enum.GetValues(typeof(GetBy)))
            {
                var posts = new List<MattPost>();
                var started = DateTime.Now.ToString(DateTimeFormatString);
                int counter = 0;
                //1000 is all we can get :(
                var postCount = 1000;
                var lastCounterValue = counter;
                var numTimesStayedSame = 0;

                while (counter < postCount)
                {
                    lastCounterValue = counter;
                    counter = GetPosts(counter, getBy, posts, out posts);
                    SaveLoadData.SavePosts(Path.Combine(SaveDirectory, string.Format(TemporarySaveFileFormatString, started, getBy, counter)), posts);
                    if (lastCounterValue == counter)
                    {
                        numTimesStayedSame++;
                    }
                    //If we've not changed the amount we're up to 4 times in a row, break out of here.
                    if (numTimesStayedSame > 4)
                    {
                        break;
                    }
                }
                SaveLoadData.SavePosts(Path.Combine(SaveDirectory, string.Format(SaveFileFormatString, getBy, started)), posts);
            }
        }

        /// <Summary>
        /// Gets Posts from reddit. Limited by 100 so we don't fall afoul of the timeout
        /// The counter is what we're up to in the collection of Posts.
        /// </Summary>
        /// <param name="counter">the number of Posts to skip before parsing the next 100 Posts</param>
        /// <param name="getBy">An enum which shows what to call in the Reddit API to get these Posts</param>
        private int GetPosts(int counter, GetBy getBy, List<MattPost> inPosts, out List<MattPost> posts)
        {
            posts = inPosts;
            try
            {
                var reddit = new Reddit();
                var subreddit = reddit.GetSubreddit(Subreddit);

                switch (getBy)
                {
                    case GetBy.GetAllViaSearch:
                        foreach (var post in subreddit.GetAllViaSearch().Skip(counter).Take(100))
                        {
                            counter++;
                            var mattPost = ConvertRedditSharpPostIntoMattPost(post);
                            posts.Add(mattPost);
                            Console.WriteLine("{0} - {1}", counter, mattPost);
                        }
                        break;<commit_msg>Moved code out of LoadPostsFromRedditViaEnum() into GetPostsByType so we can choose which to get - this way I could add a GetPostsByNew method.<commit_after>﻿using RedditSharp;
using RedditSharp.Things;
using SubredditPostGrabber.Holders;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SubredditPostGrabber.Utils;

namespace SubredditPostGrabber.Processors
{
    public class GetPostsFromReddit
    {
        public List<MattPost> AllPosts { get; set; }
        public string SaveDirectory { get; set; }
        public string Subreddit { get; set; }
        public string TemporarySaveFileFormatString { get { return Subreddit + "-{0}--{1}--{2}.xml"; } }
        public string SaveFileFormatString { get { return Subreddit + "_{0}_{1}.xml"; } }
        public string DateTimeFormatString = "yyyy-MM-dd_HH-mm-ss";

        public GetPostsFromReddit(string saveDirectory, string subreddit)
        {
            AllPosts = new List<MattPost>();
            SaveDirectory = saveDirectory;
            Subreddit = subreddit;
        }

        /// <summary>
        /// Loads the posts from reddit and saves to files.
        /// This is getting everything. All of it.
        /// </summary>
        public void LoadPostsFromRedditAndSave()
        {
            LoadPostsFromRedditViaEnum();
            LoadPostsFromRedditViaSearching();
            LoadPostsByAuthor();
        }

        public void LoadNewPostsFromRedditAndSave()
        {
            GetPostsByType(GetBy.New);
        }
        
        #region Utility
        /// <summary>
        /// Converts the reddit sharp post into a MattPost
        /// </summary>
        public MattPost ConvertRedditSharpPostIntoMattPost(Post post)
        {
            return new MattPost
            {
                Description = post.SelfText,
                URL = post.Url.ToString(),
                Title = post.Title,
                Votes = post.Upvotes,
                Id = post.Id,
                CommentCount = post.CommentCount,
                IsSelfPost = post.IsSelfPost,
                PostDate = post.CreatedUTC,
                Author = post.AuthorName,
                Tag = post.LinkFlairText,
            };
        }
        #endregion

        #region Get Posts Via Search
        public void LoadPostsFromRedditViaSearching()
        {
            var chars = "abcdefghijklmnopqrstuvwxyz".ToCharArray();
            foreach (var letter in chars)
            {
                var posts = new List<MattPost>();
                var started = DateTime.Now.ToString(DateTimeFormatString);
                int counter = 0;
                //1000 is all we can get :(
                var postCount = 1000;
                var lastCounterValue = counter;
                var numTimesStayedSame = 0;
                //postCount.Dump("Total number of Posts we are going to try to get!");
                while (counter < postCount)
                {
                    lastCounterValue = counter;

                    var reddit = new Reddit();
                    var subreddit = reddit.GetSubreddit(Subreddit);

                    foreach (var post in subreddit.GetAllViaSearch().Skip(counter).Take(100))
                    {
                        try
                        {
                            counter++;
                            var mattPost = ConvertRedditSharpPostIntoMattPost(post);
                            posts.Add(mattPost);
                            Console.WriteLine("{0} - {1}", counter, mattPost);
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine(ex.ToString());
                        }
                    }
                    SaveLoadData.SavePosts(Path.Combine(SaveDirectory, string.Format(TemporarySaveFileFormatString, started, letter, counter)), posts);
                    if (lastCounterValue == counter)
                    {
                        numTimesStayedSame++;
                    }
                    //If we've not changed the amount we're up to 4 times in a row, break out of here.
                    if (numTimesStayedSame > 4)
                    {
                        break;
                    }
                }
                SaveLoadData.SavePosts(Path.Combine(SaveDirectory, string.Format(SaveFileFormatString, letter, started)), posts);
            }
        }
        #endregion

        #region Get Posts Via Enum
        /// <summary>
        /// Loads the posts from reddit via enum.
        /// It gets posts by iterating through the enum and getting posts based off that
        /// </summary>
        public void LoadPostsFromRedditViaEnum()
        {
            foreach (GetBy getBy in (GetBy[])Enum.GetValues(typeof(GetBy)))
            {
                GetPostsByType(getBy); 
            }
        }

        public void GetPostsByType(GetBy getBy)
        {
            var posts = new List<MattPost>();
            var started = DateTime.Now.ToString(DateTimeFormatString);
            int counter = 0;
            //1000 is all we can get :(
            var postCount = 1000;
            var lastCounterValue = counter;
            var numTimesStayedSame = 0;

            while (counter < postCount)
            {
                lastCounterValue = counter;
                counter = GetPosts(counter, getBy, posts, out posts);
                SaveLoadData.SavePosts(Path.Combine(SaveDirectory, string.Format(TemporarySaveFileFormatString, started, getBy, counter)), posts);
                if (lastCounterValue == counter)
                {
                    numTimesStayedSame++;
                }
                //If we've not changed the amount we're up to 4 times in a row, break out of here.
                if (numTimesStayedSame > 4)
                {
                    break;
                }
            }
            SaveLoadData.SavePosts(Path.Combine(SaveDirectory, string.Format(SaveFileFormatString, getBy, started)), posts);
        }

        /// <Summary>
        /// Gets Posts from reddit. Limited by 100 so we don't fall afoul of the timeout
        /// The counter is what we're up to in the collection of Posts.
        /// </Summary>
        /// <param name="counter">the number of Posts to skip before parsing the next 100 Posts</param>
        /// <param name="getBy">An enum which shows what to call in the Reddit API to get these Posts</param>
        private int GetPosts(int counter, GetBy getBy, List<MattPost> inPosts, out List<MattPost> posts)
        {
            posts = inPosts;
            try
            {
                var reddit = new Reddit();
                var subreddit = reddit.GetSubreddit(Subreddit);

                switch (getBy)
                {
                    case GetBy.GetAllViaSearch:
                        foreach (var post in subreddit.GetAllViaSearch().Skip(counter).Take(100))
                        {
                            counter++;
                            var mattPost = ConvertRedditSharpPostIntoMattPost(post);
                            posts.Add(mattPost);
                            Console.WriteLine("{0} - {1}", counter, mattPost);
                        }
                        break;
{'fraction_non_alphanumeric': 0.08175131348511383, 'fraction_numerical': 0.006304728546409807, 'mean_word_length': 1.3801267089029676, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3014046', 'n_tokens_mistral': 3673, 'n_tokens_neox': 3366, 'n_words': 935}