<commit_before>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Web.Script.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using NLog;
using Offr.Json;
using Offr.Location;
using Offr.Message;
using Offr.Query;
using Offr.Repository;
using Offr.Json.Converter;
using Offr.Text;
using Offr.Twitter;

namespace Offr.Repository
{
    public class MessageRepository : BaseRepository<IMessage>, IMessageRepository, IPersistedRepository, IEnumerable<IMessage>
    {
        private static readonly Logger _log = LogManager.GetCurrentClassLogger();

        private readonly TagDex _globalTagIndex;

        public MessageRepository(IIgnoredUserRepository ignoredUsers)
        {
            _globalTagIndex = new TagDex(this,ignoredUsers);
        }

        public MessageRepository()
        {
            _globalTagIndex = new TagDex(this);    
        }

        public int MessageCount
        {
             get
             {
                 return base._list.Count;   
             }   
        }

        public IEnumerable<IMessage> AllMessages()
        {
            return base.GetAll();
        }

        public override void Save(IMessage message)
        {
            base.Save(message);
            _log.Info("Save:" + message);
            _globalTagIndex.Process(message);
        }

        public IEnumerable<IMessage> GetMessagesForTags(IEnumerable<ITag> tags, bool includeIgnoredUsers)
        {
            return _globalTagIndex.MessagesForTags(tags,includeIgnoredUsers);
        }

        public IEnumerable<IMessage> GetMessagesForTagsCreatedByUser(IEnumerable<ITag> tags, IUserPointer userPointer)
        {
            //not sure if 'true' is correct
            return _globalTagIndex.MessagesForTagsAndUser(tags, userPointer,true);
        }

        public IEnumerable<IMessage> GetMessagesCreatedByUser(IUserPointer userPointer)
        {
            return _globalTagIndex.MessagesForUser(userPointer);
        }

        public TagCounts GetTagCounts()
        {
            return _globalTagIndex.GetTagCounts();
        }

        public IEnumerator<IMessage> GetEnumerator()
        {
            return base.GetAll().GetEnumerator();
        }

        IEnumerator IEnumerable.GetEnumerator()
        {
            return GetEnumerator();
        }
        
        public TagCounts GetTagCountsForTags(IEnumerable<ITag> tags)
        {
            var tagCounts = new List<TagWithCount>();
            foreach (ITag tag in tags)
            {
                tagCounts.Add(_globalTagIndex.GetTagCountForTag(tag));
            }

            //tagCounts.Reverse();
            return new TagCounts() { Tags = tagCounts, Total = -1 };
        }
        public void InitializeFromFile()
        {
            var dummy = new OpenSocialUserPointer("ooooby", "Dummy",
                                                  "http://s3.amazonaws.com/twitter_production/profile_images/228862942/YinD_ContactSheet-003_normal.jpg",
                                                  "");
            for (int i = 0; i < 10; i++)
            {
                OfferMessage msg = new OfferMessage();
                msg.CreatedBy = dummy;
                ////msg.Source = source; //Remove this
                msg.Timestamp = DateTime.MinValue;
                msg.MessagePointer = new OpenSocialMessagePointer("ooooby");
                msg.RawText = "";
                msg.OfferText = "";
                msg.MoreInfoURL = "";
                msg.AddTag(new Tag(TagType.tag, "Ooooby"));
                msg.AddTag(new Tag(TagType.tag, "Ooooby_Cool"));
                this.Save(msg);
            }           
            //msg.AddThumbnail(GetImageUrl(sourceText));
            base.InitializeFromFile();
        }
        //public IEnumerable<IMessage> GetMessagesForKeywordAndTags(string keyword, IEnumerable<ITag> tags)
        //{
        //    // theoretically if we were caching all these objects this use of a 
        //    // MessageReceivingTagDex would mean that we get up dates from the feed
        //    // when new messages match the query, but.. since we don't cache it anyway, that
        //    // is kind of pointless
        //    //MessageProviderForKeywords 
        //    MessageProviderForKeywords keywordMessageProvider = Global.Kernel.Get<MessageProviderForKeywords>(With.Parameters.ConstructorArgument("keywords", tags));
        //    MessageReceivingTagDex tagDex = new MessageReceivingTagDex(keywordMessageProvider);
        //    return tagDex.MessagesForTags(tags);
        //}

        //public TagCounts GetTagCountsForKeywordAndTags(string keyword, IEnumerable<ITag> tags)
        //{
        //    MessageProviderForKeywords keywordMessageProvider = Global.Kernel.Get<MessageProviderForKeywords>(With.Parameters.ConstructorArgument("keywords", tags));
        //    StaticTagDex tagDex = new StaticTagDex(keywordMessageProvider.AllMessages);
        //    return tagDex.GetTagCounts();
        //}

    }<commit_msg>Added some proper dummy data- this data will continue getting larger everytime you restart the server tho!<commit_after>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Web.Script.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using NLog;
using Offr.Json;
using Offr.Location;
using Offr.Message;
using Offr.Query;
using Offr.Repository;
using Offr.Json.Converter;
using Offr.Text;
using Offr.Twitter;

namespace Offr.Repository
{
    public class MessageRepository : BaseRepository<IMessage>, IMessageRepository, IPersistedRepository, IEnumerable<IMessage>
    {
        private static readonly Logger _log = LogManager.GetCurrentClassLogger();

        private readonly TagDex _globalTagIndex;

        public MessageRepository(IIgnoredUserRepository ignoredUsers)
        {
            _globalTagIndex = new TagDex(this, ignoredUsers);
        }

        public MessageRepository()
        {
            _globalTagIndex = new TagDex(this);
        }

        public int MessageCount
        {
            get
            {
                return base._list.Count;
            }
        }

        public IEnumerable<IMessage> AllMessages()
        {
            return base.GetAll();
        }

        public override void Save(IMessage message)
        {
            base.Save(message);
            _log.Info("Save:" + message);
            _globalTagIndex.Process(message);
        }

        public IEnumerable<IMessage> GetMessagesForTags(IEnumerable<ITag> tags, bool includeIgnoredUsers)
        {
            return _globalTagIndex.MessagesForTags(tags, includeIgnoredUsers);
        }

        public IEnumerable<IMessage> GetMessagesForTagsCreatedByUser(IEnumerable<ITag> tags, IUserPointer userPointer)
        {
            //not sure if 'true' is correct
            return _globalTagIndex.MessagesForTagsAndUser(tags, userPointer, true);
        }

        public IEnumerable<IMessage> GetMessagesCreatedByUser(IUserPointer userPointer)
        {
            return _globalTagIndex.MessagesForUser(userPointer);
        }

        public TagCounts GetTagCounts()
        {
            return _globalTagIndex.GetTagCounts();
        }

        public IEnumerator<IMessage> GetEnumerator()
        {
            return base.GetAll().GetEnumerator();
        }

        IEnumerator IEnumerable.GetEnumerator()
        {
            return GetEnumerator();
        }

        public TagCounts GetTagCountsForTags(IEnumerable<ITag> tags)
        {
            var tagCounts = new List<TagWithCount>();
            foreach (ITag tag in tags)
            {
                tagCounts.Add(_globalTagIndex.GetTagCountForTag(tag));
            }

            //tagCounts.Reverse();
            return new TagCounts() { Tags = tagCounts, Total = -1 };
        }
        /*
          *Fruit
Vegetables
Eggs
Honey
Nuts & Beans
Grains
--
Food Growing Books
Food Growing Consultancy
Food Transport
Food Trees
Gardening and Yard Work
Glass/Shade Houses
Incidentals; Piping, Net, etc.
Prepared Food
Recipe Books
Seedlings
Seeds
Soils & Fertilisers
Teaching
Tools & Equipment
Workshops
Miscellaneous  
          */

        public void InitializeFromFile()
        {
            
            addDummyMessages();
            //msg.AddThumbnail(GetImageUrl(sourceText));
            base.InitializeFromFile();
        }
        private static void addDummyMessages()
        {
            var dummy = new OpenSocialUserPointer("ooooby", "Dummy",
                                               "http://s3.amazonaws.com/twitter_production/profile_images/228862942/YinD_ContactSheet-003_normal.jpg",
                                               "");

            saveDummyMessage(dummy, "ooooby", "Vegetables");
            saveDummyMessage(dummy, "ooooby", "Fruit");
            saveDummyMessage(dummy, "ooooby", "Eggs");
            saveDummyMessage(dummy, "ooooby", "Honey");
            saveDummyMessage(dummy, "ooooby", "Nuts & Beans");
            saveDummyMessage(dummy, "ooooby", "Grains");
            saveDummyMessage(dummy, "ooooby", "Food Growing Books");

        }
        private static void saveDummyMessage(IUserPointer dummy, string group, string term)
        {
            for (int i = 0; i < 10; i++)
            {
                OfferMessage msg = new OfferMessage();
                msg.CreatedBy = dummy;
                ////msg.Source = source; //Remove this
                msg.Timestamp = DateTime.MinValue;
                msg.MessagePointer = new OpenSocialMessagePointer("ooooby");
                msg.RawText = "";
                msg.OfferText = "";
                msg.MoreInfoURL = "";
                msg.AddTag(new Tag(TagType.tag, group));
                msg.AddTag(new Tag(TagType.tag, term));
                this.Save(msg);
            }
        }
        //public IEnumerable<IMessage> GetMessagesForKeywordAndTags(string keyword, IEnumerable<ITag> tags)
        //{
        //    // theoretically if we were caching all these objects this use of a 
        //    // MessageReceivingTagDex would mean that we get up dates from the feed
        //    // when new messages match the query, but.. since we don't cache it anyway, that
        //    // is kind of pointless
        //    //MessageProviderForKeywords 
        //    MessageProviderForKeywords keywordMessageProvider = Global.Kernel.Get<MessageProviderForKeywords>(With.Parameters.ConstructorArgument("keywords", tags));
        //    MessageReceivingTagDex tagDex = new MessageReceivingTagDex(keywordMessageProvider);
        //    return tagDex.MessagesForTags(tags);
        //}

        //public TagCounts GetTagCountsForKeywordAndTags(string keyword, IEnumerable<ITag> tags)
        //{
        //    MessageProviderForKeywords keywordMessageProvider = Global.Kernel.Get<MessageProviderForKeywords>(With.Parameters.ConstructorArgument("keywords", tags));
        //    StaticTagDex tagDex = new StaticTagDex(keywordMessageProvider.AllMessages);
        //    return tagDex.GetTagCounts();
        //}

    }
{'fraction_non_alphanumeric': 0.08911861687906604, 'fraction_numerical': 0.0030300329738882454, 'mean_word_length': 2.078737997256516, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7376964', 'n_tokens_mistral': 3031, 'n_tokens_neox': 2912, 'n_words': 633}