<commit_before>/*
 * Copyright (C) 2013 <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package main

import (
	"bytes"
	"encoding/binary"

	"encoding/base64"

	"net/url"
	"time"

	riak "github.com/tpjg/goriakpbc"
)

type Feed struct {
	Url url.URL `riak:"url"`

	Title     string    `riak:"title"`
	LastCheck time.Time `riak:"last_check"`

	ItemKeys        ItemKeyList `riak:"item_keys"`
	DeletedItemKeys ItemKeyList `riak:"deleted_items"`

	NextCheck time.Time `riak:"next_check"`

	riak.Model `riak:"feeds"`
}

type FeedItem struct {
	Title   string `riak:"title"`
	Author  string `riak:"author"`
	Content string `riak:"content"`

	Url url.URL `riak:"url"`

	PubDate time.Time `riak:"publication_date"`

	riak.Model `riak:"items"`
}

type ItemKey []byte

func NewItemKey(id uint64, rawId []byte) ItemKey {
	buf := &bytes.Buffer{}

	binary.Write(buf, binary.BigEndian, id)
	buf.WriteString("-")
	buf.Write(rawId)

	return buf.Bytes()
}

func (l ItemKey) Less(r Comparable) bool {
	return bytes.Compare(l, r.(ItemKey)) == -1
}
func (l *ItemKey) UnmarshalJSON(input []byte) error {
	input = input[1 : len(input)-1]

	*l = make(ItemKey, base64.StdEncoding.DecodedLen(len(input)))
	n, err := base64.StdEncoding.Decode(*l, input)

	if err != nil {
		return err
	} else {
		*l = (*l)[0:n]
		return nil
	}
}

type ItemKeyList []ItemKey

func (list *ItemKeyList) Append(key Comparable) {
	*list = append(*list, key.(ItemKey))
}

func (list ItemKeyList) Get(index int) Comparable {
	return list[index]
}

func (list *ItemKeyList) RemoveAt(index int) {
	*list = append((*list)[:index], (*list)[index+1:]...)
}

func (list ItemKeyList) Len() int {
	return len(list)
}

func (ItemKeyList) Make() ComparableArray {
	return &ItemKeyList{}
}

func (f *Feed) Resolve(siblingsCount int) error {
	// First get the siblings!
	siblingsI, err := f.Siblings(&Feed{})
	if err != nil {
		return err
	}
	siblings := siblingsI.([]Feed)

	// Set the Url, as it is constant
	f.Url = siblings[0].Url

	for i := 0; i < siblingsCount; i++ {
		// Resolve regular feed details.  Basically, take the latest version!
		// If this is the first object, it will have a zero time of year 1st.  If a feed is claiming
		// be older then that, well it just won't work.
		if siblings[i].LastCheck.After(f.LastCheck) {
			f.Title = siblings[i].Title
			f.LastCheck = siblings[i].LastCheck
			f.NextCheck = siblings[i].NextCheck
		}

		// for the item lists, merge and de-dup using insert slice sort!
		f.ItemKeys = *InsertSliceSort(&f.ItemKeys, &siblings[i].ItemKeys).(*ItemKeyList)
		f.DeletedItemKeys = *InsertSliceSort(&f.DeletedItemKeys, &siblings[i].DeletedItemKeys).(*ItemKeyList)
	}
	RemoveSliceElements(&f.ItemKeys, &f.DeletedItemKeys)

	return nil
}

func (f *FeedItem) Resolve(siblingsCount int) error {
	// First get the siblings!
	siblingsI, err := f.Siblings(&FeedItem{})
	if err != nil {
		return err
	}
	siblings := siblingsI.([]FeedItem)

	for i := 0; i < siblingsCount; i++ {
		// Feed items are simple.  What ever claims is the latest update wins.  This should come from
		// the feed when possible.  Otherwise it is generated by the system, but should still be ok.
		if siblings[i].PubDate.After(f.PubDate) {
			f.Title = siblings[i].Title
			f.Author = siblings[i].Author
			f.Content = siblings[i].Content
			f.Url = siblings[i].Url
			f.PubDate = siblings[i].PubDate
		}
	}

	///@todo: I need to merge indexes here.  Currently I can't though due to library restrictions.

	return nil
}<commit_msg>Ensure that there is always data in the resolved models.<commit_after>/*
 * Copyright (C) 2013 <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package main

import (
	"bytes"
	"encoding/binary"

	"encoding/base64"

	"net/url"
	"time"

	riak "github.com/tpjg/goriakpbc"
)

type Feed struct {
	Url url.URL `riak:"url"`

	Title     string    `riak:"title"`
	LastCheck time.Time `riak:"last_check"`

	ItemKeys        ItemKeyList `riak:"item_keys"`
	DeletedItemKeys ItemKeyList `riak:"deleted_items"`

	NextCheck time.Time `riak:"next_check"`

	riak.Model `riak:"feeds"`
}

type FeedItem struct {
	Title   string `riak:"title"`
	Author  string `riak:"author"`
	Content string `riak:"content"`

	Url url.URL `riak:"url"`

	PubDate time.Time `riak:"publication_date"`

	riak.Model `riak:"items"`
}

type ItemKey []byte

func NewItemKey(id uint64, rawId []byte) ItemKey {
	buf := &bytes.Buffer{}

	binary.Write(buf, binary.BigEndian, id)
	buf.WriteString("-")
	buf.Write(rawId)

	return buf.Bytes()
}

func (l ItemKey) Less(r Comparable) bool {
	return bytes.Compare(l, r.(ItemKey)) == -1
}
func (l *ItemKey) UnmarshalJSON(input []byte) error {
	input = input[1 : len(input)-1]

	*l = make(ItemKey, base64.StdEncoding.DecodedLen(len(input)))
	n, err := base64.StdEncoding.Decode(*l, input)

	if err != nil {
		return err
	} else {
		*l = (*l)[0:n]
		return nil
	}
}

type ItemKeyList []ItemKey

func (list *ItemKeyList) Append(key Comparable) {
	*list = append(*list, key.(ItemKey))
}

func (list ItemKeyList) Get(index int) Comparable {
	return list[index]
}

func (list *ItemKeyList) RemoveAt(index int) {
	*list = append((*list)[:index], (*list)[index+1:]...)
}

func (list ItemKeyList) Len() int {
	return len(list)
}

func (ItemKeyList) Make() ComparableArray {
	return &ItemKeyList{}
}

func (f *Feed) Resolve(siblingsCount int) error {
	// First get the siblings!
	siblingsI, err := f.Siblings(&Feed{})
	if err != nil {
		return err
	}
	siblings := siblingsI.([]Feed)

	// Set the Url, as it is constant
	f.Url = siblings[0].Url

	for i := 0; i < siblingsCount; i++ {
		// Resolve regular feed details.  Basically, take the latest version!
		// If this is the first object, it will have a zero time of year 1st.  If a feed is claiming
		// be older then that, well it just won't work.
		if i == 0 || siblings[i].LastCheck.After(f.LastCheck) {
			f.Title = siblings[i].Title
			f.LastCheck = siblings[i].LastCheck
			f.NextCheck = siblings[i].NextCheck
		}

		// for the item lists, merge and de-dup using insert slice sort!
		f.ItemKeys = *InsertSliceSort(&f.ItemKeys, &siblings[i].ItemKeys).(*ItemKeyList)
		f.DeletedItemKeys = *InsertSliceSort(&f.DeletedItemKeys, &siblings[i].DeletedItemKeys).(*ItemKeyList)
	}
	RemoveSliceElements(&f.ItemKeys, &f.DeletedItemKeys)

	return nil
}

func (f *FeedItem) Resolve(siblingsCount int) error {
	// First get the siblings!
	siblingsI, err := f.Siblings(&FeedItem{})
	if err != nil {
		return err
	}
	siblings := siblingsI.([]FeedItem)

	for i := 0; i < siblingsCount; i++ {
		// Feed items are simple.  What ever claims is the latest update wins.  This should come from
		// the feed when possible.  Otherwise it is generated by the system, but should still be ok.
		if i == 0 || siblings[i].PubDate.After(f.PubDate) {
			f.Title = siblings[i].Title
			f.Author = siblings[i].Author
			f.Content = siblings[i].Content
			f.Url = siblings[i].Url
			f.PubDate = siblings[i].PubDate
		}
	}

	///@todo: I need to merge indexes here.  Currently I can't though due to library restrictions.

	return nil
}
{'fraction_non_alphanumeric': 0.13043478260869565, 'fraction_numerical': 0.00554016620498615, 'mean_word_length': 3.8646748681898067, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2058689', 'n_tokens_mistral': 2953, 'n_tokens_neox': 2814, 'n_words': 901}