<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package abelana

import (
	"strconv"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"
)

var server string

var (
	pool          *Pool
	redisPassword = ""
)

// Note - I looked at adapting both Gary Burd's pool system and Vites pools to AppEngine, but ran
// out of time as there were too many dependencies.

func redisInit() {
	server = redisExt
	pool = newPool(server, redisPassword)
}

func newPool(server, password string) *Pool {
	return &Pool{
		MaxIdle:     3,
		IdleTimeout: 115 * time.Second,
		Dial: func(cx appengine.Context) (Conn, error) {
			c, err := Dial(cx, "tcp", server)
			if err != nil {
				return nil, err
			}
			// if _, err := c.Do("AUTH", password); err != nil {
			// 	c.Close()
			// 	return nil, err
			// }
			return c, err
		},
		TestOnBorrow: func(c Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}

// iNowFollow is Called when we need to add a folower
func iNowFollow(cx appengine.Context, userID, followerID string) {

}

// addPhoto is called to add a photo.
func addPhoto(cx appengine.Context, superID string) {

	s := strings.Split(superID, ".")
	if len(s) != 3 {
		cx.Errorf("AddPhoto -- bad superID %v", superID)
		return
	}
	u, err := findUser(cx, s[0])
	if err != nil {
		cx.Errorf("AddPhoto unable to find user %v %v", superID, err)
		return
	}

	p := &Photo{s[1], time.Now().UTC().Unix()}

	conn := pool.Get(cx)
	defer conn.Close()

	photoID := s[0] + "." + s[1]
	set, err := Int(conn.Do("HSETNX", "IM:"+photoID, "date", p.Date)) // Set Date
	if (err != nil && err != ErrNil) || set == 0 {
		cx.Errorf("AddPhoto D duplicate %v %v", err, set)
		return // block duplicate requests
	}

	// Consider that at some point these should be done in batches of 100 or so. TODO

	// Add to each follower's list
	for _, personID := range u.Persons {
		conn.Send("LPUSH", "TL:"+personID, photoID)
	}
	conn.Flush()

	// Check the result and adjust list if nescessary.
	for _, personID := range u.Persons {
		v, err := Int(conn.Receive())
		if err != nil && err != ErrNil {
			cx.Errorf("AddPhoto for %v %v", "TL:"+personID, err)
		} else {
			if v > 2000 {
				_, err := conn.Do("RPOP", "TL:"+personID)
				if err != nil {
					cx.Errorf("AddPhoto RPOP %v", err)
				}
			}
		}
	}
	k1 := datastore.NewKey(cx, "User", s[0], 0, nil)
	k2 := datastore.NewKey(cx, "Photo", s[1], 0, k1)
	_, err = datastore.Put(cx, k2, p)
	if err != nil {
		cx.Errorf("AddPhoto datastore %v", err)
	}
}

// getTimeline
func getTimeline(cx appengine.Context, userID, lastid string) ([]TLEntry, error) {
	var item string
	timeline := []TLEntry{}

	conn := pool.Get(cx)
	defer conn.Close()

	list, err := Strings(conn.Do("LRANGE", "TL:"+userID, 0, -1))
	if err != nil && err != ErrNil {
		cx.Errorf("GetTimeLine %v", err)
	}
	ix := 0
	if lastid != "0" {
		for ix, item = range list {
			if item == lastid {
				break
			}
		}
	}
	timeline = make([]TLEntry, 0, timelineBatchSize)
	for i := 0; i < timelineBatchSize && i+ix < len(list); i++ {
		photoID := list[ix+i]

		v, err := Strings(conn.Do("HMGET", "IM:"+photoID, "date", userID, "flag"))
		if err != nil && err != ErrNil {
			cx.Errorf("GetTimeLine HMGET %v", err)
		}
		if v[2] != "" {
			flags, err := strconv.Atoi(v[2])
			if err == nil && flags > 1 {
				continue // skip flag'd images
			}
		}
		likes, err := Int(conn.Do("HLEN", "IM:"+photoID))
		if err != nil && err != ErrNil {<commit_msg>my version of Campoy's changes<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package abelana

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"
)

var server string

var (
	pool          *Pool
	redisPassword = ""
)

// Note - I looked at adapting both Gary Burd's pool system and Vites pools to AppEngine, but ran
// out of time as there were too many dependencies.

func redisInit() {
	server = redisExt
	pool = newPool(server, redisPassword)
}

func newPool(server, password string) *Pool {
	return &Pool{
		MaxIdle:     3,
		IdleTimeout: 115 * time.Second,
		Dial: func(cx appengine.Context) (Conn, error) {
			c, err := Dial(cx, "tcp", server)
			if err != nil {
				return nil, err
			}
			// if _, err := c.Do("AUTH", password); err != nil {
			// 	c.Close()
			// 	return nil, err
			// }
			return c, err
		},
		TestOnBorrow: func(c Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}

// iNowFollow is Called when we need to add a folower
func iNowFollow(cx appengine.Context, userID, followerID string) {

}

// addPhoto is called to add a photo. This is allways called from a Delay, so returns are pointless
func addPhoto(cx appengine.Context, filename string) {
	err := func() error {
		s := strings.Split(filename, ".")
		if len(s) == 2 {
			return nil // User identiy photo
		}
		if len(s) != 3 {
			return fmt.Errorf("Bad filename format %v", filename)
		}
		// s[0] = userid, s[1] = random photo, s[3] = file extension
		userID, photoID := s[0], s[0]+"."+s[1]
		u, err := findUser(cx, userID)
		if err != nil {
			return fmt.Errorf("unable to find user %v %v", userID, err)
		}

		p := &Photo{photoID, time.Now().UTC().Unix()}

		k1 := datastore.NewKey(cx, "User", s[0], 0, nil)
		k2 := datastore.NewKey(cx, "Photo", s[1], 0, k1)
		_, err = datastore.Put(cx, k2, p)
		if err != nil {
			return fmt.Errorf("datastore %v", err)
		}

		conn := pool.Get(cx)
		defer conn.Close()

		set, err := Int(conn.Do("HSETNX", "IM:"+photoID, "date", p.Date)) // Set Date
		if (err != nil && err != ErrNil) || set == 0 {
			return fmt.Errorf("duplicate %v %v", err, set)
		}

		// TODO: Consider if these should be done in batches of 100 or so.

		// Add to each follower's list
		for _, personID := range u.Persons {
			conn.Send("LPUSH", "TL:"+personID, photoID)
		}
		conn.Flush()

		// Check the result and adjust list if nescessary.
		for _, personID := range u.Persons {
			v, err := Int(conn.Receive())
			if err != nil && err != ErrNil {
				cx.Errorf("Addphoto: TL:%v %v", personID, err)
			} else {
				if v > 2000 {
					_, err := conn.Do("RPOP", "TL:"+personID)
					if err != nil {
						cx.Errorf("AddPhoto: RPOP TL:%v %v", personID, err)
					}
				}
			}
		}
		return nil
	}()
	if err != nil {
		cx.Errorf("addPhoto: %v", err)
	}
}

// getTimeline returns the user's Timeline, you could insert additional things here as well.
func getTimeline(cx appengine.Context, userID, lastid string) ([]TLEntry, error) {
	timeline := []TLEntry{}

	conn := pool.Get(cx)
	defer conn.Close()

	list, err := Strings(conn.Do("LRANGE", "TL:"+userID, 0, -1))
	if err != nil && err != ErrNil {
		cx.Errorf("GetTimeLine %v", err)
	}
	ix := 0

	if lastid != "0" {
		for i, item := range list {
			if item == lastid {
				ix = i
				break
			}
		}
	}
	timeline = make([]TLEntry, 0, timelineBatchSize)
	for i := 0; i < timelineBatchSize && i+ix < len(list); i++ {
		photoID := list[ix+i]

		v, err := Strings(conn.Do("HMGET", "IM:"+photoID, "date", userID, "flag"))
		if err != nil && err != ErrNil {
			cx.Errorf("GetTimeLine HMGET %v", err)
		}
		if v[2] != "" {
			flags, err := strconv.Atoi(v[2])
			if err == nil && flags > 1 {
				continue // skip flag'd images
			}
		}
		likes, err := Int(conn.Do("HLEN", "IM:"+photoID))
		if err != nil && err != ErrNil {
{'fraction_non_alphanumeric': 0.15881513204853676, 'fraction_numerical': 0.01034975017844397, 'mean_word_length': 3.0263409961685825, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1637593', 'n_tokens_mistral': 3357, 'n_tokens_neox': 3013, 'n_words': 983}