<commit_before>package db

// https://godoc.org/cloud.google.com/go/datastore

import(
	"fmt"
	"golang.org/x/net/context"
	"cloud.google.com/go/datastore"
)

// CloudDSProvider implements the DatastoreProvider interface using the cloud datastore API,
// for use outside of appengine environments.
type CloudDSProvider struct {
	Project string
}

func (p CloudDSProvider)flattenQuery(in *Query) *datastore.Query {
	out := datastore.NewQuery(in.Kind)
	if in.AncestorKeyer != nil { out = out.Ancestor(in.AncestorKeyer.(*datastore.Key)) }
	for _,filter := range in.Filters {
		out = out.Filter(filter.Field, filter.Value)
	}
	if in.OrderStr != "" { out = out.Order(in.OrderStr) }
	if in.KeysOnlyVal    { out = out.KeysOnly() }
	return out
}

func  (p CloudDSProvider)unpackKeyer(in Keyer) *datastore.Key {
	if in == nil { return nil }
	return in.(*datastore.Key)
}
func (p CloudDSProvider)unpackKeyers(in []Keyer) []*datastore.Key {
	out := []*datastore.Key{}
	for _,keyer := range in {
		out = append(out, p.unpackKeyer(keyer))
	}
	return out
}

func (p CloudDSProvider)GetAll(ctx context.Context, q *Query, dst interface{}) ([]Keyer, error) {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return nil, fmt.Errorf("GetAll{cloud}: %v\nQuery: %s", err, q) }

	dsQuery := p.flattenQuery(q)

	keys,err := dsClient.GetAll(ctx, dsQuery, dst)
	if err != nil {
		return nil, fmt.Errorf("GetAll{cloud}: %v", err)
	}

	keyers := []Keyer{}
	for _,k := range keys {
		keyers = append(keyers, Keyer(k))
	}
	return keyers,nil
}


func (p CloudDSProvider)Get(ctx context.Context, keyer Keyer, dst interface{}) error {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return err }
	return dsClient.Get(ctx, keyer.(*datastore.Key), dst)
}

func (p CloudDSProvider)Put(ctx context.Context, keyer Keyer, src interface{}) (Keyer, error) {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return nil,err }

	key,error := dsClient.Put(ctx, keyer.(*datastore.Key), src)
	return Keyer(key), error
}	
func (p CloudDSProvider)Delete(ctx context.Context, keyer Keyer) error {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return err }
	return dsClient.Delete(ctx, keyer.(*datastore.Key))
}	
func (p CloudDSProvider)DeleteMulti(ctx context.Context, keyers []Keyer) error {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return err }
	return dsClient.DeleteMulti(ctx, p.unpackKeyers(keyers))
}	


func (p CloudDSProvider)DecodeKey(encoded string) (Keyer, error) {
	key, err := datastore.DecodeKey(encoded)
	return Keyer(key), err
}


func (p CloudDSProvider)NewIncompleteKey(ctx context.Context, kind, name string, root Keyer) Keyer {
	key := datastore.IncompleteKey(kind, p.unpackKeyer(root))
	return Keyer(key)
}
func (p CloudDSProvider)NewNameKey(ctx context.Context, kind, name string, root Keyer) Keyer {
	key := datastore.NameKey(kind, name, p.unpackKeyer(root))
	return Keyer(key)
}
func (p CloudDSProvider)NewIDKey(ctx context.Context, kind string, id int64, root Keyer) Keyer {
	key := datastore.IDKey(kind, id, p.unpackKeyer(root))
	return Keyer(key)
}
<commit_msg>Make cloud provider useable again<commit_after>package db

// https://godoc.org/cloud.google.com/go/datastore

import(
	"fmt"
	"golang.org/x/net/context"
	"cloud.google.com/go/datastore"
)

// CloudDSProvider implements the DatastoreProvider interface using the cloud datastore API,
// for use outside of appengine environments.
type CloudDSProvider struct {
	Project string
}

func (p CloudDSProvider)flattenQuery(in *Query) *datastore.Query {
	out := datastore.NewQuery(in.Kind)
	if in.AncestorKeyer != nil { out = out.Ancestor(in.AncestorKeyer.(*datastore.Key)) }
	for _,filter := range in.Filters {
		out = out.Filter(filter.Field, filter.Value)
	}
	if in.OrderStr != "" { out = out.Order(in.OrderStr) }
	if in.KeysOnlyVal    { out = out.KeysOnly() }
	return out
}

func  (p CloudDSProvider)unpackKeyer(in Keyer) *datastore.Key {
	if in == nil { return nil }
	return in.(*datastore.Key)
}
func (p CloudDSProvider)unpackKeyers(in []Keyer) []*datastore.Key {
	out := []*datastore.Key{}
	for _,keyer := range in {
		out = append(out, p.unpackKeyer(keyer))
	}
	return out
}

func (p CloudDSProvider)GetAll(ctx context.Context, q *Query, dst interface{}) ([]Keyer, error) {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return nil, fmt.Errorf("GetAll{cloud}: %v\nQuery: %s", err, q) }

	dsQuery := p.flattenQuery(q)

	keys,err := dsClient.GetAll(ctx, dsQuery, dst)
	if err != nil {
		return nil, fmt.Errorf("GetAll{cloud}: %v", err)
	}

	keyers := []Keyer{}
	for _,k := range keys {
		keyers = append(keyers, Keyer(k))
	}
	return keyers,nil
}


func (p CloudDSProvider)Get(ctx context.Context, keyer Keyer, dst interface{}) error {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return err }
	return dsClient.Get(ctx, keyer.(*datastore.Key), dst)
}

func (p CloudDSProvider)Put(ctx context.Context, keyer Keyer, src interface{}) (Keyer, error) {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return nil,err }

	key,error := dsClient.Put(ctx, keyer.(*datastore.Key), src)
	return Keyer(key), error
}	
func (p CloudDSProvider)Delete(ctx context.Context, keyer Keyer) error {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return err }
	return dsClient.Delete(ctx, keyer.(*datastore.Key))
}	
func (p CloudDSProvider)DeleteMulti(ctx context.Context, keyers []Keyer) error {
	dsClient, err := datastore.NewClient(ctx, p.Project)
	if err != nil { return err }
	return dsClient.DeleteMulti(ctx, p.unpackKeyers(keyers))
}	


func (p CloudDSProvider)DecodeKey(encoded string) (Keyer, error) {
	key, err := datastore.DecodeKey(encoded)
	return Keyer(key), err
}


func (p CloudDSProvider)NewIncompleteKey(ctx context.Context, kind string, root Keyer) Keyer {
	key := datastore.IncompleteKey(kind, p.unpackKeyer(root))
	return Keyer(key)
}
func (p CloudDSProvider)NewNameKey(ctx context.Context, kind, name string, root Keyer) Keyer {
	key := datastore.NameKey(kind, name, p.unpackKeyer(root))
	return Keyer(key)
}
func (p CloudDSProvider)NewIDKey(ctx context.Context, kind string, id int64, root Keyer) Keyer {
	key := datastore.IDKey(kind, id, p.unpackKeyer(root))
	return Keyer(key)
}

{'fraction_non_alphanumeric': 0.14382696558257024, 'fraction_numerical': 0.0006315124723713293, 'mean_word_length': 4.876623376623376, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5480957', 'n_tokens_mistral': 2350, 'n_tokens_neox': 2323, 'n_words': 563}