<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/siesta/neo4j"
	"github.com/streadway/amqp"
	"io/ioutil"
	"koding/databases/mongo"
	oldNeo "koding/databases/neo4j"
	"koding/tools/amqputil"
	"koding/tools/config"
	"log"
	"net/http"
	"strconv"
	"strings"
)

type strToInf map[string]interface{}

var GRAPH_URL = config.Current.Neo4j.Write + ":" + strconv.Itoa(config.Current.Neo4j.Port)

func main() {
	amqpChannel := connectToRabbitMQ()

	coll := mongo.GetCollection("relationships")
	query := strToInf{
		"targetName": strToInf{"$nin": oldNeo.NotAllowedNames},
		"sourceName": strToInf{"$nin": oldNeo.NotAllowedNames},
	}
	iter := coll.Find(query).Sort("-timestamp").Iter()

	var result oldNeo.Relationship
	for iter.Next(&result) {
		if relationshipNeedsToBeSynced(result) {
			createRelationship(result, amqpChannel)
		}
	}

	log.Println("Neo4j is now synced with Mongodb.")
}

func connectToRabbitMQ() *amqp.Channel {
	conn := amqputil.CreateConnection("syncWorker")
	amqpChannel, err := conn.Channel()
	if err != nil {
		panic(err)
	}
	return amqpChannel
}

func createRelationship(rel oldNeo.Relationship, amqpChannel *amqp.Channel) {<commit_msg>sync: batch size of 1000 when querying mongo<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/siesta/neo4j"
	"github.com/streadway/amqp"
	"io/ioutil"
	"koding/databases/mongo"
	oldNeo "koding/databases/neo4j"
	"koding/tools/amqputil"
	"koding/tools/config"
	"log"
	"net/http"
	"strconv"
	"strings"
)

type strToInf map[string]interface{}

var GRAPH_URL = config.Current.Neo4j.Write + ":" + strconv.Itoa(config.Current.Neo4j.Port)

func main() {
	amqpChannel := connectToRabbitMQ()

	coll := mongo.GetCollection("relationships")
	query := strToInf{
		"targetName": strToInf{"$nin": oldNeo.NotAllowedNames},
		"sourceName": strToInf{"$nin": oldNeo.NotAllowedNames},
	}
	iter := coll.Find(query).Batch(1000).Sort("-timestamp").Iter()

	var result oldNeo.Relationship
	for iter.Next(&result) {
		if relationshipNeedsToBeSynced(result) {
			createRelationship(result, amqpChannel)
		}
	}

	log.Println("Neo4j is now synced with Mongodb.")
}

func connectToRabbitMQ() *amqp.Channel {
	conn := amqputil.CreateConnection("syncWorker")
	amqpChannel, err := conn.Channel()
	if err != nil {
		panic(err)
	}
	return amqpChannel
}

func createRelationship(rel oldNeo.Relationship, amqpChannel *amqp.Channel) {
{'fraction_non_alphanumeric': 0.14685022945348353, 'fraction_numerical': 0.007509386733416771, 'mean_word_length': 5.261096605744125, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6419027', 'n_tokens_mistral': 972, 'n_tokens_neox': 934, 'n_words': 155}