<commit_before>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"github.com/cdale77/go_git/Godeps/_workspace/src/github.com/melvinmt/firebase"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

type StoredCommit struct {
	Date    string
	Login   string
	Avatar  string
	Message string
	Url     string
}

type CommitCommit struct {
	Message string
	Url     string
}

type CommitPayload struct {
	Size    int
	Commits []CommitCommit
}

type CommitActor struct {
	Login      string
	Avatar_url string
}

type Commit struct {
	Type       string
	Created_at string
	Actor      CommitActor
	Payload    CommitPayload
}

func main() {
	fullDate := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
	getData(fullDate)
}

func storeCommit(commit Commit) bool {
	fmt.Println("storing commit:")
	fmt.Println(commit)
	authToken := os.Getenv("FIREBASE_SECRET")

	url := os.Getenv("FIREBASE_URL")

	fireBase := firebase.NewReference(url).Auth(authToken)

	var storedCommit StoredCommit
	storedCommit.Date = commit.Created_at
	storedCommit.Login = commit.Actor.Login
	storedCommit.Avatar = commit.Actor.Avatar_url
	// This only stores the first commit message. Fix it so it stores one of these
	// for every message
	//storedCommit.Message = commit.Payload.Commits[0]
	storedCommit.Url = commit.Payload.Commits[0].Url

	err := fireBase.Push(storedCommit)
	if err != nil {
		fmt.Println("Firebase error")
		fmt.Println(err)
		return false
	} else {
		fmt.Println("Firebase success")
		return true
	}
}

// There must be a better way to do this. Probably sort cussWords alpha
// and use a lookup table.
func isDirty(message string) bool {

	result := false

	cussWords := []string{
		"fuck",
		"bitch",
		"stupid",
		"tits",
		"asshole",
		"cocksucker",
		"cunt",
		"hell",
		"douche",
		"testicle",
		"twat",
		"bastard",
		"sperm",
		"shit",
		"dildo",
		"wanker",
		"prick",
		"penis",
		"vagina",
		"whore"}

	messageWords := strings.Split(message, " ")

	for _, cussWord := range cussWords {
		for _, word := range messageWords {
			if word == cussWord {
				result = true
			}
		}

	}
	return result
}

func parseCommit(line string) {
	var commit Commit

	jsonErr := json.Unmarshal([]byte(line), &commit)
	if jsonErr != nil {
		fmt.Println("Could not parse json.")
		fmt.Println(jsonErr)
	}

	if commit.Type == "PushEvent" && commit.Payload.Size > 0 {
		// TO-DO this is only checking the first commit. Probably build
		// the storage struct here and pass it to the storeCommit()
		// function
		if isDirty(commit.Payload.Commits[0].Message) {
			storeCommit(commit)
		}
	}

}

func parseFile(fName string) {
	// https://groups.google.com/forum/#!topic/golang-nuts/GjIkryuCyAY
	// TODO: standardize use of file api
	// https://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
	fileOS, err := os.Open(fName)
	if err != nil {<commit_msg>mvp backend<commit_after>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"github.com/cdale77/go_git/Godeps/_workspace/src/github.com/melvinmt/firebase"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

type StoredCommit struct {
	Date    string
	Login   string
	Avatar  string
	Message string
	Url     string
}

type CommitCommit struct {
	Message string
	Url     string
}

type CommitPayload struct {
	Size    int
	Commits []CommitCommit
}

type CommitActor struct {
	Login      string
	Avatar_url string
}

type Commit struct {
	Type       string
	Created_at string
	Actor      CommitActor
	Payload    CommitPayload
}

func main() {
	fullDate := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
	getData(fullDate)
}

func storeCommit(commit Commit, commitMessage string, commitUrl string) bool {
	fmt.Println("storing commit:")
	fmt.Println(commit)
	authToken := os.Getenv("FIREBASE_SECRET")

	url := os.Getenv("FIREBASE_URL")

	fireBase := firebase.NewReference(url).Auth(authToken)

	var storedCommit StoredCommit
	storedCommit.Date = commit.Created_at
	storedCommit.Login = commit.Actor.Login
	storedCommit.Avatar = commit.Actor.Avatar_url
	storedCommit.Message = commitMessage
	storedCommit.Url = commitUrl

	err := fireBase.Push(storedCommit)
	if err != nil {
		fmt.Println("Firebase error")
		fmt.Println(err)
		return false
	} else {
		fmt.Println("Firebase success")
		return true
	}
}

// There must be a better way to do this. Probably sort cussWords alpha
// and use a lookup table.
func isDirty(message string) bool {

	result := false

	cussWords := []string{
		"fuck",
		"bitch",
		"stupid",
		"tits",
		"asshole",
		"cocksucker",
		"cunt",
		"hell",
		"douche",
		"testicle",
		"twat",
		"bastard",
		"sperm",
		"shit",
		"dildo",
		"wanker",
		"prick",
		"penis",
		"vagina",
		"whore"}

	messageWords := strings.Split(message, " ")

	for _, cussWord := range cussWords {
		for _, word := range messageWords {
			if word == cussWord {
				result = true
			}
		}
	}
	return result
}

func parseCommit(line string) {
	var commit Commit

	jsonErr := json.Unmarshal([]byte(line), &commit)
	if jsonErr != nil {
		fmt.Println("Could not parse json.")
		fmt.Println(jsonErr)
	}

	if commit.Type == "PushEvent" && commit.Payload.Size > 0 {
		// TO-DO this is only checking the first commit.
		message := commit.Payload.Commits[0].Message
		if isDirty(message) {
			url := commit.Payload.Commits[0].Url
			storeCommit(commit, message, url)
		}
	}

}

func parseFile(fName string) {
	// https://groups.google.com/forum/#!topic/golang-nuts/GjIkryuCyAY
	// TODO: standardize use of file api
	// https://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
	fileOS, err := os.Open(fName)
	if err != nil {
{'fraction_non_alphanumeric': 0.13054230701289524, 'fraction_numerical': 0.00830242006712595, 'mean_word_length': 3.5809061488673137, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6284812', 'n_tokens_mistral': 2216, 'n_tokens_neox': 2090, 'n_words': 484}