<commit_before>package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/codegangsta/martini"
	"github.com/martini-contrib/render"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	s "strings"
	"time"
)

var data = make(map[string]interface{})

const filename = "db.txt"

type Proj struct {
	Title       string
	Description string
}

func ScrapProj() {
	fmt.Println("scraping ..")
	doc, err := goquery.NewDocument("https://github.com/karan/Projects/blob/master/README.md")
	if err != nil {
		log.Fatal(err)
	}

	if !isFileExist(filename) {
		// if the file doesn't exist, create a new one
		w, err := os.Create(filename)
		if err != nil {
			panic(err)
		}
		defer w.Close()

		// write empty one first
		err = ioutil.WriteFile(filename, []byte(""), 0644)

		doc.Find(".markdown-body p").Slice(7, 95).Each(func(i int, s *goquery.Selection) {
			txt := s.Text() + "\n"
			f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0600)
			if err != nil {
				panic(err)
			}

			defer f.Close()

			if _, err = f.WriteString(txt); err != nil {
				panic(err)
			}
		})
	}
}

func ReadProjFromFile() []string {
	fmt.Println("reading from file ..")
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	projs := string(b)
	return s.Split(projs, "\n")
}

func random(min, max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max-min) + min
}

func main() {

	m := martini.Classic()
	m.Use(render.Renderer(render.Options{
		Layout: "layout",
	}))

	m.Get("/suggest", func(ren render.Render) {
		// i := random(7, 95)
		// ren.HTML(200, "index", ScrapProj(i))
	})

	m.Get("/", func(ren render.Render) {
		ren.HTML(200, "index", nil)
	})

	// ScrapProj()
	fmt.Println("total : ", len(ReadProjFromFile()))
}

// Check if file already exists or not
func isFileExist(filename string) bool {
	if _, err := os.Stat(filename); err == nil {
		return true
	}
	return false
}<commit_msg>almost done!<commit_after>package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/codegangsta/martini"
	"github.com/martini-contrib/render"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	s "strings"
	"time"
)

var data = make(map[string]interface{})

const filename = "db.txt"

type Proj struct {
	Title       string
	Description string
}

func ScrapProj() {
	fmt.Println("scraping ..")
	doc, err := goquery.NewDocument("https://github.com/karan/Projects/blob/master/README.md")
	if err != nil {
		log.Fatal(err)
	}

	if !isFileExist(filename) {
		// if the file doesn't exist, create a new one
		w, err := os.Create(filename)
		if err != nil {
			panic(err)
		}
		defer w.Close()

		// write empty one first
		err = ioutil.WriteFile(filename, []byte(""), 0644)

		doc.Find(".markdown-body p").Slice(7, 95).Each(func(i int, s *goquery.Selection) {
			txt := s.Text() + "\n"
			f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0600)
			if err != nil {
				panic(err)
			}

			defer f.Close()

			if _, err = f.WriteString(txt); err != nil {
				panic(err)
			}
		})
	}
}

func ReadProjFromFile() []string {
	fmt.Println("reading from file ..")
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return s.Split(string(b), "\n")
}

func GetRandomProj(projs []string) map[string]interface{} {
	i := random(0, len(projs))
	project := projs[i]

	data["title"] = s.Split(project, " - ")[0]
	data["desc"] = s.Split(project, " - ")[1]
	return data
}

func random(min, max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max-min) + min
}

func main() {

	m := martini.Classic()
	m.Use(render.Renderer(render.Options{
		Layout: "layout",
	}))

	m.Get("/suggest", func(ren render.Render) {
		ren.HTML(200, "index", GetRandomProj(ReadProjFromFile()))
	})

	m.Get("/", func(ren render.Render) {
		ren.HTML(200, "index", nil)
	})

	m.Run()
}

// Check if file already exists or not
func isFileExist(filename string) bool {
	if _, err := os.Stat(filename); err == nil {
		return true
	}
	return false
}
{'fraction_non_alphanumeric': 0.1823455233291299, 'fraction_numerical': 0.01008827238335435, 'mean_word_length': 3.4561797752808987, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4161251', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1565, 'n_words': 337}