<commit_before>package main

import (
	"database/sql"
	"html/template"
	"log"
	"net/http"
	"os"
	"sort"
	"strconv"
	"time"

	_ "github.com/yansal/pollbc/Godeps/_workspace/src/github.com/lib/pq"
)

var paris *time.Location

func init() {
	var err error
	paris, err = time.LoadLocation("Europe/Paris")
	if err != nil {
		log.Fatal(err)
	}
}

type Place struct {
	ID             int
	City           string
	Department     string
	Arrondissement string
}

type Announce struct {
	ID    string
	Date  time.Time
	Price string
	Title string

	PlaceID int

	Fetched time.Time
}

type ByDate []Announce

func (d ByDate) Len() int           { return len(d) }
func (d ByDate) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
func (d ByDate) Less(i, j int) bool { return d[i].Date.After(d[j].Date) }

var db *sql.DB

func init() {
	var err error
	db, err = sql.Open("postgres", os.Getenv("DATABASE_URL"))
	if err != nil {
		log.Fatal(err)
	}
	err = db.Ping()
	if err != nil {
		log.Fatal(err)
	}

	err = createTableAnnounces(db)
	if err != nil {
		log.Print(err)
	}

	err = createTablePlaces(db)
	if err != nil {
		log.Print(err)
	}

}

func poll() {
	for {
		doc, err := fetch()
		if err != nil {
			log.Print(err)
			time.Sleep(time.Minute)
			continue
		}
		nodes := queryAnnounces(doc)

		count := 0
		for _, n := range nodes {
			place, err := queryPlace(n)
			if err != nil {
				log.Print(err)
				continue
			}

			ok, err := hasPlace(db, place)
			if err != nil {
				log.Print(err)
			} else if !ok {
				err := insertPlace(db, place)
				if err != nil {
					log.Print(err)
				}
			}
			placeID, err := selectIDFromPlaces(db, place)
			if err != nil {
				log.Print(err)
			}

			id := queryID(n)
			if id == "" {
				continue
			}
			ok, err = hasAnnounce(db, id)
			if err != nil {
				log.Print(err)
			} else if !ok {
				count++
				ann := Announce{ID: id, Fetched: time.Now()}
				ann.Date = queryDate(n)
				ann.PlaceID = placeID
				ann.Price = queryPrice(n)
				ann.Title = queryTitle(n)
				err := insertAnnounce(db, ann)
				if err != nil {
					log.Print(err)
				}
			}
		}

		if count > 0 {
			// TODO: Notify by email?
			log.Printf("Number of new announces fetched:\t%d\n", count)
		}
		time.Sleep(5 * time.Second)
	}
}

func serveHTTP(w http.ResponseWriter, r *http.Request) {
	var ann []Announce
	var err error
	q := map[string][]string(r.URL.Query())
	placeIDs, ok := q["placeID"]
	if !ok {
		ann, err = selectAnnounces(db)
		if err != nil {
			log.Print(err)
		}
	} else {
		for _, placeID := range placeIDs {
			placeID, err := strconv.Atoi(placeID)
			if err != nil {
				log.Print(err)
			}
			newAnn, err := selectAnnouncesWherePlaceID(db, placeID)
			if err != nil {
				log.Print(err)
			}
			ann = append(ann, newAnn...)
		}
	}

	places := make(map[int]Place)
	for _, a := range ann {
		_, ok := places[a.PlaceID]
		if ok {
			continue
		}
		place, err := selectPlace(db, a.PlaceID)
		if err != nil {
			log.Print(err)
			continue
		}
		places[a.PlaceID] = place
	}
	sort.Sort(ByDate(ann))
	ann = ann[:35]

	data := struct {
		Announces []Announce
		Places    map[int]Place
	}{ann, places}
	t := template.Must(template.ParseFiles("template.html"))
	err = t.Execute(w, data)
	if err != nil {
		log.Print(err)
	}
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("$PORT must be set")
	}
	log.Printf("Listening on port %v", port)

	go poll()
	log.Fatal(http.ListenAndServe(":"+port, http.HandlerFunc(serveHTTP)))
}<commit_msg>Fix bug introduced in #15<commit_after>package main

import (
	"database/sql"
	"html/template"
	"log"
	"net/http"
	"os"
	"sort"
	"strconv"
	"time"

	_ "github.com/yansal/pollbc/Godeps/_workspace/src/github.com/lib/pq"
)

var paris *time.Location

func init() {
	var err error
	paris, err = time.LoadLocation("Europe/Paris")
	if err != nil {
		log.Fatal(err)
	}
}

type Place struct {
	ID             int
	City           string
	Department     string
	Arrondissement string
}

type Announce struct {
	ID    string
	Date  time.Time
	Price string
	Title string

	PlaceID int

	Fetched time.Time
}

type ByDate []Announce

func (d ByDate) Len() int           { return len(d) }
func (d ByDate) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
func (d ByDate) Less(i, j int) bool { return d[i].Date.After(d[j].Date) }

var db *sql.DB

func init() {
	var err error
	db, err = sql.Open("postgres", os.Getenv("DATABASE_URL"))
	if err != nil {
		log.Fatal(err)
	}
	err = db.Ping()
	if err != nil {
		log.Fatal(err)
	}

	err = createTableAnnounces(db)
	if err != nil {
		log.Print(err)
	}

	err = createTablePlaces(db)
	if err != nil {
		log.Print(err)
	}

}

func poll() {
	for {
		doc, err := fetch()
		if err != nil {
			log.Print(err)
			time.Sleep(time.Minute)
			continue
		}
		nodes := queryAnnounces(doc)

		count := 0
		for _, n := range nodes {
			place, err := queryPlace(n)
			if err != nil {
				log.Print(err)
				continue
			}

			ok, err := hasPlace(db, place)
			if err != nil {
				log.Print(err)
			} else if !ok {
				err := insertPlace(db, place)
				if err != nil {
					log.Print(err)
				}
			}
			placeID, err := selectIDFromPlaces(db, place)
			if err != nil {
				log.Print(err)
			}

			id := queryID(n)
			if id == "" {
				continue
			}
			ok, err = hasAnnounce(db, id)
			if err != nil {
				log.Print(err)
			} else if !ok {
				count++
				ann := Announce{ID: id, Fetched: time.Now()}
				ann.Date = queryDate(n)
				ann.PlaceID = placeID
				ann.Price = queryPrice(n)
				ann.Title = queryTitle(n)
				err := insertAnnounce(db, ann)
				if err != nil {
					log.Print(err)
				}
			}
		}

		if count > 0 {
			// TODO: Notify by email?
			log.Printf("Number of new announces fetched:\t%d\n", count)
		}
		time.Sleep(5 * time.Second)
	}
}

func serveHTTP(w http.ResponseWriter, r *http.Request) {
	var ann []Announce
	var err error
	q := map[string][]string(r.URL.Query())
	placeIDs, ok := q["placeID"]
	if !ok {
		ann, err = selectAnnounces(db)
		if err != nil {
			log.Print(err)
		}
	} else {
		for _, placeID := range placeIDs {
			placeID, err := strconv.Atoi(placeID)
			if err != nil {
				log.Print(err)
			}
			newAnn, err := selectAnnouncesWherePlaceID(db, placeID)
			if err != nil {
				log.Print(err)
			}
			ann = append(ann, newAnn...)
		}
	}

	places := make(map[int]Place)
	for _, a := range ann {
		_, ok := places[a.PlaceID]
		if ok {
			continue
		}
		place, err := selectPlace(db, a.PlaceID)
		if err != nil {
			log.Print(err)
			continue
		}
		places[a.PlaceID] = place
	}
	sort.Sort(ByDate(ann))
	if len(ann) > 35 {
		ann = ann[:35]
	}

	data := struct {
		Announces []Announce
		Places    map[int]Place
	}{ann, places}
	t := template.Must(template.ParseFiles("template.html"))
	err = t.Execute(w, data)
	if err != nil {
		log.Print(err)
	}
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("$PORT must be set")
	}
	log.Printf("Listening on port %v", port)

	go poll()
	log.Fatal(http.ListenAndServe(":"+port, http.HandlerFunc(serveHTTP)))
}
{'fraction_non_alphanumeric': 0.15673352435530086, 'fraction_numerical': 0.0020057306590257878, 'mean_word_length': 2.6742105263157896, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1279355', 'n_tokens_mistral': 3163, 'n_tokens_neox': 2767, 'n_words': 669}