<commit_before>package server

import (
	"net/http"

	"encoding/json"
	"log"

	"html/template"
	"strings"

	"mozilla.org/crec/content"
	"mozilla.org/crec/ingester"
)

// Server to host public API for content consumption
type Server struct {
	Addr    string            // Address to start server e.g. ":8080"
	Path    string            // Path to bind handler function e.g. "/content"
	Indexer *ingester.Indexer // Aavailable content
}

// Start a server to provide an API for content consumption
func (s *Server) Start() {
	http.HandleFunc(s.Path, s.contentHandler)
	http.ListenAndServe(s.Addr, nil)
}

func (s *Server) contentHandler(w http.ResponseWriter, r *http.Request) {
	topic := r.URL.Query().Get("t")
	format := r.URL.Query().Get("f")
	query := r.URL.Query().Get("q")
	acceptHeader := r.Header.Get("Accept")

	c := s.fetchContent(topic, format, query)

	if strings.Contains(acceptHeader, "html") && !strings.EqualFold(format, "json") {
		s.respondWithHTML(w, c)
	} else if strings.Contains(acceptHeader, "json") || strings.EqualFold(format, "json") {
		s.respondWithJSON(w, c)
	} else {
		log.Println("No acceptable content format found.")
	}
}

func (s *Server) fetchContent(topic string, format string, query string) []*content.Content {
	var c []*content.Content
	if topic != "" {
		splits := strings.Split(topic, ",")
		splitMap := make(map[string]bool)
		for _, s := range splits {
			splitMap[strings.ToLower(s)] = true
		}

		c = content.Filter(s.Indexer.Content, func(c *content.Content) bool {
			for _, t := range c.Tags() {
				if _, ok := splitMap[strings.ToLower(t)]; ok {
					return true
				}
			}
			return false
		})
	} else if query != "" {
		c = s.queryIndexForContent(query)
	} else {
		c = s.Indexer.Content
	}
	return c
}

func (s *Server) queryIndexForContent(q string) []*content.Content {
	c, err := s.Indexer.Query(q)
	if err != nil {
		log.Fatal("Failed to query index:", err)
	}
	return c
}

func (s *Server) respondWithHTML(w http.ResponseWriter, c []*content.Content) {
	t, err := template.ParseFiles("template/item.html")
	if err != nil {
		log.Fatal("Failed to parse template: ", err)
	}

	w.Header().Set("Content-Type", "text/html;charset=UTF-8")
	for _, r := range c {
		t.Execute(w, &content.RenderedContent{Title: r.Title(), Link: r.Link(), Summary: r.Summary()})
	}
}
<commit_msg>Fall back to full text index if not enough token-based hits<commit_after>package server

import (
	"net/http"

	"encoding/json"
	"log"

	"html/template"
	"strings"

	"mozilla.org/crec/content"
	"mozilla.org/crec/ingester"
)

// Server to host public API for content consumption
type Server struct {
	Addr    string            // Address to start server e.g. ":8080"
	Path    string            // Path to bind handler function e.g. "/content"
	Indexer *ingester.Indexer // Aavailable content
}

// Start a server to provide an API for content consumption
func (s *Server) Start() {
	http.HandleFunc(s.Path, s.contentHandler)
	http.ListenAndServe(s.Addr, nil)
}

const minPageSize = 5

func (s *Server) contentHandler(w http.ResponseWriter, r *http.Request) {
	topic := r.URL.Query().Get("t")
	format := r.URL.Query().Get("f")
	query := r.URL.Query().Get("q")
	acceptHeader := r.Header.Get("Accept")

	c := s.fetchContent(topic, format, query)

	if strings.Contains(acceptHeader, "html") && !strings.EqualFold(format, "json") {
		s.respondWithHTML(w, c)
	} else if strings.Contains(acceptHeader, "json") || strings.EqualFold(format, "json") {
		s.respondWithJSON(w, c)
	} else {
		log.Println("No acceptable content format found.")
	}
}

func (s *Server) fetchContent(topic string, format string, query string) []*content.Content {
	var c []*content.Content
	if topic != "" {
		splits := strings.Split(topic, ",")
		splitMap := make(map[string]bool)
		for _, s := range splits {
			splitMap[strings.ToLower(s)] = true
		}

		c = content.Filter(s.Indexer.Content, func(c *content.Content) bool {
			for _, t := range c.Tags() {
				if _, ok := splitMap[strings.ToLower(t)]; ok {
					return true
				}
			}
			return false
		})

		if len(c) < minPageSize {
			for _, topic := range splits {
				c = append(c, s.queryIndexForContent(topic)...)
			}
		}
	} else if query != "" {
		c = s.queryIndexForContent(query)
	} else {
		c = s.Indexer.Content
	}
	return c
}

func (s *Server) queryIndexForContent(q string) []*content.Content {
	c, err := s.Indexer.Query(q)
	if err != nil {
		log.Fatal("Failed to query index:", err)
	}
	return c
}

func (s *Server) respondWithHTML(w http.ResponseWriter, c []*content.Content) {
	t, err := template.ParseFiles("template/item.html")
	if err != nil {
		log.Fatal("Failed to parse template: ", err)
	}

	w.Header().Set("Content-Type", "text/html;charset=UTF-8")
	for _, r := range c {
		t.Execute(w, &content.RenderedContent{Title: r.Title(), Link: r.Link(), Summary: r.Summary()})
	}
}

{'fraction_non_alphanumeric': 0.16133690942851248, 'fraction_numerical': 0.0022694450175366207, 'mean_word_length': 3.6436781609195403, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2080444', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1685, 'n_words': 427}