<commit_before>package pigae

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

//
// This file is about full text search of idioms and implementations.
//

// SearchResultsFacade is the facade for the Search Results page.
type SearchResultsFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Q           string
	Results     []*Idiom
}

// User types terms, but it's important to recognize when a term is a language name.
func separateLangKeywords(terms []string) (words, langs []string) {
	words = make([]string, 0, len(terms))
	for _, term := range terms {
		lang := normLang(term)
		if lang == "" {
			words = append(words, term)
		} else {
			langs = append(langs, lang)
		}
	}
	return words, langs
}

// This is a "word by word" search, not a rdbms "like" filter
func search(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	q := vars["q"]
	//q := url.QueryUnescape(q)  Not needed, so it seems.

	// This is a premature hack. TODO find a graceful way to handle "c++",
	// and stop confounding spaces and pluses.
	q = strings.Replace(q, "C++", "Cpp", -1)
	q = strings.Replace(q, "c++", "cpp", -1)
	q = strings.Replace(q, "C#", "Csharp", -1)
	q = strings.Replace(q, "c#", "csharp", -1)

	terms := SplitForSearching(q, true)

	words, typedLangs := separateLangKeywords(terms)

	words = FilterStrings(words, func(chunk string) bool {
		// Small terms (1 or 2 chars) must be discarded,
		// because they weren't indexed in the first place.
		// (Words only, not language names.)
		return len(chunk) >= 3 || RegexpDigitsOnly.MatchString(chunk)<commit_msg>issues/67 Search with sharp sign.<commit_after>package pigae

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

//
// This file is about full text search of idioms and implementations.
//

// SearchResultsFacade is the facade for the Search Results page.
type SearchResultsFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Q           string
	Results     []*Idiom
}

// User types terms, but it's important to recognize when a term is a language name.
func separateLangKeywords(terms []string) (words, langs []string) {
	words = make([]string, 0, len(terms))
	for _, term := range terms {
		lang := normLang(term)
		if lang == "" {
			words = append(words, term)
		} else {
			langs = append(langs, lang)
		}
	}
	return words, langs
}

// This is a "word by word" search, not a rdbms "like" filter
func search(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	q := vars["q"]
	//q := url.QueryUnescape(q)  Not needed, so it seems.

	// This is a premature hack. TODO find a graceful way to handle "c++",
	// and stop confounding spaces and pluses.
	q = strings.Replace(q, "C++", "Cpp", -1)
	q = strings.Replace(q, "c++", "cpp", -1)
	q = strings.Replace(q, "C#", "Csharp", -1)
	q = strings.Replace(q, "c#", "csharp", -1)
	q = strings.Replace(q, "C♯", "Csharp", -1)
	q = strings.Replace(q, "c♯", "csharp", -1)

	terms := SplitForSearching(q, true)

	words, typedLangs := separateLangKeywords(terms)

	words = FilterStrings(words, func(chunk string) bool {
		// Small terms (1 or 2 chars) must be discarded,
		// because they weren't indexed in the first place.
		// (Words only, not language names.)
		return len(chunk) >= 3 || RegexpDigitsOnly.MatchString(chunk)
{'fraction_non_alphanumeric': 0.14505844845908608, 'fraction_numerical': 0.005313496280552604, 'mean_word_length': 3.6196319018404908, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1741996', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1309, 'n_words': 403}