<commit_before>package main

import (
	"bytes"
	"fmt"
	"golang.org/x/net/html"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
)

func main() {
	MLSNumbers := getMLSNumbers()
	MLSURLs := getMLSDetails(MLSNumbers)
	fmt.Println(MLSURLs)
}

func getMLSNumbers() []string {
	MLSNums := []string{}

	searchURL := "http://www.gainesvillemls.com"
	searchPath := "/gan/idx/search.php"

	data := url.Values{}
	data.Set("LM_MST_prop_fmtYNNT", "1")
	data.Add("LM_MST_prop_cdYYNT", "1,9,10,11,12,13,14")
	data.Add("LM_MST_mls_noYYNT", "")
	// Minimum Price
	data.Add("LM_MST_list_prcYNNB", "")
	// Maximum Price
	data.Add("LM_MST_list_prcYNNE", "175000")
	data.Add("LM_MST_prop_cdYNNL[]", "9")
	// Minimum Square Footage
	data.Add("LM_MST_sqft_nYNNB", "")
	// Maximum Square Footage
	data.Add("LM_MST_sqft_nYNNE", "")
	// Minimum Year Built
	data.Add("LM_MST_yr_bltYNNB", "1981")
	// Maximum Year Built
	data.Add("LM_MST_yr_bltYNNE", "")
	// Minimum Bedrooms
	data.Add("LM_MST_bdrmsYNNB", "3")
	// Maximum Bedrooms
	data.Add("LM_MST_bdrmsYNNE", "")
	// Minimum Bathrooms
	data.Add("LM_MST_bathsYNNB", "2")
	// Maximum Bathrooms
	data.Add("LM_MST_bathsYNNE", "")
	data.Add("LM_MST_hbathYNNB", "")
	data.Add("LM_MST_hbathYNNE", "")
	// County
	data.Add("LM_MST_countyYNCL[]", "ALA")
	data.Add("LM_MST_str_noY1CS", "")
	data.Add("LM_MST_str_namY1VZ", "")
	data.Add("LM_MST_remarksY1VZ", "")
	data.Add("openHouseStartDt_B", "")
	data.Add("openHouseStartDt_E", "")
	data.Add("ve_info", "")
	data.Add("ve_rgns", "1")
	data.Add("LM_MST_LATXX6I", "")
	data.Add("poi", "")
	data.Add("count", "1")
	data.Add("key", "<KEY>")
	data.Add("isLink", "0")
	data.Add("custom", "")

	u, _ := url.ParseRequestURI(searchURL)
	u.Path = searchPath
	urlStr := fmt.Sprintf("%v", u)

	client := &http.Client{}
	request, requestErr := http.NewRequest("POST", urlStr, bytes.NewBufferString(data.Encode()))
	if requestErr != nil {
		log.Fatalf("Problem creating new httpRequest", "%s", requestErr)
	}
	request.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
	request.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))

	resp, responseError := client.Do(request)
	if responseError != nil {
		log.Fatalf("Problem creating new httpRequest", "%s", responseError)
	}

	responseBody := resp.Body
	defer responseBody.Close()

	parsedHTML := html.NewTokenizer(responseBody)

	MLSNumber := ""
	MLSFlag := false

	for {
		tt := parsedHTML.Next()
		switch {
		case tt == html.ErrorToken:
			return MLSNums
		case tt == html.StartTagToken:
			t := parsedHTML.Token()
			if t.String() == `<span class="mls">` {
				MLSFlag = true
			}
		case tt == html.TextToken:
			t := parsedHTML.Token().String()
			tLower := strings.ToLower(t)
			if strings.Contains(tLower, ", fl") {
				// We don't want homes not in Gainesville
				if !strings.Contains(tLower, "gainesville") {
					break
				}
			}
			if MLSFlag == true {
				MLSNumber = t
				MLSNums = append(MLSNums, MLSNumber)
				MLSFlag = false
			}
		}
	}
}

func removeEmpty(inputArray []string) []string {
	returnArray := []string{}
	for _, str := range inputArray {
		if str != "" {
			returnArray = append(returnArray, str)
		}
	}
	return returnArray<commit_msg>really removing houses not in gainesville<commit_after>package main

import (
	"bytes"
	"fmt"
	"golang.org/x/net/html"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
)

func main() {
	MLSNumbers := getMLSNumbers()
	MLSURLs := getMLSDetails(MLSNumbers)
	fmt.Println(MLSURLs)
}

func getMLSNumbers() []string {
	MLSNums := []string{}

	searchURL := "http://www.gainesvillemls.com"
	searchPath := "/gan/idx/search.php"

	data := url.Values{}
	data.Set("LM_MST_prop_fmtYNNT", "1")
	data.Add("LM_MST_prop_cdYYNT", "1,9,10,11,12,13,14")
	data.Add("LM_MST_mls_noYYNT", "")
	// Minimum Price
	data.Add("LM_MST_list_prcYNNB", "")
	// Maximum Price
	data.Add("LM_MST_list_prcYNNE", "175000")
	data.Add("LM_MST_prop_cdYNNL[]", "9")
	// Minimum Square Footage
	data.Add("LM_MST_sqft_nYNNB", "")
	// Maximum Square Footage
	data.Add("LM_MST_sqft_nYNNE", "")
	// Minimum Year Built
	data.Add("LM_MST_yr_bltYNNB", "1981")
	// Maximum Year Built
	data.Add("LM_MST_yr_bltYNNE", "")
	// Minimum Bedrooms
	data.Add("LM_MST_bdrmsYNNB", "3")
	// Maximum Bedrooms
	data.Add("LM_MST_bdrmsYNNE", "")
	// Minimum Bathrooms
	data.Add("LM_MST_bathsYNNB", "2")
	// Maximum Bathrooms
	data.Add("LM_MST_bathsYNNE", "")
	data.Add("LM_MST_hbathYNNB", "")
	data.Add("LM_MST_hbathYNNE", "")
	// County
	data.Add("LM_MST_countyYNCL[]", "ALA")
	data.Add("LM_MST_str_noY1CS", "")
	data.Add("LM_MST_str_namY1VZ", "")
	data.Add("LM_MST_remarksY1VZ", "")
	data.Add("openHouseStartDt_B", "")
	data.Add("openHouseStartDt_E", "")
	data.Add("ve_info", "")
	data.Add("ve_rgns", "1")
	data.Add("LM_MST_LATXX6I", "")
	data.Add("poi", "")
	data.Add("count", "1")
	data.Add("key", "<KEY>")
	data.Add("isLink", "0")
	data.Add("custom", "")

	u, _ := url.ParseRequestURI(searchURL)
	u.Path = searchPath
	urlStr := fmt.Sprintf("%v", u)

	client := &http.Client{}
	request, requestErr := http.NewRequest("POST", urlStr, bytes.NewBufferString(data.Encode()))
	if requestErr != nil {
		log.Fatalf("Problem creating new httpRequest", "%s", requestErr)
	}
	request.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
	request.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))

	resp, responseError := client.Do(request)
	if responseError != nil {
		log.Fatalf("Problem creating new httpRequest", "%s", responseError)
	}

	responseBody := resp.Body
	defer responseBody.Close()

	parsedHTML := html.NewTokenizer(responseBody)

	MLSNumber := ""
	MLSFlag := false

	for {
		tt := parsedHTML.Next()
		switch {
		case tt == html.ErrorToken:
			return MLSNums
		case tt == html.StartTagToken:
			t := parsedHTML.Token()
			if t.String() == `<span class="mls">` {
				MLSFlag = true
			}
		case tt == html.TextToken:
			t := parsedHTML.Token().String()
			tLower := strings.ToLower(t)
			if strings.Contains(tLower, ", fl") {
				// We don't want homes not in Gainesville
				if !strings.Contains(tLower, "gainesville") {
					// Remove the latest MLS number from the array
					MLSNums = MLSNums[:len(MLSNums)-1]
					break
				}
			}
			if MLSFlag == true {
				MLSNumber = t
				MLSNums = append(MLSNums, MLSNumber)
				MLSFlag = false
			}
		}
	}
}

func removeEmpty(inputArray []string) []string {
	returnArray := []string{}
	for _, str := range inputArray {
		if str != "" {
			returnArray = append(returnArray, str)
		}
	}
	return returnArray
{'fraction_non_alphanumeric': 0.18080561489166921, 'fraction_numerical': 0.01052792187976808, 'mean_word_length': 4.3597710547833195, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1279284', 'n_tokens_mistral': 2860, 'n_tokens_neox': 2638, 'n_words': 442}