<commit_before>package main

import (
        "fmt"
        "net"
        "strings"
        "os"
        "bufio"
        "flag"
)

func LoadGooglebotList() []string {
        file, err := os.Open("googlebot.data")
        if err != nil {
                fmt.Println("Couldn't open googlebot.data:", err)
                os.Exit(1)
        }
        defer file.Close()

        var lines []string
        scanner := bufio.NewScanner(file)
        for scanner.Scan() {
                lines = append(lines, scanner.Text())
        }

        return lines
}

func IdentifiesAsGooglebot(list []string, useragent string) bool {
        for _, test := range list {
                if strings.Contains(test, useragent) {
                        return true
                }
        }

        return false
}

func IsGooglebot(lookupResult []string) bool {
        domain := strings.Join(strings.Split(lookupResult[0], ".")[1:3], ".")
        //      fmt.Println(domain)
        if domain == "google.com" || domain == "googlebot.com" {
                return true
        } else {
                return false
        }
}

func main() {
        addressPtr   := flag.String("address", "", "IP address of requester")
        useragentPtr := flag.String("useragent", "", "User Agent of requester")
        flag.Parse()

        list := LoadGooglebotList()

        if IdentifiesAsGooglebot(list, *useragentPtr) {
                addr, err := net.LookupAddr(*addressPtr)
                if err != nil {
                        fmt.Println(err)
                        os.Exit(1)
                }
                if IsGooglebot(addr) {
                        fmt.Println("YES")
                } else {
                        fmt.Println("NO")
                }
        }
}<commit_msg>Extract and look through all reverse domains<commit_after>package main

import (
        "fmt"
        "net"
        "strings"
        "os"
        "bufio"
        "flag"
)

func LoadGooglebotList() []string {
        file, err := os.Open("googlebot.data")
        if err != nil {
                fmt.Println("Couldn't open googlebot.data:", err)
                os.Exit(1)
        }
        defer file.Close()

        var lines []string
        scanner := bufio.NewScanner(file)
        for scanner.Scan() {
                lines = append(lines, scanner.Text())
        }

        return lines
}

func IdentifiesAsGooglebot(list []string, useragent string) bool {
        for _, test := range list {
                if strings.Contains(test, useragent) {
                        return true
                }
        }

        return false
}

func IsGooglebot(lookupResult []string) bool {
        for _, result := range lookupResult {
                parts := strings.Split(result, ".")
                domain := strings.Join(parts[len(parts) - 3:len(parts) - 1], ".")
                fmt.Println(domain)
                if domain == "google.com" || domain == "googlebot.com" {
                        return true
		}
	}
	return false
}

func Lookup(address string, useragent string, list []string) {
        if IdentifiesAsGooglebot(list, useragent) {
                addr, err := net.LookupAddr(address)
                if err != nil {
                        fmt.Println(err)
                        os.Exit(1)
                }
                if IsGooglebot(addr) {
                        fmt.Println("YES")
                } else {
                        fmt.Println("NO")
                }
        }
}

func main() {
        addressPtr   := flag.String("address", "", "IP address of requester")
        useragentPtr := flag.String("useragent", "", "User Agent of requester")
        flag.Parse()

        list := LoadGooglebotList()

        Lookup(*addressPtr, *useragentPtr, list)
}
{'fraction_non_alphanumeric': 0.11741319904585211, 'fraction_numerical': 0.0023853697323085077, 'mean_word_length': 1.3946700507614214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7340367', 'n_tokens_mistral': 1063, 'n_tokens_neox': 998, 'n_words': 232}