<commit_before>package crawler

import (
	"fmt"
	"io"
	"log"
	"strings"

	"os"

	"github.com/PuerkitoBio/goquery"
	"github.com/andybalholm/cascadia"
)

const (
	operation_normal_identifier                 = "делник"
	operation_pre_holiday_identifier            = "предпразник"
	operation_holiday_identifier                = "празник"
	operation_pre_holiday_or_holiday_identifier = "предпразник / празник"
	operation_pre_holiday_or_holiday_prefix     = "предпразник, празник"
)

type OperationType int

const (
	OPERATION_NORMAL OperationType = iota
	OPERATION_PRE_HOLIDAY
	OPERATION_HOLIDAY
)

var operationStrings = [...]string{OPERATION_NORMAL: "Weekday",
	OPERATION_PRE_HOLIDAY: "Pre-Holiday",
	OPERATION_HOLIDAY:     "Holiday"}

func getLineOperationType(doc *goquery.Document) OperationType {
	operationTypeRaw := doc.Find(".schedule_active_list_active_tab").Text()
	switch strings.TrimSpace(operationTypeRaw) {
	case operation_normal_identifier:
		return OPERATION_NORMAL
	case operation_pre_holiday_or_holiday_identifier:
		return OPERATION_HOLIDAY
	case operation_holiday_identifier:
		return OPERATION_HOLIDAY
	case operation_pre_holiday_identifier:
		return OPERATION_PRE_HOLIDAY
	default:
		panic(fmt.Errorf("No operation mode found %v", operationTypeRaw))
	}
}

type Direction struct {
	Name string
	ID   string
	URL  string
}
type Directions [2]Direction
type StopTuple struct {
	Name         string
	Number       string
	URL          string
	StopViewLink string
}
type StopsNames []StopTuple

//ID which seems to be unique and it's for every diferent type of operation
type SCID string
type SCIDS [3]SCID

//SCID
func findDirections(doc *goquery.Document) Directions {
	directions := Directions{}
	doc.Find(".schedule_view_direction_tabs").First().Find("a").Each(func(i int, s *goquery.Selection) {

		name := strings.TrimSpace(s.Text())
		url, ok := s.Attr("href")
		if !ok {
			//TODO-Log
			fmt.Fprintf(os.Stderr, "No URL for direction view with direction with name [%v]", name)
		}
		splits := strings.Split(url, "/")
		id := splits[len(splits)-1]
		directions[i] = Direction{name, id, url}
	})
	return directions
}

func findSCIDs(doc *goquery.Document) SCIDS {
	scids := SCIDS{}
	matcher := cascadia.MustCompile(`[id^="schedule"][id$="button"]:not([id*="direction"])`)
	doc.FindMatcher(matcher).Each(func(i int, s *goquery.Selection) {
		attribute_id, _ := s.Attr("id")
		splits := strings.Split(attribute_id, "_")
		scids[i] = SCID(splits[1])
	})
	return scids
}

//get sschedule from schedule_6484_direction_172_sign_693
func findStopNames(doc *goquery.Document) StopsNames {
	stopNames := make(StopsNames, 0)
	doc.Find(".schedule_view_route_directions").
		First().
		Find(".schedule_direction_sign_wrapper ul").
		First().
		Find("li").Each(func(i int, s *goquery.Selection) {

		nameSelection := s.Find(".stop_change")
		name := nameSelection.Text()
		stopViewLink, ok := nameSelection.Attr("href")
		if !ok {
			//TODO-Log
			fmt.Fprintf(os.Stderr, "No URL for stop view for stop with name [%v]", name)
		}

		numberSelection := s.Find(".stop_link")
		number := numberSelection.Text()
		url, ok := numberSelection.Attr("href")
		if !ok {
			//TODO-Log
			fmt.Fprintf(os.Stderr, "No URL for stop with name [%v] and number [%v]", name, number)
		}

		stopNames = append(stopNames, StopTuple{name, number, url, stopViewLink})
	})

	return stopNames
}

func getSelectionForOperationType(doc *goquery.Document, operationType OperationType) {
	doc.Find(".schedule_active_list_content").Each(func(i int, operationTypeSelection *goquery.Selection) {
		str := strings.TrimSpace(operationTypeSelection.ChildrenFiltered("h3").Text())
		var operationType OperationType
		switch true {
		case strings.HasPrefix(str, string(operation_normal_identifier)):
			operationType = OPERATION_NORMAL
		case strings.HasPrefix(str, string(operation_pre_holiday_or_holiday_prefix)):
			operationType = OPERATION_HOLIDAY
		case strings.HasPrefix(str, string(operation_holiday_identifier)):
			operationType = OPERATION_HOLIDAY
		case strings.HasPrefix(str, string(operation_pre_holiday_identifier)):
			operationType = OPERATION_PRE_HOLIDAY
		}

		fmt.Printf("Operation type: [%v]\n", operationStrings[operationType])

	})

}

func CrawlLine(line LineNameAndURL, r io.Reader) {
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		log.Fatal(err)
	}
	getSelectionForOperationType(doc, OPERATION_HOLIDAY)
	directions := findDirections(doc)
	fmt.Println(directions)
	stopsNames := findStopNames(doc)
	fmt.Println(stopsNames)
	scids := findSCIDs(doc)
	fmt.Println(scids)
}
<commit_msg>Change types to something more useful and understandable Add both of stop directions Add a function for finding operation type and get all operation ids for the line Add simple get times implementation that could be used in the future, but a more advanced one by parsing some JS could be in use as well<commit_after>package crawler

import (
	"fmt"
	"io"
	"log"
	"strings"

	"os"

	"github.com/PuerkitoBio/goquery"
)

const (
	operation_normal_identifier                 = "делник"
	operation_pre_holiday_identifier            = "предпразник"
	operation_holiday_identifier                = "празник"
	operation_pre_holiday_or_holiday_identifier = "предпразник / празник"
	operation_pre_holiday_or_holiday_prefix     = "предпразник, празник"
)

type OperationType int

type OperationTypes []OperationType

const (
	OPERATION_NORMAL OperationType = iota
	OPERATION_PRE_HOLIDAY
	OPERATION_HOLIDAY
	OPERATION_UNKNOWN
)

var operationsIdentifiersMap = map[string]OperationType{operation_normal_identifier: OPERATION_NORMAL,
	operation_pre_holiday_or_holiday_identifier: OPERATION_HOLIDAY,
	operation_holiday_identifier:                OPERATION_HOLIDAY,
	operation_pre_holiday_identifier:            OPERATION_PRE_HOLIDAY}

var operationStrings = [...]string{OPERATION_NORMAL: "Weekday",
	OPERATION_PRE_HOLIDAY: "Pre-Holiday",
	OPERATION_HOLIDAY:     "Holiday"}

func (o OperationType) String() string {
	return operationStrings[o]
}

// ===============================================
//Current date and page postion related functions
func getLineOperationType(doc *goquery.Document) OperationType {
	operationTypeRaw := doc.Find(".schedule_active_list_active_tab").Text()
	switch strings.TrimSpace(operationTypeRaw) {
	case operation_normal_identifier:
		return OPERATION_NORMAL
	case operation_pre_holiday_or_holiday_identifier:
		return OPERATION_HOLIDAY
	case operation_holiday_identifier:
		return OPERATION_HOLIDAY
	case operation_pre_holiday_identifier:
		return OPERATION_PRE_HOLIDAY
	default:
		panic(fmt.Errorf("No operation mode found %v", operationTypeRaw))
	}
}

// =============================================
type Direction struct {
	Name string
	ID   string
	URL  string
}

type Directions [2]Direction

type StopTuple struct {
	Name         string
	Number       string
	URL          string
	StopViewLink string
}
type StopsNames []StopTuple

//ID which seems to be unique and it's for every diferent type of operation
type OperationID string
type OperationTypeIDMap map[OperationType]OperationID

//
func findDirections(doc *goquery.Document) Directions {
	directions := Directions{}
	doc.Find(".schedule_view_direction_tabs").First().Find("a").Each(func(i int, s *goquery.Selection) {

		name := strings.TrimSpace(s.Text())
		url, ok := s.Attr("href")
		if !ok {
			//TODO-Log
			fmt.Fprintf(os.Stderr, "No URL for direction view with direction with name [%v]", name)
		}
		splits := strings.Split(url, "/")
		id := splits[len(splits)-1]
		directions[i] = Direction{name, id, url}
	})
	return directions
}

//get sschedule from schedule_6484_direction_172_sign_693
func findStopNames(doc *goquery.Document) (StopsNames, StopsNames) {
	stopNames := make(StopsNames, 0)
	doc.Find(".schedule_view_route_directions").
		First().
		Find(".schedule_direction_sign_wrapper ul").
		//First().
		Find("li").Each(func(i int, s *goquery.Selection) {

		nameSelection := s.Find(".stop_change")
		name := nameSelection.Text()
		stopViewLink, ok := nameSelection.Attr("href")
		if !ok {
			//TODO-Log
			fmt.Fprintf(os.Stderr, "No URL for stop view for stop with name [%v]", name)
		}

		numberSelection := s.Find(".stop_link")
		number := numberSelection.Text()
		url, ok := numberSelection.Attr("href")
		if !ok {
			//TODO-Log
			fmt.Fprintf(os.Stderr, "No URL for stop with name [%v] and number [%v]", name, number)
		}

		stopNames = append(stopNames, StopTuple{name, number, url, stopViewLink})
	})

	return stopNames[:len(stopNames)/2], stopNames[len(stopNames)/2:]
}
func findOperationType(s string) OperationType {
	for k, v := range operationsIdentifiersMap {
		if s == k {
			return v
		}
	}
	return OPERATION_UNKNOWN
}

func getOperationMap(doc *goquery.Document) OperationTypeIDMap {
	operationMap := make(OperationTypeIDMap)
	doc.Find(".schedule_active_list_tabs li a").Each(func(i int, linkSelection *goquery.Selection) {
		operationTypeString := strings.TrimSpace(linkSelection.Text())
		operationType := findOperationType(operationTypeString)
		if attributeID, ok := linkSelection.Attr("id"); ok {
			attributeParts := strings.Split(attributeID, "_")
			if len(attributeParts) > 1 {
				operationID := attributeParts[1]
				operationMap[operationType] = OperationID(operationID)
			} else {
				log.Printf("id of this link is not in the format 'schedule_xxxx_button' as needed: %v", attributeID)
			}
		} else {
			log.Println("This element does not have attribute id, which was expected for normal processing")

		}

	})
	return operationMap
}
func getTimes(doc *goquery.Document) []string {
	//TODO - Only One selection
	times := make([]string, 0)
	doc.Find(".schedule_times tbody").First().Find("a").Each(func(i int, s *goquery.Selection) {
		times = append(times, strings.TrimSpace(s.Text()))
	})
	return times
}

//func advancedTimes(){
//}

func CrawlLine(line LineNameAndURL, r io.Reader) {
	//doc, err := goquery.NewDocument("http://schedules.sofiatraffic.bg/server/html/schedule_load/6672/2696/377")
	//fmt.Println(doc.Text())
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		log.Fatal(err)
	}
	m := getOperationMap(doc)
	fmt.Println(m)
	directions := findDirections(doc)
	fmt.Println(directions)
	stopsNames1, stopnames2 := findStopNames(doc)
	fmt.Println(stopsNames1)
	fmt.Println(stopnames2)
	times := getTimes(doc)
	fmt.Println(times)

}

{'fraction_non_alphanumeric': 0.1218078442709839, 'fraction_numerical': 0.004721981304808711, 'mean_word_length': 4.899943149516771, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3166749', 'n_tokens_mistral': 3628, 'n_tokens_neox': 3434, 'n_words': 757}