<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"log"
	"net/url"
	"os"
	"strings"
)

var (
	EcAsics = []string{"http://www.asics.com/gb/en-gb", "http://www.asics.com/fr/fr-fr", "http://www.asics.com/es/es-es", "http://www.asics.com/it/it-it"}

	NoEcAsics = []string{"https://www.asics.co.za/", "https://www.asics.pl", "https://de.asics.ch"}
)

type ParseData struct {
	Locale string
	Header []Link
	Footer []Link
}

type Link struct {
	Title string
	Url   string
	Index int
	img   string
	Items []ItemLink
}

type ItemLink struct {
	Title string
	Url   string
	Index int
	img   string
}

func ParseECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	parsedata := ParseData{
		Locale: strings.Split(target_url.Path, "/")[2],
	}

	//parse the header
	// doc.Find("header #main-menu").Children().Each(func(i int, s *goquery.Selection) {

	//parse the footer
	doc.Find("footer").Children().Filter(".tiger-clearfix-toggle").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title: s.Children().Filter("h4").First().Text(),
			Url:   "#",
			Index: i + 1,
			img:   "",
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{<commit_msg>modify the img to exportable<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"log"
	"net/url"
	"os"
	"strings"
)

var (
	EcAsics = []string{"http://www.asics.com/gb/en-gb", "http://www.asics.com/fr/fr-fr", "http://www.asics.com/es/es-es", "http://www.asics.com/it/it-it"}

	NoEcAsics = []string{"https://www.asics.co.za/", "https://www.asics.pl", "https://de.asics.ch"}
)

type ParseData struct {
	Locale string
	Header []Link
	Footer []Link
}

type Link struct {
	Title string
	Url   string
	Index int
	Img   string
	Items []ItemLink
}

type ItemLink struct {
	Title string
	Url   string
	Index int
	Img   string
}

func ParseECHtml(url_string string) {

	target_url, _ := url.Parse(url_string)
	doc, err := goquery.NewDocument(url_string)
	if err != nil {
		log.Fatal(err)
	}

	parsedata := ParseData{
		Locale: strings.Split(target_url.Path, "/")[2],
	}

	//parse the header
	// doc.Find("header #main-menu").Children().Each(func(i int, s *goquery.Selection) {

	//parse the footer
	doc.Find("footer").Children().Filter(".tiger-clearfix-toggle").Each(func(i int, s *goquery.Selection) {
		link := Link{
			Title: s.Children().Filter("h4").First().Text(),
			Url:   "#",
			Index: i + 1,
			img:   "",
		}

		s.Children().Filter("ul").Children().Each(func(i int, s *goquery.Selection) {
			itemlink := ItemLink{
{'fraction_non_alphanumeric': 0.18927911275415896, 'fraction_numerical': 0.0022181146025878, 'mean_word_length': 3.6098807495741054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1621428', 'n_tokens_mistral': 1105, 'n_tokens_neox': 1054, 'n_words': 203}