<commit_before>package ytdl

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	log "github.com/Sirupsen/logrus"
)

const youtubeBaseURL = "https://www.youtube.com/watch"
const youtubeEmbededBaseURL = "https://www.youtube.com/embed/"
const youtubeVideoEURL = "https://youtube.googleapis.com/v/"
const youtubeVideoInfoURL = "https://www.youtube.com/get_video_info"
const youtubeDateFormat = "2006-01-02"

// VideoInfo contains the info a youtube video
type VideoInfo struct {
	// The video ID
	ID string `json:"id"`
	// The video title
	Title string `json:"title"`
	// The video description
	Description string `json:"description"`
	// The date the video was published
	DatePublished time.Time `json:"datePublished"`
	// Formats the video is available in
	Formats FormatList `json:"formats"`
	// List of keywords associated with the video
	Keywords []string `json:"keywords"`
	// Author of the video
	Author string `json:"author"`
	// Duration of the video
	Duration time.Duration

	htmlPlayerFile string
}<commit_msg>fix logrus uppercase import issue<commit_after>package ytdl

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	log "github.com/sirupsen/logrus"
)

const youtubeBaseURL = "https://www.youtube.com/watch"
const youtubeEmbededBaseURL = "https://www.youtube.com/embed/"
const youtubeVideoEURL = "https://youtube.googleapis.com/v/"
const youtubeVideoInfoURL = "https://www.youtube.com/get_video_info"
const youtubeDateFormat = "2006-01-02"

// VideoInfo contains the info a youtube video
type VideoInfo struct {
	// The video ID
	ID string `json:"id"`
	// The video title
	Title string `json:"title"`
	// The video description
	Description string `json:"description"`
	// The date the video was published
	DatePublished time.Time `json:"datePublished"`
	// Formats the video is available in
	Formats FormatList `json:"formats"`
	// List of keywords associated with the video
	Keywords []string `json:"keywords"`
	// Author of the video
	Author string `json:"author"`
	// Duration of the video
	Duration time.Duration

	htmlPlayerFile string
}
{'fraction_non_alphanumeric': 0.12736660929432014, 'fraction_numerical': 0.0068846815834767644, 'mean_word_length': 4.369515011547344, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 6, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '53771', 'n_tokens_mistral': 824, 'n_tokens_neox': 768, 'n_words': 214}