<commit_before>#!/usr/bin/env python3
import re
import json
import time
import requests
from bs4 import BeautifulSoup
import subprocess

baseUrl = "https://www.bitchute.com"

class VideoLink:
	def __init__(self, containerSoup):
		titleDiv = containerSoup.findAll('div', "channel-videos-title")[0]
		linkSoup = titleDiv.findAll('a')[0]
		
		self.title = linkSoup.string
		self.pageUrl = linkSoup.get("href")
		self.id = self.pageUrl.split("/")[-1]
		self.thumbnail = None
		self.url = None
		#before we can find thumnails let's strip out play button images.
		for playButton in containerSoup.findAll('img', "play-overlay-icon"):
			playButton.extract()
		
		thumbnailMatches = containerSoup.findAll('img', "img-responsive")
		
		if thumbnailMatches:
			self.thumbnail = baseUrl + thumbnailMatches[0].get("src")

	def getUrl(self, channelId):
		return(baseUrl + "/torrent/" + channelId + "/" + self.id + ".torrent")
	def setUrl(self, channelId):
		self.url = self.getUrl(channelId)

class Channel:
	def __init__(self, channelName):
		self.channelName = channelName
		self.videos = []
		self.thumbnail = None
		self.page = 1
		self.hasPrevPage = False
		self.hasNextPage = False

		r = requests.get(baseUrl + "/" + self.channelName)
		soup = BeautifulSoup(r.text, 'html.parser')

		thumbnailImages = soup.findAll("img", id="fileupload-medium-icon-2")
		if thumbnailImages:
			self.thumbnail = baseUrl + thumbnailImages[0].get("src")

		for videoContainer in soup.findAll('div', "channel-videos-container"):
			self.videos.append(VideoLink(videoContainer))

		paginationLists = soup.findAll("ul", "pagination")
		for paginationList in paginationLists:
			for page in paginationList.findAll("li"):
				# skip any page number list items that have the "disabled" class.
				if page.has_attr("class"):
					if "disabled" in page['class']:
						continue 
				# it's not disabled, keep on trucking.
				if page.findAll("i", "fa-angle-double-left"):
					self.hasPrevPage = True
				if page.findAll("i", "fa-angle-double-right"):
					self.hasNextPage = True
		
		# for now I only know how to find the channel's ID from a video, so take the last item
		# in videos and find the channel's ID.
		videoRequest = requests.get(baseUrl + self.videos[-1].pageUrl)
		channelIdMatches = re.search('/torrent/\d+', videoRequest.text)
		if channelIdMatches:
			self.id = channelIdMatches.group().split("/")[-1]
		else:
			raise ValueError("channel Id not found for " + self.channelName + ".")
		
		# armed with a channelId we can set the url for all our videos.
		for video in self.videos:
			video.setUrl(self.id)

def login():
	#BitChute uses a token to prevent csrf attacks, get the token to make our request.
	r = requests.get(baseUrl)
	csrfJar = r.cookies
	soup = BeautifulSoup(r.text, 'html.parser')
	csrftoken = soup.findAll("input", {"name":"csrfmiddlewaretoken"})[0].get("value")

	username = None
	password = <PASSWORD>

	#Fetch the user info from settings.json
	settingsFile = open("settings.json", "r")
	settings = json.loads(settingsFile.read())
	settingsFile.close()<commit_msg>Testing out a method to get different pages of videos.<commit_after>#!/usr/bin/env python3
import re
import json
import time
import requests
from bs4 import BeautifulSoup
import subprocess

baseUrl = "https://www.bitchute.com"

class VideoLink:
	def __init__(self, containerSoup):
		titleDiv = containerSoup.findAll('div', "channel-videos-title")[0]
		linkSoup = titleDiv.findAll('a')[0]
		
		self.title = linkSoup.string
		self.pageUrl = linkSoup.get("href")
		self.id = self.pageUrl.split("/")[-1]
		self.thumbnail = None
		self.url = None
		#before we can find thumnails let's strip out play button images.
		for playButton in containerSoup.findAll('img', "play-overlay-icon"):
			playButton.extract()
		
		thumbnailMatches = containerSoup.findAll('img', "img-responsive")
		
		if thumbnailMatches:
			self.thumbnail = baseUrl + thumbnailMatches[0].get("src")

	def getUrl(self, channelId):
		return(baseUrl + "/torrent/" + channelId + "/" + self.id + ".torrent")
	def setUrl(self, channelId):
		self.url = self.getUrl(channelId)

class Channel:
	def __init__(self, channelName):
		self.channelName = channelName
		self.videos = []
		self.thumbnail = None
		self.page = 1
		self.hasPrevPage = False
		self.hasNextPage = False

		self.setPage(self.page)

	def setPage(self, pageNumber):
		r = requests.get(baseUrl + "/" + self.channelName)
		soup = BeautifulSoup(r.text, 'html.parser')

		thumbnailImages = soup.findAll("img", id="fileupload-medium-icon-2")
		if thumbnailImages:
			self.thumbnail = baseUrl + thumbnailImages[0].get("src")

		for videoContainer in soup.findAll('div', "channel-videos-container"):
			self.videos.append(VideoLink(videoContainer))

		paginationLists = soup.findAll("ul", "pagination")
		for paginationList in paginationLists:
			for page in paginationList.findAll("li"):
				# skip any page number list items that have the "disabled" class.
				if page.has_attr("class"):
					if "disabled" in page['class']:
						continue 
				# it's not disabled, keep on trucking.
				if page.findAll("i", "fa-angle-double-left"):
					self.hasPrevPage = True
				if page.findAll("i", "fa-angle-double-right"):
					self.hasNextPage = True
		
		# for now I only know how to find the channel's ID from a video, so take the last item
		# in videos and find the channel's ID.
		videoRequest = requests.get(baseUrl + self.videos[-1].pageUrl)
		channelIdMatches = re.search('/torrent/\d+', videoRequest.text)
		if channelIdMatches:
			self.id = channelIdMatches.group().split("/")[-1]
		else:
			raise ValueError("channel Id not found for " + self.channelName + ".")
		
		# armed with a channelId we can set the url for all our videos.
		for video in self.videos:
			video.setUrl(self.id)


def login():
	#BitChute uses a token to prevent csrf attacks, get the token to make our request.
	r = requests.get(baseUrl)
	csrfJar = r.cookies
	soup = BeautifulSoup(r.text, 'html.parser')
	csrftoken = soup.findAll("input", {"name":"csrfmiddlewaretoken"})[0].get("value")

	username = None
	password = <PASSWORD>

	#Fetch the user info from settings.json
	settingsFile = open("settings.json", "r")
	settings = json.loads(settingsFile.read())
	settingsFile.close()
{'fraction_non_alphanumeric': 0.13219035410991828, 'fraction_numerical': 0.0038455375741067136, 'mean_word_length': 4.099673202614379, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '268243', 'n_tokens_mistral': 2285, 'n_tokens_neox': 2057, 'n_words': 536}