<commit_before>import sys
import argparse
import requests
import urllib
try:
	from urllib.request import urlopen
except ImportError:
	from urllib2 import urlopen
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from bs4 import BeautifulSoup
from .downloader import download_series, download_parallel
from .utils import FILE_EXTENSIONS, THREAT_EXTENSIONS, DEFAULTS

search_url = "https://www.google.com/search"

s = requests.Session()
# Max retries and back-off strategy so all requests to http:// sleep before retrying
retries = Retry(total=5,
				backoff_factor=0.1,
				status_forcelist=[ 500, 502, 503, 504 ])
s.mount('http://', HTTPAdapter(max_retries=retries))


def scrape(html):
	"""
	function to scrape file links from html response
	"""
	soup = BeautifulSoup(html, 'lxml')
	results = soup.findAll('h3', {'class': 'r'})
	links = []
	for result in results:
		link = result.a['href'][7:].split('&')[0]
		links.append(link)
	return links


def get_links(limit, params, headers):
	"""
	function to fetch links equal to limit

	every Google search result page has a start index.
	every page contains 10 search results.
	"""
	links = []
	for start_index in range(0, limit, 10):
		params['start'] = start_index
		resp = s.get(search_url, params = params, headers = headers)
		page_links = scrape(resp.content)
		links.extend(page_links)
	return links[:limit]


def get_url_nofollow(url):
	""" 
	function to get return code of a url

	Credits: http://blog.jasonantman.com/2013/06/python-script-to-check-a-list-of-urls-for-return-code-and-final-return-code-if-redirected/
	"""
	try:
		response = urlopen(url)
		code = response.getcode()
		return code
	except urllib.error.HTTPError as e:
		return e.code
	except:
		return 0


def validate_links(links):
	"""
	function to validate urls based on http(s) prefix and return code
	"""
	valid_links = []
	for link in links:
		if link[:7] in "http://" or link[:8] in "https://":
			valid_links.append(link)
	
	if not valid_links:
		print("No files found.")

	# checking valid urls for return code
	urls = {}
	for link in valid_links:
		if 'github.com' and '/blob/' in link:
			link = link.replace('/blob/', '/raw/')
		urls[link] = {'code': get_url_nofollow(link)}
		
	
	# printing valid urls with return code 200
	available_urls = []
	for url in urls:
		print("code: %d\turl: %s" % (urls[url]['code'], url))
		if urls[url]['code'] != 0:
			available_urls.append(url)

	return available_urls


def search(query, website, file_type = 'pdf', limit = 10):
	"""
	main function to search for links and return valid ones
	"""
	if website is None:<commit_msg>Added sys.exit() for successful termination.<commit_after>import sys
import argparse
import requests
import urllib
try:
	from urllib.request import urlopen
except ImportError:
	from urllib2 import urlopen
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from bs4 import BeautifulSoup
from .downloader import download_series, download_parallel
from .utils import FILE_EXTENSIONS, THREAT_EXTENSIONS, DEFAULTS

search_url = "https://www.google.com/search"

s = requests.Session()
# Max retries and back-off strategy so all requests to http:// sleep before retrying
retries = Retry(total=5,
				backoff_factor=0.1,
				status_forcelist=[ 500, 502, 503, 504 ])
s.mount('http://', HTTPAdapter(max_retries=retries))


def scrape(html):
	"""
	function to scrape file links from html response
	"""
	soup = BeautifulSoup(html, 'lxml')
	results = soup.findAll('h3', {'class': 'r'})
	links = []
	for result in results:
		link = result.a['href'][7:].split('&')[0]
		links.append(link)
	return links


def get_links(limit, params, headers):
	"""
	function to fetch links equal to limit

	every Google search result page has a start index.
	every page contains 10 search results.
	"""
	links = []
	for start_index in range(0, limit, 10):
		params['start'] = start_index
		resp = s.get(search_url, params = params, headers = headers)
		page_links = scrape(resp.content)
		links.extend(page_links)
	return links[:limit]


def get_url_nofollow(url):
	""" 
	function to get return code of a url

	Credits: http://blog.jasonantman.com/2013/06/python-script-to-check-a-list-of-urls-for-return-code-and-final-return-code-if-redirected/
	"""
	try:
		response = urlopen(url)
		code = response.getcode()
		return code
	except urllib.error.HTTPError as e:
		return e.code
	except:
		return 0


def validate_links(links):
	"""
	function to validate urls based on http(s) prefix and return code
	"""
	valid_links = []
	for link in links:
		if link[:7] in "http://" or link[:8] in "https://":
			valid_links.append(link)
	
	if not valid_links:
		print("No files found.")
		sys.exit(0)

	# checking valid urls for return code
	urls = {}
	for link in valid_links:
		if 'github.com' and '/blob/' in link:
			link = link.replace('/blob/', '/raw/')
		urls[link] = {'code': get_url_nofollow(link)}
		
	
	# printing valid urls with return code 200
	available_urls = []
	for url in urls:
		print("code: %d\turl: %s" % (urls[url]['code'], url))
		if urls[url]['code'] != 0:
			available_urls.append(url)

	return available_urls


def search(query, website, file_type = 'pdf', limit = 10):
	"""
	main function to search for links and return valid ones
	"""
	if website is None:
{'fraction_non_alphanumeric': 0.12708841749577623, 'fraction_numerical': 0.015581002440397973, 'mean_word_length': 4.238938053097345, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3248634', 'n_tokens_mistral': 1942, 'n_tokens_neox': 1805, 'n_words': 544}