<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
__name__="db_stats"
__doc__= "A tool to chekc integrity of the crawl"
__author__="<NAME>"

import sys
from database import TaskDB, Database

def check_project(name):
	'''check the parameters of the project'''
	taskdb = TaskDB()
	project = taskdb.coll.find_one({"name": name})
	print "The project has been run %d times for:" %len(project["date"])
	
		
	if project is not None:
		for a,s in zip(project["action"], project["status"]):
			print "-", a, s
		#~ for k, v  in project.items():
			#~ if v is not False:
				#~ print "_______________________________________"
				#~ print "|",k,"|", v,"|"
				#~ 
		#~ print "_______________________________________"
		return project
	else:
		sys.exit("No project found")
	
	
def check_sources(project_name):
	'''check the sources status'''
	project_db = Database(project_name)
	project_db.create_colls()
	sources_nb = project_db.sources.count()
	error_nb = len([n["status"][-1] for n in project_db.sources.find() if n["status"][-1] is False])
	ok_nb = len([n["status"][-1] for n in project_db.sources.find() if n["status"][-1] is True])
	http_error = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] == 400])
	content_error = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] == 404])
	forbidden_error = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] == 403])
	extraction_error = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] == 700])
	others = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] not in [700, 400, 403, 404]])
	print "Error nb: %d sources on %d total" %(error_nb, sources_nb)
	print "Error type:"
	print "- %d network errors (impossible to acess to the website)" %(http_error+forbidden_error)
	print "- %d errors because page is not HTML (PDF or Video, or img ...)" %content_error
	print "- %d errors on extracting the text" %extraction_error
	print "- %d errors undefined" %others
	print "\n"
	print "Details:"
	for n in project_db.sources.find():
		if n["code"][-1] != 100 and n["code"][-1] not in [700, 400, 403, 404]:
			print "\t-", n["url"]
			print n["code"][-1],"\t", n["msg"][-1]

def check_results(project_name):
	project_db = Database(project_name)
	project_db.create_colls()
	nb_results = project_db.results.count()
	results = project_db.results.find()
	print nb_results, "results"
	#~ for n in results:
		#~ print "url", n["url"]
		#~ print "outlinks", len(set(n["cited_links_ids"]))
		#~ print "depth", n["depth"]

def check_logs(project_name):
	project_db = Database(project_name)
	project_db.create_colls()
	nb_logs = project_db.logs.count()
	print nb_logs, "errors"
	logs = project_db.logs.find()
	
	http_error = len([n["code"] for n in project_db.logs.find() if n["code"] in [400, 401, 402,403]])
	content_error = len([n["code"] for n in project_db.logs.find() if n["code"] == 404])
	extraction_error = len([n["code"] for n in project_db.logs.find() if n["code"] == 700])
	others = len([n["code"] for n in project_db.logs.find() if n["code"] not in [700, 400, 403, 404]])
	for n in logs:
		if n["code"] not in [700,401, 400, 403, 404]:
			print n["code"], n["msg"]
		#~ print n["status"][-1], n["code"][-1]
		#n["status"], n["msg"]
	
def main(project_name):
	check_project(project_name)
	check_sources(project_name)
	check_results(project_name)
	check_logs(project_name)
	sys.exit()

main("RRI_ET_4")<commit_msg>Name of db in command line easier<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
__name__="db_stats"
__description__= "A tool to chekc integrity of the crawl"
__author__="<NAME>"
__version__ = "0.0.1"
__doc__ = __doc__ = '''Crawtext.
Description:
A simple tool to check integrity of the crawl. Provides Simple stats.

Usage:
    db_stats.py (<name>)
'''


import sys
from docopt import docopt
from database import TaskDB, Database

def check_project(name):
	'''check the parameters of the project'''
	taskdb = TaskDB()
	try:
		project = taskdb.coll.find_one({"name": name})
		print "The project has been run %d times for:" %len(project["date"])
		
			
		if project is not None:
			for a,s in zip(project["action"], project["status"]):
				print "-", a, s
			#~ for k, v  in project.items():
				#~ if v is not False:
					#~ print "_______________________________________"
					#~ print "|",k,"|", v,"|"
					#~ 
			#~ print "_______________________________________"
			return project
		else:
			sys.exit("No project found")
	except Exception:
		sys.exit("No project found")	
	
def check_sources(project_name):
	'''check the sources status'''
	project_db = Database(project_name)
	project_db.create_colls()
	sources_nb = project_db.sources.count()
	error_nb = len([n["status"][-1] for n in project_db.sources.find() if n["status"][-1] is False])
	ok_nb = len([n["status"][-1] for n in project_db.sources.find() if n["status"][-1] is True])
	http_error = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] == 400])
	content_error = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] == 404])

	forbidden_error = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] == 403])
	extraction_error = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] == 700])
	others = len([n["code"][-1] for n in project_db.sources.find() if n["code"][-1] not in [700, 400, 403, 404]])
	print "Error nb: %d sources on %d total" %(error_nb, sources_nb)
	print "Error type:"
	print "- %d network errors (impossible to acess to the website)" %(http_error+forbidden_error)
	print "- %d errors because page is not HTML (PDF or Video, or img ...)" %content_error
	print "- %d errors on extracting the text" %extraction_error
	print "- %d errors undefined" %others
	print "\n"
	print "Details:"
	for n in project_db.sources.find():
		if n["code"][-1] != 100 and n["code"][-1] not in [700, 400, 403, 404]:
			print "\t-", n["url"]
			print n["code"][-1],"\t", n["msg"][-1]

def check_results(project_name):
	project_db = Database(project_name)
	project_db.create_colls()
	nb_results = project_db.results.count()
	results = project_db.results.find()
	print nb_results, "results"
	#~ for n in results:
		#~ print "url", n["url"]
		#~ print "outlinks", len(set(n["cited_links_ids"]))
		#~ print "depth", n["depth"]

def check_logs(project_name):
	
	project_db = Database(project_name)
	project_db.create_colls()
	nb_logs = project_db.logs.count()
	print nb_logs, "errors"
	logs = project_db.logs.find()
	
	http_error = len([n["code"] for n in project_db.logs.find() if n["code"] in [400, 401, 402,403]])
	content_error = len([n["code"] for n in project_db.logs.find() if n["code"] == 404])
	extraction_error = len([n["code"] for n in project_db.logs.find() if n["code"] == 700])
	others = len([n["code"] for n in project_db.logs.find() if n["code"] not in [700, 400, 403, 404]])
	for n in logs:
		if n["code"] not in [700,401, 400, 403, 404]:
			print n["code"], n["msg"]
		#~ print n["status"][-1], n["code"][-1]
		#n["status"], n["msg"]
	
def main(project_name):
	check_project(project_name)
	check_sources(project_name)
	check_results(project_name)
	check_logs(project_name)
	sys.exit()

if __name__ == "db_stats":
	print docopt(__doc__)
	main(docopt(__doc__)['<name>'])
{'fraction_non_alphanumeric': 0.1831275720164609, 'fraction_numerical': 0.029355281207133058, 'mean_word_length': 4.727415553809898, 'pattern_counts': {'":': 3, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6318079', 'n_tokens_mistral': 2869, 'n_tokens_neox': 2657, 'n_words': 731}