<commit_before>Dataset of edge ids 0-5556:
	number of segments 5556
	num_of_segments_with_score 1396
	num_of_image_urls_to_attempt_to_down 8376

	For example 30m blocks
	number of segments longer than:
	- (0,30m> 4416
	- (30,60m> 1164
	- (60,90m> 1380
	- (90,inf) 1416

	//ps: 4416+1164+1380+1416 = 8376

	25 meters can be minimum
	( 0,50m> keep
	(50,75m> split in half
	(75,100m> split in thirds
	(100,inf> split in quarters

min_allowed_distance => num_of_image_urls_to_attempt_to_down
10 	=>	35376
15 	=>	24060
20 * 	=>	18534
25	=>	15408
30 	=>	13350<commit_msg>correct vals for 5556x and 1200x<commit_after>Dataset of edge ids 0-5556:
	number of segments 5556
	num_of_segments_with_score 1396
	num_of_image_urls_to_attempt_to_down 8376

	For example 30m blocks
	number of segments longer than:
	- (0,30m> 4416
	- (30,60m> 1164
	- (60,90m> 1380
	- (90,inf) 1416

	//ps: 4416+1164+1380+1416 = 8376

	25 meters can be minimum
	( 0,50m> keep
	(50,75m> split in half
	(75,100m> split in thirds
	(100,inf> split in quarters

min_allowed_distance => num_of_image_urls_to_attempt_to_down
5556x
10 	=>	35376
15 	=>	24060
20 * 	=>	18534
25	=>	15408
30 	=>	13350

min_allowed_distance => num_of_image_urls_to_attempt_to_down
1200x
10 	=>	17796
15 	=>	11934
20  	=>	9012
25	=>	7338
30 	=>	6282
{'fraction_non_alphanumeric': 0.0933852140077821, 'fraction_numerical': 0.22412451361867705, 'mean_word_length': 4.270491803278689, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5697797', 'n_tokens_mistral': 759, 'n_tokens_neox': 577, 'n_words': 146}