<commit_before>from lxml import html
import requests
import codecs
import json
from lxml import objectify


home = 'http://gatn.mosreg.ru'
links = []
pics = []
items = []
texts = []
for i in range(5, 0, -1):

	page = requests.get(home + '/multimedia/novosti/novosti/?PAGEN_1=%s'%i)
	tree = html.fromstring(page.content)

	links_current = tree.xpath('//div[@class="news-item"]/a[@class="link"]/@href')
	pics_current = tree.xpath('//div[@class="news-item"]/a[@class="link"]/div[@class="pic"]/img/@src') 
	items_current = tree.xpath('//div[@class="news-item"]/a[@class="link"]/div[@class="title"]/text()')

	links.extend(home + s.strip(' \t\n\r') for s in links_current)
	pics.extend(home + s.strip(' \t\n\r') for s in pics_current)
	items.extend(items_current)



	for elem in links_current:
		page = requests.get(home + elem)
		tree = html.fromstring(page.content)
		text = tree.xpath('//div[@class="text"]/p')
		p_array = [html.tostring(elem, encoding='unicode', pretty_print=True) for elem in text]
		p_string = "".join(p_array)
		texts.append(p_string + "<p>Добавлено 27.04.2016</p>")
		


	print(i)


zipped = list(zip(links, pics, items, texts))<commit_msg>new gatn site<commit_after>from lxml import html
import requests
import codecs
import json
from lxml import objectify
from datetime import datetime


home = 'http://gatn.mosreg.ru'
links = []
pics = []
items = []
texts = []
for i in range(5, 0, -1):

	page = requests.get(home + '/sobytiya/novosti_ministerstva/?page=%s'%i)
	tree = html.fromstring(page.content)

	links_current = tree.xpath('//div[@class="event__content"]/h3[@class="event__title"]/a/@href')
	pics_current = tree.xpath('//div[@class="event__image-wrap"]/a[@class="event__image-link"]/img/@src')
	items_current = tree.xpath('//div[@class="event__content"]/h3/a/text()')

	links.extend(home + s.strip(' \t\n\r') for s in links_current)
	pics.extend(home + s.strip(' \t\n\r') for s in pics_current)
	items.extend(items_current)

	# print(items_current)

	for elem in links_current:
		page = requests.get(home + elem)
		tree = html.fromstring(page.content)
		text = tree.xpath('//div[@class="article__inner  typical"]/p')
		p_array = [html.tostring(elem, encoding='unicode', pretty_print=True) for elem in text]
		p_string = "".join(p_array)
		date = datetime.now().strftime('%d.%m.%Y')
		texts.append(p_string + "<p>Добавлено %s</p>" % date)



	print(i)


zipped = list(zip(links, pics, items, texts))
{'fraction_non_alphanumeric': 0.18912237330037082, 'fraction_numerical': 0.007004532344458179, 'mean_word_length': 4.822541966426859, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2047152', 'n_tokens_mistral': 976, 'n_tokens_neox': 916, 'n_words': 174}