<commit_before>import os
import json


def download_wall(domain, dataset_folder, cut=1000000):
	import vk
    session = vk.Session() #'c9a2edf19ed42f0ee1ba6a768ad6de25016971d24589165769987567a92cabbb2364bf3f4c050a868969f')
    api = vk.API(session)
    info = dict() 
    docs_folder = os.path.join(dataset_folder, "documents")
    os.makedirs(docs_folder, exist_ok=True)
    os.makedirs(os.path.join(dataset_folder, "meta"), exist_ok=True)
    
    
    id = 0
    offset = 0 
    while True:
        posts = api.wall.get(domain=domain, offset = offset, count=100)
        for i in range(1, len(posts)):
            post = posts[i]
            text = post["text"].replace("<br>","\n")
            if len(text)>50:
                id += 1
                text_id = "%06d.txt" % id
                info[text_id] = dict()        
                info[text_id]["url"] = "https://vk.com/" + domain + "?w=wall" + str(post["from_id"]) + "_" + str(post["id"]) 
                info[text_id]["time"] = post["date"]
                text_file_name = os.path.join(docs_folder, text_id)
                with open(text_file_name, "w", encoding = 'utf-8') as f:
                    f.write(text)
            if id == cut:
                break
        offset += 100
        print (offset)
        if len(posts) != 101:
            break 
        if id == cut:
            break
        
    with open(os.path.join(dataset_folder, "meta", "meta.json"), "wb") as f: 
        f.write(json.dumps(info).encode("UTF-8"))
    

     
if __name__ == "__main__":
    download_wall('miptstream', "D:\\visartm\\data\\datasets\\miptstream", cut=50)<commit_msg>tools<commit_after>import os
import json


def download_wall(domain, dataset_folder, cut=1000000):
	import vk
	session = vk.Session()
	api = vk.API(session)
	info = dict() 
	docs_folder = os.path.join(dataset_folder, "documents")
	os.makedirs(docs_folder, exist_ok=True)
	os.makedirs(os.path.join(dataset_folder, "meta"), exist_ok=True)
	
	
	id = 0
	offset = 0 
	while True:
		posts = api.wall.get(domain=domain, offset = offset, count=100)
		for i in range(1, len(posts)):
			post = posts[i]
			text = post["text"].replace("<br>","\n")
			if len(text)>50:
				id += 1
				text_id = "%06d.txt" % id
				info[text_id] = dict()		
				info[text_id]["url"] = "https://vk.com/" + domain + "?w=wall" + str(post["from_id"]) + "_" + str(post["id"]) 
				info[text_id]["time"] = post["date"]
				text_file_name = os.path.join(docs_folder, text_id)
				with open(text_file_name, "w", encoding = 'utf-8') as f:
					f.write(text)
			if id == cut:
				break
		offset += 100
		print (offset)
		if len(posts) != 101:
			break 
		if id == cut:
			break
		
	with open(os.path.join(dataset_folder, "meta", "meta.json"), "wb") as f: 
		f.write(json.dumps(info).encode("UTF-8"))
	

	 
if __name__ == "__main__":
	download_wall('miptstream', "D:\\visartm\\data\\datasets\\miptstream", cut=50)
{'fraction_non_alphanumeric': 0.16311080523055746, 'fraction_numerical': 0.038196834136269786, 'mean_word_length': 2.6156716417910446, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7353495', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1050, 'n_words': 204}