<commit_before>#!/usr/bin/env python
#coding=utf-8
import urllib
import sys

from termcolor import colored

from bs4 import BeautifulSoup

# get basic info
word = sys.argv[1]
base = "https://slovari.yandex.ru/"
tail = "{word}/%D0%BF%D0%B5%D1%80%D0%B5%D0%B2%D0%BE%D0%B4/".format(word=word)
url = base + tail

#fetch
response = urllib.urlopen(url)

# parse
soup = BeautifulSoup(urllib.urlopen(url), "lxml")
title = soup.find(class_="b-translation__title")

# word exists
if title:
    title_word = title.find(class_="b-translation__text")
    title_pronunciation = title.find(class_="b-translation__tr")

    groups = soup.select(".b-translation__group")

    # print
    def print_tag(tag, color=None, attrs=[]):
        if tag:
            text = tag.get_text().encode("utf-8")
            if color or attrs:
                print(colored(text, color=color, attrs=attrs))<commit_msg>Prettify url to cyrillic letters<commit_after>#!/usr/bin/env python
#coding=utf-8
import urllib
import sys

from termcolor import colored

from bs4 import BeautifulSoup

# get basic info
word = sys.argv[1]
base = "https://slovari.yandex.ru/"
tail = "{word}/перевод/".format(word=word)
url = base + tail

#fetch
response = urllib.urlopen(url)

# parse
soup = BeautifulSoup(urllib.urlopen(url), "lxml")
title = soup.find(class_="b-translation__title")

# word exists
if title:
    title_word = title.find(class_="b-translation__text")
    title_pronunciation = title.find(class_="b-translation__tr")

    groups = soup.select(".b-translation__group")

    # print
    def print_tag(tag, color=None, attrs=[]):
        if tag:
            text = tag.get_text().encode("utf-8")
            if color or attrs:
                print(colored(text, color=color, attrs=attrs))
{'fraction_non_alphanumeric': 0.1417142857142857, 'fraction_numerical': 0.012, 'mean_word_length': 3.797260273972603, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4448500', 'n_tokens_mistral': 633, 'n_tokens_neox': 623, 'n_words': 127}