<commit_before>#!/usr/bin/python3
import time
import pycurl
import os
import errno
import json
import sys
import getopt
import filecmp
import certifi
from urllib.parse import urlencode
from replace_post_tex import replace_dollar_tex
from replace_post_tex import replace_display_tex
from replace_post_tex import replace_inline_tex
from slimit import ast
from slimit.parser import Parser
from io import BytesIO
from bs4 import BeautifulSoup

root_url = "https://artofproblemsolving.com"
vt100_BLUE = '\033[94m'
vt100_WARNING = '\033[93m'
vt100_RESET = '\033[0m'
DIVISIONS = 500

# load parser globally for sharing, it is painfully slow
slimit_parser = Parser()

def print_err(err_str):
    f = open("error.log", "a")
    print(vt100_WARNING)
    err_str = "[error] " + err_str
    f.write(err_str + '\n')
    print(err_str)
    print(vt100_RESET)
    f.close()

def curl(sub_url, c, post = None):
    buf = BytesIO()
    print('[curl] %s' % sub_url)
    url = root_url + sub_url
    url = url.encode('iso-8859-1')
    c.setopt(c.URL, url)
    c.setopt(c.WRITEFUNCTION, buf.write)
    #c.setopt(c.VERBOSE, True)
    if post is not None:
        c.setopt(c.POST, 1)
        c.setopt(c.POSTFIELDS, urlencode(post))
    errs = 0
    while True:
        try:
            c.perform()
        except (KeyboardInterrupt, SystemExit):
            print('user aborting...')
            raise
        except:
            errs = errs + 1
            if errs > 3:
                buf.close()
                raise
            print("[curl] try again...")
            continue
        break
    res_str = buf.getvalue()
    buf.close()
    return res_str

def parse_op_name(obj):
    if isinstance(obj, ast.DotAccessor):
        l = obj.node.value + "." + obj.identifier.value
    elif isinstance(obj, ast.String):
        l = obj.value
        l = l.strip('"')
        l = l.encode().decode('unicode_escape')
    elif hasattr(obj, 'value'):
        l = obj.value
    else:
        l = "<UnknownName>"
    return l<commit_msg>fixed issue with striping too much of quotes, could later cause issue with escaping the characters properly<commit_after>#!/usr/bin/python3
import time
import pycurl
import os
import errno
import json
import sys
import getopt
import filecmp
import certifi
from urllib.parse import urlencode
from replace_post_tex import replace_dollar_tex
from replace_post_tex import replace_display_tex
from replace_post_tex import replace_inline_tex
from slimit import ast
from slimit.parser import Parser
from io import BytesIO
from bs4 import BeautifulSoup

root_url = "https://artofproblemsolving.com"
vt100_BLUE = '\033[94m'
vt100_WARNING = '\033[93m'
vt100_RESET = '\033[0m'
DIVISIONS = 500

# load parser globally for sharing, it is painfully slow
slimit_parser = Parser()

def print_err(err_str):
    f = open("error.log", "a")
    print(vt100_WARNING)
    err_str = "[error] " + err_str
    f.write(err_str + '\n')
    print(err_str)
    print(vt100_RESET)
    f.close()

def curl(sub_url, c, post = None):
    buf = BytesIO()
    print('[curl] %s' % sub_url)
    url = root_url + sub_url
    url = url.encode('iso-8859-1')
    c.setopt(c.URL, url)
    c.setopt(c.WRITEFUNCTION, buf.write)
    #c.setopt(c.VERBOSE, True)
    if post is not None:
        c.setopt(c.POST, 1)
        c.setopt(c.POSTFIELDS, urlencode(post))
    errs = 0
    while True:
        try:
            c.perform()
        except (KeyboardInterrupt, SystemExit):
            print('user aborting...')
            raise
        except:
            errs = errs + 1
            if errs > 3:
                buf.close()
                raise
            print("[curl] try again...")
            continue
        break
    res_str = buf.getvalue()
    buf.close()
    return res_str

def parse_op_name(obj):
    if isinstance(obj, ast.DotAccessor):
        l = obj.node.value + "." + obj.identifier.value
    elif isinstance(obj, ast.String):
        l = obj.value
        # no using strip here, because it can remove more than 1
        # instance of quotes, which is not desired and can cause issues
        if l.startswith('"') and l.endswith('"'):
            l = l[1:-1]
        l = l.encode().decode('unicode_escape')
    elif hasattr(obj, 'value'):
        l = obj.value
    else:
        l = "<UnknownName>"
    return l
{'fraction_non_alphanumeric': 0.10837209302325582, 'fraction_numerical': 0.02069767441860465, 'mean_word_length': 2.5225225225225225, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7479329', 'n_tokens_mistral': 1493, 'n_tokens_neox': 1407, 'n_words': 389}