<commit_before>import os
import re
import tempfile
import requests
from urllib.parse import unquote
import cudatext as app
from . import opt


def get_url(url, fn):
    fn_temp = fn+'.download'
    if os.path.isfile(fn_temp):
        os.remove(fn_temp)

    if opt.proxy:
        proxies = { 'http': opt.proxy, 'https': opt.proxy, }
    else:
        proxies = None
    #print('proxy', proxies)

    while True:
        try:
            r = requests.get(url, proxies=proxies, stream=True)
            with open(fn_temp, 'wb') as f:
                for chunk in r.iter_content(chunk_size=1024):
                    if chunk: # filter out keep-alive new chunks
                        f.write(chunk)
                        #f.flush() commented by recommendation

            if os.path.isfile(fn_temp):
                os.rename(fn_temp, fn)
            return

        except Exception as e:
            res = app.msg_box('Cannot download:\n%s\n%s\n\nRetry?' % (url, str(e)),<commit_msg>Addonman: fix for get_url<commit_after>import os
import re
import tempfile
import requests
from urllib.parse import unquote
import cudatext as app
from . import opt


def get_url(url, fn):
    fn_temp = fn+'.download'
    if os.path.isfile(fn_temp):
        os.remove(fn_temp)

    if opt.proxy:
        proxies = { 'http': opt.proxy, 'https': opt.proxy, }
    else:
        proxies = None
    #print('proxy', proxies)

    while True:
        try:
            r = requests.get(url, proxies=proxies, stream=True)
            with open(fn_temp, 'wb') as f:
                for chunk in r.iter_content(chunk_size=1024):
                    if chunk: # filter out keep-alive new chunks
                        f.write(chunk)
                        #f.flush() commented by recommendation

            if os.path.isfile(fn_temp):
                if os.path.isfile(fn):
                    os.remove(fn)
                os.rename(fn_temp, fn)
            return

        except Exception as e:
            res = app.msg_box('Cannot download:\n%s\n%s\n\nRetry?' % (url, str(e)),
{'fraction_non_alphanumeric': 0.10733365711510442, 'fraction_numerical': 0.003885381253035454, 'mean_word_length': 1.821917808219178, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1347212', 'n_tokens_mistral': 667, 'n_tokens_neox': 638, 'n_words': 167}