<commit_before>import os
import re
import tempfile
import requests
from urllib.parse import unquote

option_proxy = ''


def get_url(url, fn):
    if os.path.isfile(fn):
        os.remove(fn)

    if option_proxy:
        proxy_dict = { 'http': option_proxy, 'https': option_proxy, }
    else:
        proxy_dict = {}

    try:
        r = requests.get(url, proxies=proxy_dict, stream=True)
        with open(fn, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024): 
                if chunk: # filter out keep-alive new chunks
                    f.write(chunk)
                    #f.flush() commented by recommendation
    except Exception as e:
        print(e)
        

def get_plugin_zip(url):
    if not url: return
    fn = os.path.join(tempfile.gettempdir(), 'cudatext_addon.zip')
    get_url(url, fn)
    return fn


def get_channel_list(url):
    temp_fn = os.path.join(tempfile.gettempdir(), 'cuda_addons_dir.txt')
    get_url(url, temp_fn)
    if not os.path.isfile(temp_fn): return

    text = open(temp_fn, encoding='utf8').read()
    
    #regex has 3 groups: (..(type)..(name)..)
    RE = r'(http.+/(\w+)\.(.+?)\.zip)\b(.*)'
    
    res = re.findall(RE, text)
    res = [(r[0], r[1]+': '+unquote(r[2]), r[3]) for r in res]
    
    #print('debug:')
    #print(res)
    return res


def get_remote_addons_list(channels):
    res = []
    print('Read channels:')
    for s in channels: print('  '+s)
    for ch in channels:
        items = get_channel_list(ch)
        if items:
            res+=items
    return sorted(res)
<commit_msg>Addonman fix<commit_after>import os
import re
import tempfile
import requests
from urllib.parse import unquote

option_proxy = ''


def get_url(url, fn):
    if os.path.isfile(fn):
        os.remove(fn)

    if option_proxy:
        proxy_dict = { 'http': option_proxy, 'https': option_proxy, }
    else:
        proxy_dict = None

    try:
        r = requests.get(url, proxies=proxy_dict, stream=True)
        with open(fn, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024): 
                if chunk: # filter out keep-alive new chunks
                    f.write(chunk)
                    #f.flush() commented by recommendation
    except Exception as e:
        print(e)
        

def get_plugin_zip(url):
    if not url: return
    fn = os.path.join(tempfile.gettempdir(), 'cudatext_addon.zip')
    get_url(url, fn)
    return fn


def get_channel_list(url):
    temp_fn = os.path.join(tempfile.gettempdir(), 'cuda_addons_dir.txt')
    get_url(url, temp_fn)
    if not os.path.isfile(temp_fn): return

    text = open(temp_fn, encoding='utf8').read()
    
    #regex has 3 groups: (..(type)..(name)..)
    RE = r'(http.+/(\w+)\.(.+?)\.zip)\b(.*)'
    
    res = re.findall(RE, text)
    res = [(r[0], r[1]+': '+unquote(r[2]), r[3]) for r in res]
    
    #print('debug:')
    #print(res)
    return res


def get_remote_addons_list(channels):
    res = []
    print('Read channels:')
    for s in channels: print('  '+s)
    for ch in channels:
        items = get_channel_list(ch)
        if items:
            res+=items
    return sorted(res)

{'fraction_non_alphanumeric': 0.14062997136493796, 'fraction_numerical': 0.006363347120585428, 'mean_word_length': 2.358974358974359, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1668574', 'n_tokens_mistral': 1136, 'n_tokens_neox': 1089, 'n_words': 264}