<commit_before>import os
import re
import time
import urllib.request
import tempfile
from cudatext import *

MINUTES = 3 #download if list is older
V_REG = 'Registry'
V_REG_VER = 'RegistryVersions'
TEMP = tempfile.gettempdir()

def get_url(url, fn):
    if os.path.isfile(fn):
        os.remove(fn)
    try:
        urllib.request.urlretrieve(url, fn)
    except:
        pass
        
def file_aged(fn):
    if os.path.isfile(fn):
        age = int(time.time() - os.stat(fn).st_mtime)
        return age > MINUTES*60
    else:
        return True

def get_wiki_text(name):
    REGEX = "text=\w?('.*?'\s)"
    url = 'http://sourceforge.net/p/synwrite/wiki/%s/raw' % name
    fn = os.path.join(TEMP, 'cudatext_%s.txt' % name)
    #print('Downloading wiki: '+url)
    if file_aged(fn):
        get_url(url, fn)
    if not os.path.isfile(fn): 
        msg_box('Cannot download wiki', MB_OK)
    s = open(fn, 'r').read()
    r = re.compile(REGEX, re.S)
    if not r: return ValueError
    rr = r.search(s)
    if not rr: raise ValueError 
    s = rr.group(1)
    s = s.replace('\n', '')
    s = s.replace('  ', ' ')
    s = eval(s)
    return s

def get_item_url(item):
    try:
        url = 'http://sourceforge.net/projects/synwrite-addons/files/' + item + '/download'
        res = urllib.request.urlopen(url)
        return res.geturl()
    except:
        return

def get_avail_list():
    msg_status('Wait for list')
    text = get_wiki_text(V_REG)
    msg_status('')
    if not text:
        return "CantGetList"
    
    items = text.split('\r\n')
    
    #remove leading wiki text
    last = False
    for i in range(len(items)):
        if items[0].startswith('--'):
            last = True
        del items[0]
        if last: break

    #make list of lists, item[3] is empty
    items = sorted(items, key=str.lower) #case-insensitive
    res = []
    for s in items:
        if '|' in s:
            props = s.split('|')
            props = [l.strip() for l in props]
            props = [props[0], props[1], props[2], '']
            res += [props]

    #write version to item[3] for each item
    msg_status('Wait for ver') 
    text = get_wiki_text(V_REG_VER)
    msg_status('')
    if text:
        items = text.split('\r\n')
        for item in items:
            verinfo = item.split('=')
            if len(verinfo)==2:
                for r in res:
                    if r[0] == verinfo[0]:
                        r[3] = verinfo[1]
                        break
<commit_msg>py plugman<commit_after>import os
import re
import time
import urllib.request
import tempfile
from cudatext import *

MINUTES = 3 #download if list is older
V_REG = 'Registry'
V_REG_VER = 'RegistryVersions'
TEMP = tempfile.gettempdir()

def get_url(url, fn):
    if os.path.isfile(fn):
        os.remove(fn)
    try:
        urllib.request.urlretrieve(url, fn)
    except:
        pass
        
def file_aged(fn):
    if os.path.isfile(fn):
        age = int(time.time() - os.stat(fn).st_mtime)
        return age > MINUTES*60
    else:
        return True

def get_wiki_text(name):
    REGEX = "text=\w?('.*?'\s)"
    url = 'http://sourceforge.net/p/synwrite/wiki/%s/raw' % name
    fn = os.path.join(TEMP, 'cudatext_%s.txt' % name)
    #print('Downloading wiki: '+url)
    if file_aged(fn):
        get_url(url, fn)
    if not os.path.isfile(fn): 
        msg_box('Cannot download wiki', MB_OK)
        return
    s = open(fn, 'r').read()
    r = re.compile(REGEX, re.S)
    if not r: return ValueError
    rr = r.search(s)
    if not rr: raise ValueError 
    s = rr.group(1)
    s = s.replace('\n', '')
    s = s.replace('  ', ' ')
    s = eval(s)
    return s

def get_item_url(item):
    try:
        url = 'http://sourceforge.net/projects/synwrite-addons/files/' + item + '/download'
        res = urllib.request.urlopen(url)
        return res.geturl()
    except:
        return

def get_avail_list():
    msg_status('Wait for list')
    text = get_wiki_text(V_REG)
    msg_status('')
    if not text:
        return "Cannot download list"
    
    items = text.split('\r\n')
    
    #remove leading wiki text
    last = False
    for i in range(len(items)):
        if items[0].startswith('--'):
            last = True
        del items[0]
        if last: break

    #make list of lists, item[3] is empty
    items = sorted(items, key=str.lower) #case-insensitive
    res = []
    for s in items:
        if '|' in s:
            props = s.split('|')
            props = [l.strip() for l in props]
            props = [props[0], props[1], props[2], '']
            res += [props]

    #write version to item[3] for each item
    msg_status('Wait for version list') 
    text = get_wiki_text(V_REG_VER)
    msg_status('')
    if text:
        items = text.split('\r\n')
        for item in items:
            verinfo = item.split('=')
            if len(verinfo)==2:
                for r in res:
                    if r[0] == verinfo[0]:
                        r[3] = verinfo[1]
                        break

{'fraction_non_alphanumeric': 0.1379035472299721, 'fraction_numerical': 0.006377042646472699, 'mean_word_length': 1.9733412322274881, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4803721', 'n_tokens_mistral': 1774, 'n_tokens_neox': 1751, 'n_words': 442}