<commit_before>'''
Miscellaneous shared code for spiderosm
'''

import os
import urllib
import sys
import time
import zipfile

# print immediately (no buffering)
def print_now(s, noNewLine=False):
    if noNewLine:
        print s,
    else:
        print s
    sys.stdout.flush()

def date_ymd():
    return time.strftime("%Y%m%d")         

def date_ymdhms():
    return time.strftime("%Y%m%d_%H%M%S")  

def update_file_from_url(filename,url):
    if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename))

    # get current file size
    file_size = None
    if os.path.exists(filename):
        file_size = os.stat(filename).st_size

    # get size of url content
    url_size = None
    f = urllib.urlopen(url)
    i = f.info()
    url_size = int(i.getheader('content-length'))
    f.close()

    print 'file_size', file_size
    print 'url_size', url_size
    if file_size != url_size:
        print 'Downloading: %s to %s' % (url,filename)
        urllib.urlretrieve(url,filename)

def unzip(Filename):
    zfile = zipfile.ZipFile(Filename)
    (zipDir,zipFile) = os.path.split(Filename)
    for name in zfile.namelist():
        (subDir, filename) = os.path.split(name)
        dirname = os.path.join(zipDir,subDir)
        if dirname=='': dirname='.'
        print "Decompressing " + filename + " on " + dirname
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        zfile.extract(name, dirname)

def test():
    print 'date_ymd:', date_ymd()
    print 'date_ymdhms:', date_ymdhms()

    print_now('a',noNewLine=True)
    time.sleep(2)
    print_now('b',noNewLine=True)
    time.sleep(2)
    print_now('c')
    time.sleep(5)
    print_now('d')

if __name__ == '__main__':
    test()
<commit_msg>added check for '..' in directories in unzip for security.<commit_after>'''
Miscellaneous shared code for spiderosm
'''

import os
import urllib
import sys
import time
import zipfile

# print immediately (no buffering)
def print_now(s, noNewLine=False):
    if noNewLine:
        print s,
    else:
        print s
    sys.stdout.flush()

def date_ymd():
    return time.strftime("%Y%m%d")         

def date_ymdhms():
    return time.strftime("%Y%m%d_%H%M%S")  

def update_file_from_url(filename,url):
    if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename))

    # get current file size
    file_size = None
    if os.path.exists(filename):
        file_size = os.stat(filename).st_size

    # get size of url content
    url_size = None
    f = urllib.urlopen(url)
    i = f.info()
    url_size = int(i.getheader('content-length'))
    f.close()

    print 'file_size', file_size
    print 'url_size', url_size
    if file_size != url_size:
        print 'Downloading: %s to %s' % (url,filename)
        urllib.urlretrieve(url,filename)

def unzip(Filename):
    zfile = zipfile.ZipFile(Filename)
    (zipDir,zipFile) = os.path.split(Filename)
    for name in zfile.namelist():
        (subDir, filename) = os.path.split(name)
        dirname = os.path.join(zipDir,subDir)
        # paranoid about escaping from current dir with '..'
        assert not '..' in dirname
        assert not '//' in dirname
        assert not '\\' in dirname
        if dirname=='': dirname='.'
        print "Decompressing " + filename + " on " + dirname
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        zfile.extract(name, dirname)

def test():
    print 'date_ymd:', date_ymd()
    print 'date_ymdhms:', date_ymdhms()

    print_now('a',noNewLine=True)
    time.sleep(2)
    print_now('b',noNewLine=True)
    time.sleep(2)
    print_now('c')
    time.sleep(5)
    print_now('d')

if __name__ == '__main__':
    test()

{'fraction_non_alphanumeric': 0.13279785809906292, 'fraction_numerical': 0.001606425702811245, 'mean_word_length': 2.9038662486938347, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1801054', 'n_tokens_mistral': 1334, 'n_tokens_neox': 1305, 'n_words': 293}