<commit_before>#!/usr/bin/python

"""nrvr.util.download - A download manager

Class provided by this module is Download.

Works in Linux and Windows.

Idea and first implementation - <NAME> <srguiwiz12 AT nrvr DOT com>

Public repository - https://github.com/srguiwiz/nrvr-commander

Copyright (c) Nirvana Research 2006-2013.
Modified BSD License"""

import os.path
import posixpath
import shutil
import sys
import urllib2
import urlparse

from nrvr.util.user import ScriptUser

class Download(object):
    """A download manager."""

    semaphoreExtenstion = ".wait"

    @classmethod
    def basename(cls, url):
        """Base name from a dowload URL.
        
        Implemented for the purpose of all code using and relying on this
        one implementation only."""
        urlParseResult = urlparse.urlparse(url)
        baseName = posixpath.basename(urlParseResult.path)
        return baseName

    @classmethod
    def fromUrl(cls, url, ticker=True):
        """Download file or use previously downloaded file.
        
        As implemented uses urllib2.
        
        Return file path."""
        urlFilename = cls.basename(url)
        downloadDir = ScriptUser.loggedIn.userHomeRelative("Downloads")
        downloadPath = os.path.join(downloadDir, urlFilename)
        semaphorePath = downloadPath + cls.semaphoreExtenstion
        #
        if not os.path.exists(downloadDir):
            try:
                os.makedirs(downloadDir)
            except OSError:
                if os.path.exists(downloadDir): # concurrently made
                    pass
                else: # failure
                    raise
        #
        if os.path.exists(downloadPath):
            if not os.path.exists(semaphorePath):
                # file exists and not download in progress,
                # assume it is good
                return downloadPath
            else:
                # file exists and download in progress,
                # presumably from another script running in another process or thread,<commit_msg>Make Downloads directory if needed<commit_after>#!/usr/bin/python

"""nrvr.util.download - A download manager

Class provided by this module is Download.

Works in Linux and Windows.

Idea and first implementation - <NAME> <srguiwiz12 AT nrvr DOT com>

Public repository - https://github.com/srguiwiz/nrvr-commander

Copyright (c) Nirvana Research 2006-2013.
Modified BSD License"""

import os
import os.path
import posixpath
import shutil
import sys
import urllib2
import urlparse

from nrvr.util.user import ScriptUser

class Download(object):
    """A download manager."""

    semaphoreExtenstion = ".wait"

    @classmethod
    def basename(cls, url):
        """Base name from a dowload URL.
        
        Implemented for the purpose of all code using and relying on this
        one implementation only."""
        urlParseResult = urlparse.urlparse(url)
        baseName = posixpath.basename(urlParseResult.path)
        return baseName

    @classmethod
    def fromUrl(cls, url, ticker=True):
        """Download file or use previously downloaded file.
        
        As implemented uses urllib2.
        
        Return file path."""
        urlFilename = cls.basename(url)
        downloadDir = ScriptUser.loggedIn.userHomeRelative("Downloads")
        if not os.path.exists(downloadDir): # possibly on an international version OS
            os.mkdir(downloadDir)
        downloadPath = os.path.join(downloadDir, urlFilename)
        semaphorePath = downloadPath + cls.semaphoreExtenstion
        #
        if not os.path.exists(downloadDir):
            try:
                os.makedirs(downloadDir)
            except OSError:
                if os.path.exists(downloadDir): # concurrently made
                    pass
                else: # failure
                    raise
        #
        if os.path.exists(downloadPath):
            if not os.path.exists(semaphorePath):
                # file exists and not download in progress,
                # assume it is good
                return downloadPath
            else:
                # file exists and download in progress,
                # presumably from another script running in another process or thread,
{'fraction_non_alphanumeric': 0.07255967856298748, 'fraction_numerical': 0.0056724178681162845, 'mean_word_length': 2.1558538404175986, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 41, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6479914', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1115, 'n_words': 357}