<commit_before>import tempfile, os, os.path, gzip, urllib, itertools, optparse, pkg_resources, sys
import pdb, subprocess, shutil
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
from xml.etree import ElementTree as ET

from django.core.management.base import NoArgsCommand
from pywurfl import wurflprocessor

class Command(NoArgsCommand):
    option_list = NoArgsCommand.option_list
    help = "Updates wurfl data"

    requires_model_validation = False

    WURFL_URL = 'http://kent.dl.sourceforge.net/project/wurfl/WURFL/latest/wurfl-latest.xml.gz'
    WEB_PATCH_URL = 'http://wurfl.sourceforge.net/web_browsers_patch.xml'

    def handle_noargs(self, **options):
        tempdir = tempfile.mkdtemp()
        
        final_filename = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..', 'wurfl_data.py'))
        local_filename = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..', 'data', 'local_patch.xml'))
        
        try:
            wurfl_gz = urllib.urlopen(Command.WURFL_URL)
            wurfl_f = gzip.GzipFile(fileobj=StringIO(wurfl_gz.read()))
            wurfl_filename = os.path.join(tempdir, 'wurfl.xml')

            web_patch_f = urllib.urlopen(Command.WEB_PATCH_URL)
            local_patch_f = open(local_filename, 'r')
            
            wurfl = ET.parse(wurfl_f)
            web_patch = ET.parse(web_patch_f)
            local_patch = ET.parse(local_patch_f)
            
            devices = {}
            devices_et = itertools.chain(
                wurfl.findall('devices/device'),
                web_patch.findall('devices/device'),
                local_patch.findall('devices/device'),<commit_msg>MOLLY-236: Use Wurfl 2.1, which is released under an acceptable license<commit_after>import tempfile, os, os.path, gzip, urllib, itertools, optparse, pkg_resources, sys
import pdb, subprocess, shutil
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
from xml.etree import ElementTree as ET

from django.core.management.base import NoArgsCommand
from pywurfl import wurflprocessor

class Command(NoArgsCommand):
    option_list = NoArgsCommand.option_list
    help = "Updates wurfl data"

    requires_model_validation = False

    WURFL_URL = 'http://kent.dl.sourceforge.net/project/wurfl/WURFL/2.1.1/wurfl-2.1.1.xml.gz'
    WEB_PATCH_URL = 'http://kent.dl.sourceforge.net/project/wurfl/WURFL/2.1.1/web_browsers_patch.xml'

    def handle_noargs(self, **options):
        tempdir = tempfile.mkdtemp()
        
        final_filename = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..', 'wurfl_data.py'))
        local_filename = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..', 'data', 'local_patch.xml'))
        
        try:
            wurfl_gz = urllib.urlopen(Command.WURFL_URL)
            wurfl_f = gzip.GzipFile(fileobj=StringIO(wurfl_gz.read()))
            wurfl_filename = os.path.join(tempdir, 'wurfl.xml')

            web_patch_f = urllib.urlopen(Command.WEB_PATCH_URL)
            local_patch_f = open(local_filename, 'r')
            
            wurfl = ET.parse(wurfl_f)
            web_patch = ET.parse(web_patch_f)
            local_patch = ET.parse(local_patch_f)
            
            devices = {}
            devices_et = itertools.chain(
                wurfl.findall('devices/device'),
                web_patch.findall('devices/device'),
                local_patch.findall('devices/device'),
{'fraction_non_alphanumeric': 0.11662883087400681, 'fraction_numerical': 0.003972758229284903, 'mean_word_length': 2.726215644820296, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3099492', 'n_tokens_mistral': 1195, 'n_tokens_neox': 1184, 'n_words': 189}