<commit_before># -*- coding: utf-8 -*-
import os
import mimetypes
import shutil
import zipfile

from django.conf import settings
from django.core.cache import cache
from django.core.files.storage import default_storage as storage
from django import forms

import pytest
import flufl.lock
from mock import Mock, patch
from freezegun import freeze_time

from olympia import amo
from olympia.amo.tests import TestCase
from olympia.amo.urlresolvers import reverse
from olympia.files.helpers import FileViewer, DiffHelper, extract_file
from olympia.files.models import File
from olympia.files.utils import SafeUnzip, get_all_files

root = os.path.join(settings.ROOT, 'src/olympia/files/fixtures/files')


def get_file(filename):
    return os.path.join(root, filename)


def make_file(pk, file_path, **kwargs):
    obj = Mock()
    obj.id = obj.pk = pk
    for k, v in kwargs.items():
        setattr(obj, k, v)
    obj.file_path = file_path
    obj.current_file_path = file_path
    obj.__str__ = lambda x: x.pk
    obj.version = Mock()
    obj.version.version = 1
    return obj


class TestFileViewer(TestCase):

    def setUp(self):
        super(TestFileViewer, self).setUp()
        self.viewer = FileViewer(make_file(1, get_file('dictionary-test.xpi')))

    def tearDown(self):
        self.viewer.cleanup()
        super(TestFileViewer, self).tearDown()

    def test_files_not_extracted(self):
        assert not self.viewer.is_extracted()

    def test_files_extracted(self):
        self.viewer.extract()
        assert self.viewer.is_extracted()

    def test_recurse_extract(self):
        self.viewer.src = get_file('recurse.xpi')
        self.viewer.extract()
        assert self.viewer.is_extracted()

    def test_recurse_contents(self):
        self.viewer.src = get_file('recurse.xpi')
        self.viewer.extract()
        files = self.viewer.get_files()
        file_list = [
            'recurse/recurse.xpi/chrome/test-root.txt',
            'recurse/somejar.jar/recurse/recurse.xpi/chrome/test.jar',
            'recurse/somejar.jar/recurse/recurse.xpi/chrome/test.jar/test'
        ]
        for name in file_list:
            assert name in files

    def test_recurse_contents_of_zip(self):
        self.viewer.src = get_file('recurse.zip')
        self.viewer.extract()
        files = self.viewer.get_files()
        file_list = [
            'recurse/recurse.xpi/chrome/test-root.txt',
            'recurse/somejar.jar/recurse/recurse.xpi/chrome/test.jar',
            'recurse/somejar.jar/recurse/recurse.xpi/chrome/test.jar/test'
        ]
        for name in file_list:
            assert name in files

    def test_locked(self):
        self.viewer.src = get_file('dictionary-test.xpi')

        # Lock was successfully attained
        assert self.viewer.extract()

        lock = flufl.lock.Lock(os.path.join(
            settings.TMP_PATH, 'file-viewer-%s.lock' % self.viewer.file.pk
        ))

        assert not lock.is_locked

        lock.lock()

        assert lock.is_locked

        # Not extracting, the viewer is locked, lock could not be attained
        assert not self.viewer.extract()

    def test_extract_file_locked_message(self):
        self.viewer.src = get_file('dictionary-test.xpi')
        assert not self.viewer.is_extracted()

        lock = flufl.lock.Lock(os.path.join(
            settings.TMP_PATH, 'file-viewer-%s.lock' % self.viewer.file.pk
        ))

        assert not lock.is_locked

        lock.lock()

        assert lock.is_locked

        msg = extract_file(self.viewer)
        assert str(msg.get()).startswith(u'File viewer is locked')
        msg.delete()

    def test_cleanup(self):
        self.viewer.extract()
        self.viewer.cleanup()
        assert not self.viewer.is_extracted()

    @freeze_time('2017-02-08 02:01:00')
    def test_dest(self):
        assert self.viewer.dest == os.path.join(
            settings.TMP_PATH, 'file_viewer',
            '0208', str(self.viewer.file.pk))

    def test_isbinary(self):
        binary = self.viewer._is_binary
        for f in ['foo.rdf', 'foo.xml', 'foo.js', 'foo.py'
                  'foo.html', 'foo.txt', 'foo.dtd', 'foo.xul', 'foo.sh',
                  'foo.properties', 'foo.json', 'foo.src', 'CHANGELOG']:
            m, encoding = mimetypes.guess_type(f)
            assert not binary(m, f), '%s should not be binary' % f

        for f in ['foo.png', 'foo.gif', 'foo.exe', 'foo.swf']:
            m, encoding = mimetypes.guess_type(f)
            assert binary(m, f), '%s should be binary' % f

        filename = os.path.join(settings.TMP_PATH, 'test_isbinary')
        for txt in ['#!/usr/bin/python', '#python', u'\0x2']:<commit_msg>Fix test dependency on what date it's being run. (#4645)<commit_after># -*- coding: utf-8 -*-
import os
import mimetypes
import shutil
import zipfile

from django.conf import settings
from django.core.cache import cache
from django.core.files.storage import default_storage as storage
from django import forms

import pytest
import flufl.lock
from mock import Mock, patch
from freezegun import freeze_time

from olympia import amo
from olympia.amo.tests import TestCase
from olympia.amo.urlresolvers import reverse
from olympia.files.helpers import FileViewer, DiffHelper, extract_file
from olympia.files.models import File
from olympia.files.utils import SafeUnzip, get_all_files

root = os.path.join(settings.ROOT, 'src/olympia/files/fixtures/files')


def get_file(filename):
    return os.path.join(root, filename)


def make_file(pk, file_path, **kwargs):
    obj = Mock()
    obj.id = obj.pk = pk
    for k, v in kwargs.items():
        setattr(obj, k, v)
    obj.file_path = file_path
    obj.current_file_path = file_path
    obj.__str__ = lambda x: x.pk
    obj.version = Mock()
    obj.version.version = 1
    return obj


class TestFileViewer(TestCase):

    def setUp(self):
        super(TestFileViewer, self).setUp()
        self.viewer = FileViewer(make_file(1, get_file('dictionary-test.xpi')))

    def tearDown(self):
        self.viewer.cleanup()
        super(TestFileViewer, self).tearDown()

    def test_files_not_extracted(self):
        assert not self.viewer.is_extracted()

    def test_files_extracted(self):
        self.viewer.extract()
        assert self.viewer.is_extracted()

    def test_recurse_extract(self):
        self.viewer.src = get_file('recurse.xpi')
        self.viewer.extract()
        assert self.viewer.is_extracted()

    def test_recurse_contents(self):
        self.viewer.src = get_file('recurse.xpi')
        self.viewer.extract()
        files = self.viewer.get_files()
        file_list = [
            'recurse/recurse.xpi/chrome/test-root.txt',
            'recurse/somejar.jar/recurse/recurse.xpi/chrome/test.jar',
            'recurse/somejar.jar/recurse/recurse.xpi/chrome/test.jar/test'
        ]
        for name in file_list:
            assert name in files

    def test_recurse_contents_of_zip(self):
        self.viewer.src = get_file('recurse.zip')
        self.viewer.extract()
        files = self.viewer.get_files()
        file_list = [
            'recurse/recurse.xpi/chrome/test-root.txt',
            'recurse/somejar.jar/recurse/recurse.xpi/chrome/test.jar',
            'recurse/somejar.jar/recurse/recurse.xpi/chrome/test.jar/test'
        ]
        for name in file_list:
            assert name in files

    def test_locked(self):
        self.viewer.src = get_file('dictionary-test.xpi')

        # Lock was successfully attained
        assert self.viewer.extract()

        lock = flufl.lock.Lock(os.path.join(
            settings.TMP_PATH, 'file-viewer-%s.lock' % self.viewer.file.pk
        ))

        assert not lock.is_locked

        lock.lock()

        assert lock.is_locked

        # Not extracting, the viewer is locked, lock could not be attained
        assert not self.viewer.extract()

    def test_extract_file_locked_message(self):
        self.viewer.src = get_file('dictionary-test.xpi')
        assert not self.viewer.is_extracted()

        lock = flufl.lock.Lock(os.path.join(
            settings.TMP_PATH, 'file-viewer-%s.lock' % self.viewer.file.pk
        ))

        assert not lock.is_locked

        lock.lock()

        assert lock.is_locked

        msg = extract_file(self.viewer)
        assert str(msg.get()).startswith(u'File viewer is locked')
        msg.delete()

    def test_cleanup(self):
        self.viewer.extract()
        self.viewer.cleanup()
        assert not self.viewer.is_extracted()

    @freeze_time('2017-01-08 02:01:00')
    def test_dest(self):
        viewer = FileViewer(make_file(1, get_file('webextension.xpi')))
        assert viewer.dest == os.path.join(
            settings.TMP_PATH, 'file_viewer',
            '0108', str(self.viewer.file.pk))

    def test_isbinary(self):
        binary = self.viewer._is_binary
        for f in ['foo.rdf', 'foo.xml', 'foo.js', 'foo.py'
                  'foo.html', 'foo.txt', 'foo.dtd', 'foo.xul', 'foo.sh',
                  'foo.properties', 'foo.json', 'foo.src', 'CHANGELOG']:
            m, encoding = mimetypes.guess_type(f)
            assert not binary(m, f), '%s should not be binary' % f

        for f in ['foo.png', 'foo.gif', 'foo.exe', 'foo.swf']:
            m, encoding = mimetypes.guess_type(f)
            assert binary(m, f), '%s should be binary' % f

        filename = os.path.join(settings.TMP_PATH, 'test_isbinary')
        for txt in ['#!/usr/bin/python', '#python', u'\0x2']:
{'fraction_non_alphanumeric': 0.12040945546644154, 'fraction_numerical': 0.005382017728999578, 'mean_word_length': 2.513904338153504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6188121', 'n_tokens_mistral': 3223, 'n_tokens_neox': 3126, 'n_words': 643}