<commit_before>#!/usr/bin/env python
# Copyright 2016 Scraper Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import os
import shutil
import tempfile
import unittest

import freezegun
import mock
import scraper


class TestScraper(unittest.TestCase):

    def test_file_locking(self):
        tempd = tempfile.mkdtemp()
        self.assertTrue(os.path.exists(tempd))
        lockfile = tempd + '/testlockfile'
        scraper.acquire_lock_or_die(lockfile)
        self.assertTrue(os.path.exists(lockfile))
        try:
            scraper.acquire_lock_or_die(lockfile)
            self.fail('Should have died before now')
        except AssertionError:
            pass
        os.remove(lockfile)
        scraper.acquire_lock_or_die(lockfile)
        os.remove(lockfile)
        self.assertFalse(os.path.exists(lockfile))
        os.rmdir(tempd)
        self.assertFalse(os.path.exists(tempd))

    def test_args(self):
        rsync_host = 'example.com'
        lockfile_dir = '/tmp/shouldnotexist/'
        rsync_module = 'ndt'
        data_dir = '/tmp/bigplaceforbackup'
        rsync_binary = '/usr/bin/rsync'
        spreadsheet = '1234567890abcdef'
        rsync_port = 1234
        max_uncompressed_size = 1024
        args = scraper.parse_cmdline([
            '--rsync_host', rsync_host, '--lockfile_dir', lockfile_dir,
            '--rsync_module', rsync_module, '--data_dir', data_dir,
            '--rsync_binary', rsync_binary, '--rsync_port', str(rsync_port),
            '--spreadsheet', spreadsheet, '--max_uncompressed_size',
            str(max_uncompressed_size)
        ])
        self.assertEqual(args.rsync_host, rsync_host)
        self.assertEqual(args.lockfile_dir, lockfile_dir)
        self.assertEqual(args.rsync_module, rsync_module)
        self.assertEqual(args.data_dir, data_dir)
        self.assertEqual(args.rsync_binary, rsync_binary)
        self.assertEqual(args.rsync_port, rsync_port)
        self.assertEqual(args.spreadsheet, spreadsheet)
        self.assertEqual(args.max_uncompressed_size, max_uncompressed_size)
        args = scraper.parse_cmdline([
            '--rsync_host', rsync_host, '--lockfile_dir', lockfile_dir,
            '--rsync_module', rsync_module, '--data_dir', data_dir,
            '--spreadsheet', spreadsheet])
        self.assertEqual(args.rsync_host, rsync_host)
        self.assertEqual(args.lockfile_dir, lockfile_dir)
        self.assertEqual(args.rsync_module, rsync_module)
        self.assertEqual(args.data_dir, data_dir)
        self.assertEqual(args.rsync_binary, rsync_binary)
        self.assertEqual(args.rsync_port, rsync_port)
        self.assertEqual(args.spreadsheet, spreadsheet)
        self.assertEqual(args.max_uncompressed_size, max_uncompressed_size)

    def test_args_help(self):
        with self.assertRaises(SystemExit):
            scraper.parse_cmdline(['-h'])

    @mock.patch('subprocess.check_output')
    def test_list_rsync_files(self, patched_subprocess):
        serverfiles = """\
drwxr-xr-x          4,096 2016/01/06 05:43:33 .
drwxr-xr-x          4,096 2016/10/01 00:06:59 2016
drwxr-xr-x          4,096 2016/01/15 01:03:29 2016/01
drwxr-xr-x          4,096 2016/01/06 22:32:01 2016/01/06
-rw-r--r--              0 2016/01/06 22:32:01 2016/01/06/.gz
-rw-r--r--            103 2016/01/06 05:43:36 2016/01/06/20160106T05:43:32.741066000Z_:0.cputime.gz
-rw-r--r--            716 2016/01/06 05:43:36 2016/01/06/20160106T05:43:32.741066000Z_:0.meta
-rw-r--r--            101 2016/01/06 18:07:37 2016/01/06/20160106T18:07:33.122784000Z_:0.cputime.gz<commit_msg>Cleaned up arg parsing unit test<commit_after>#!/usr/bin/env python
# Copyright 2016 Scraper Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import os
import shutil
import tempfile
import unittest

import freezegun
import mock
import scraper


class TestScraper(unittest.TestCase):

    def test_file_locking(self):
        tempd = tempfile.mkdtemp()
        self.assertTrue(os.path.exists(tempd))
        lockfile = tempd + '/testlockfile'
        scraper.acquire_lock_or_die(lockfile)
        self.assertTrue(os.path.exists(lockfile))
        try:
            scraper.acquire_lock_or_die(lockfile)
            self.fail('Should have died before now')
        except AssertionError:
            pass
        os.remove(lockfile)
        scraper.acquire_lock_or_die(lockfile)
        os.remove(lockfile)
        self.assertFalse(os.path.exists(lockfile))
        os.rmdir(tempd)
        self.assertFalse(os.path.exists(tempd))

    def test_args(self):
        rsync_host = 'example.com'
        lockfile_dir = '/tmp/shouldnotexist/'
        rsync_module = 'ndt'
        data_dir = '/tmp/bigplaceforbackup'
        rsync_binary = '/usr/bin/rsync'
        spreadsheet = '1234567890abcdef'
        rsync_port = 1234
        max_uncompressed_size = 1024
        args = scraper.parse_cmdline([
            '--rsync_host', rsync_host, '--lockfile_dir', lockfile_dir,
            '--rsync_module', rsync_module, '--data_dir', data_dir,
            '--rsync_binary', rsync_binary, '--rsync_port', str(rsync_port),
            '--spreadsheet', spreadsheet, '--max_uncompressed_size',
            str(max_uncompressed_size)
        ])
        self.assertEqual(args.rsync_host, rsync_host)
        self.assertEqual(args.lockfile_dir, lockfile_dir)
        self.assertEqual(args.rsync_module, rsync_module)
        self.assertEqual(args.data_dir, data_dir)
        self.assertEqual(args.rsync_binary, rsync_binary)
        self.assertEqual(args.rsync_port, rsync_port)
        self.assertEqual(args.spreadsheet, spreadsheet)
        self.assertEqual(args.max_uncompressed_size, max_uncompressed_size)
        args = scraper.parse_cmdline([
            '--rsync_host', rsync_host, '--lockfile_dir', lockfile_dir,
            '--rsync_module', rsync_module, '--data_dir', data_dir,
            '--spreadsheet', spreadsheet])
        self.assertEqual(args.rsync_binary, '/usr/bin/rsync')
        self.assertEqual(args.rsync_port, 7999)
        self.assertEqual(args.max_uncompressed_size, 1000000000)

    def test_args_help(self):
        with self.assertRaises(SystemExit):
            scraper.parse_cmdline(['-h'])

    @mock.patch('subprocess.check_output')
    def test_list_rsync_files(self, patched_subprocess):
        serverfiles = """\
drwxr-xr-x          4,096 2016/01/06 05:43:33 .
drwxr-xr-x          4,096 2016/10/01 00:06:59 2016
drwxr-xr-x          4,096 2016/01/15 01:03:29 2016/01
drwxr-xr-x          4,096 2016/01/06 22:32:01 2016/01/06
-rw-r--r--              0 2016/01/06 22:32:01 2016/01/06/.gz
-rw-r--r--            103 2016/01/06 05:43:36 2016/01/06/20160106T05:43:32.741066000Z_:0.cputime.gz
-rw-r--r--            716 2016/01/06 05:43:36 2016/01/06/20160106T05:43:32.741066000Z_:0.meta
-rw-r--r--            101 2016/01/06 18:07:37 2016/01/06/20160106T18:07:33.122784000Z_:0.cputime.gz
{'fraction_non_alphanumeric': 0.11156083986845435, 'fraction_numerical': 0.07412092081963066, 'mean_word_length': 2.803270803270803, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4353312', 'n_tokens_mistral': 3031, 'n_tokens_neox': 2611, 'n_words': 508}