<commit_before>
"""
test_client
----------------------------------

Tests for `alpenhorn.client` module.
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import pytest
from click.testing import CliRunner
import os
import re
from io import StringIO

try:
    from unittest.mock import patch, call
except ImportError:
    from mock import patch, call


import alpenhorn.db as db
import alpenhorn.client as cli
import alpenhorn.acquisition as ac
import alpenhorn.archive as ar
import alpenhorn.storage as st

import test_import as ti


@pytest.fixture
def fixtures(tmpdir):
    return ti.fixtures(tmpdir)


@pytest.fixture(autouse=True)
def no_cli_init(monkeypatch):
    monkeypatch.setattr(cli, '_init_config_db', lambda: None )


def test_import_schema(fixtures):
    assert set(db.database_proxy.get_tables()) == {
        u'storagegroup', u'storagenode',
        u'acqtype', u'archiveacq',
        u'filetype', u'archivefile',
        u'archivefilecopyrequest', u'archivefilecopy',
        u'zabinfo', u'quuxinfo', u'zxcinfo', u'spqrinfo', u'loginfo'
    }
    groups = set(st.StorageGroup.select(st.StorageGroup.name).tuples())
    assert groups == { ( 'foo', ), ( 'bar', ), ( 'transport', ) }
    assert st.StorageGroup.get(st.StorageGroup.name == 'bar').notes == 'Some bar!'


def test_command_line_interface(fixtures):
    """Check the output when no commands are invoked"""
    runner = CliRunner()
    result = runner.invoke(cli.cli)
    assert result.exit_code == 0
    assert 'Client interface for alpenhorn' in result.output


def test_sync(fixtures):
    """Test the 'sync' command"""
    runner = CliRunner()

    help_result = runner.invoke(cli.sync, ['--help'])
    assert help_result.exit_code == 0
    assert 'Copy all files from NODE to GROUP that are not already present.' in help_result.output
    assert 'Options:\n  --acq ACQ              Sync only this acquisition.' in help_result.output

    result = runner.invoke(cli.sync, args=['x', 'bar'])
    assert result.exit_code == 0
    assert result.output == 'No files to copy from node x.\n'

    # now pretend node 'x' has a copy of 'fred', 1 GB in size
    file_copy = (ar.ArchiveFileCopy
                 .select()
                 .join(ac.ArchiveFile)
                 .where(ac.ArchiveFile.name == 'fred')
                 .get())
    file_copy.has_file = 'Y'
    file_copy.file.size_b = 1073741824.0
    file_copy.save()
    file_copy.file.save()

    result = runner.invoke(cli.sync, args=['--force', '--show_acq', '--show_files', 'x', 'bar'])
    assert result.exit_code == 0
    assert re.match(r'x \[1 files\]\n' +
                    r'x/fred\n' +
                    r'Will request that 1 files \(1\.0 GB\) be copied from node x to group bar\.\n' +
                    r'Updating 0 existing requests and inserting 1 new ones\.\n$',
                    result.output, re.DOTALL)

    ## by default, the cleaned copy should be marked as 'maybe wanted'
    copy_request = (ar.ArchiveFileCopyRequest
                    .select()
                    .join(ac.ArchiveFile)
                    .where(ac.ArchiveFile.name == 'fred')).get()
    assert copy_request.node_from.name == 'x'
    assert copy_request.group_to.name == 'bar'
    assert not copy_request.completed
    assert not copy_request.cancelled
    assert copy_request.n_requests == 1

    ## if we run sync again, the copy request will simply get the 'n_requests' count incremented by 1
    result = runner.invoke(cli.sync, args=['--force', '--show_acq', '--show_files', 'x', 'bar'])

    print(result.output)
    print(result.exception)

    assert result.exit_code == 0
    assert re.match(r'x \[1 files\]\n' +
                    r'x/fred\n' +
                    r'Will request that 1 files \(1\.0 GB\) be copied from node x to group bar\.\n' +
                    r'Updating 1 existing requests and inserting 0 new ones\.\n$',
                    result.output, re.DOTALL)

    ## by default, the cleaned copy should be marked as 'maybe wanted'
    copy_request = (ar.ArchiveFileCopyRequest
                    .select()
                    .join(ac.ArchiveFile)
                    .where(ac.ArchiveFile.name == 'fred')).get()
    assert copy_request.node_from.name == 'x'
    assert copy_request.group_to.name == 'bar'
    assert not copy_request.completed
    assert not copy_request.cancelled
    assert copy_request.n_requests == 2


def test_status(fixtures):
    """Test the output of the 'status' command"""
    runner = CliRunner()

    help_result = runner.invoke(cli.status, ['--help'])
    assert help_result.exit_code == 0
    assert 'Summarise the status of alpenhorn storage nodes.' in help_result.output
    assert 'Options:\n  --all   Show the status of all nodes,' in help_result.output

    # we start off with no good file copies, so the output should only contain
    # the header
    result = runner.invoke(cli.status, args=['--all'])
    assert result.exit_code == 0
    output = result.output.splitlines()
    assert len(output) == 2
    assert re.match(r'^Node\s+Files\s+Size \[TB\]\s+', output[0])
    assert re.match(r'^[- ]+$', output[1])

    # now pretend node 'x' has a copy of 'fred'
    file_copy = (ar.ArchiveFileCopy
                 .select()
                 .join(ac.ArchiveFile)
                 .where(ac.ArchiveFile.name == 'fred')
                 .get())
    file_copy.has_file = 'Y'
    file_copy.file.size_b = 1
    file_copy.save()
    file_copy.file.save()

    result = runner.invoke(cli.status, args=['--all'])
    assert result.exit_code == 0
    output = result.output.splitlines()
    assert len(output) == 3
    assert re.match(r'x\s+1\s+0\.0\s+33\.3\s+100\.0\s+foo.example.com:', output[2])


def test_verify(fixtures):
    """Test the output of the 'verify' command"""
    tmpdir = fixtures['root']

    runner = CliRunner()
    result = runner.invoke(cli.verify, ['x'])
    assert result.exit_code == 0
    assert re.match(r'.*\n=== Summary ===\n' +
                    '  0 total files\n' +
                    '  0 missing files\n' +
                    '  0 corrupt files',
                    result.output, re.DOTALL)

    # now pretend node 'x' has a copy of 'fred'
    file_copy = (ar.ArchiveFileCopy
                 .select()
                 .join(ac.ArchiveFile)
                 .where(ac.ArchiveFile.name == 'fred')
                 .get())
    file_copy.has_file = 'Y'
    file_copy.save()
    result = runner.invoke(cli.verify, ['x'])
    assert result.exit_code == 2
    assert re.match(r'.*\n=== Missing files ===\n' +
                    str(tmpdir.join(file_copy.file.acq.name, file_copy.file.name)),
                    result.output, re.DOTALL)
    assert re.match(r'.*\n=== Summary ===\n' +
                    '  1 total files\n' +
                    '  1 missing files\n' +
                    '  0 corrupt files',
                    result.output, re.DOTALL)


def test_clean(fixtures):
    """Test the 'clean' command"""
    runner = CliRunner()

    help_result = runner.invoke(cli.clean, ['--help'])
    assert help_result.exit_code == 0
    assert 'Clean up NODE by marking older files as potentially removable.' in help_result.output
    assert 'Options:\n  -d, --days INTEGER     Clean files older than <days>.' in help_result.output

    ## pretend 'fred' is 1 GB in size
    f = ac.ArchiveFile.get(ac.ArchiveFile.name == 'fred')
    f.size_b = 1073741824.0
    f.save()
<commit_msg>Add a test for #35<commit_after>
"""
test_client
----------------------------------

Tests for `alpenhorn.client` module.
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import pytest
from click.testing import CliRunner
import os
import re
from io import StringIO

try:
    from unittest.mock import patch, call
except ImportError:
    from mock import patch, call


import alpenhorn.db as db
import alpenhorn.client as cli
import alpenhorn.acquisition as ac
import alpenhorn.archive as ar
import alpenhorn.storage as st

import test_import as ti


@pytest.fixture
def fixtures(tmpdir):
    return ti.fixtures(tmpdir)


@pytest.fixture(autouse=True)
def no_cli_init(monkeypatch):
    monkeypatch.setattr(cli, '_init_config_db', lambda: None )


def test_import_schema(fixtures):
    assert set(db.database_proxy.get_tables()) == {
        u'storagegroup', u'storagenode',
        u'acqtype', u'archiveacq',
        u'filetype', u'archivefile',
        u'archivefilecopyrequest', u'archivefilecopy',
        u'zabinfo', u'quuxinfo', u'zxcinfo', u'spqrinfo', u'loginfo'
    }
    groups = set(st.StorageGroup.select(st.StorageGroup.name).tuples())
    assert groups == { ( 'foo', ), ( 'bar', ), ( 'transport', ) }
    assert st.StorageGroup.get(st.StorageGroup.name == 'bar').notes == 'Some bar!'


def test_command_line_interface(fixtures):
    """Check the output when no commands are invoked"""
    runner = CliRunner()
    result = runner.invoke(cli.cli)
    assert result.exit_code == 0
    assert 'Client interface for alpenhorn' in result.output


def test_sync(fixtures):
    """Test the 'sync' command"""
    runner = CliRunner()

    help_result = runner.invoke(cli.sync, ['--help'])
    assert help_result.exit_code == 0
    assert 'Copy all files from NODE to GROUP that are not already present.' in help_result.output
    assert 'Options:\n  --acq ACQ              Sync only this acquisition.' in help_result.output

    result = runner.invoke(cli.sync, args=['x', 'bar'])
    assert result.exit_code == 0
    assert result.output == 'No files to copy from node x.\n'

    # now pretend node 'x' has a copy of 'fred', 1 GB in size
    file_copy = (ar.ArchiveFileCopy
                 .select()
                 .join(ac.ArchiveFile)
                 .where(ac.ArchiveFile.name == 'fred')
                 .get())
    file_copy.has_file = 'Y'
    file_copy.file.size_b = 1073741824.0
    file_copy.save()
    file_copy.file.save()

    result = runner.invoke(cli.sync, args=['--force', '--show_acq', '--show_files', 'x', 'bar'])
    assert result.exit_code == 0
    assert re.match(r'x \[1 files\]\n' +
                    r'x/fred\n' +
                    r'Will request that 1 files \(1\.0 GB\) be copied from node x to group bar\.\n' +
                    r'Updating 0 existing requests and inserting 1 new ones\.\n$',
                    result.output, re.DOTALL)

    ## by default, the cleaned copy should be marked as 'maybe wanted'
    copy_request = (ar.ArchiveFileCopyRequest
                    .select()
                    .join(ac.ArchiveFile)
                    .where(ac.ArchiveFile.name == 'fred')).get()
    assert copy_request.node_from.name == 'x'
    assert copy_request.group_to.name == 'bar'
    assert not copy_request.completed
    assert not copy_request.cancelled
    assert copy_request.n_requests == 1

    ## if we run sync again, the copy request will simply get the 'n_requests' count incremented by 1
    result = runner.invoke(cli.sync, args=['--force', '--show_acq', '--show_files', 'x', 'bar'])

    print(result.output)
    print(result.exception)

    assert result.exit_code == 0
    assert re.match(r'x \[1 files\]\n' +
                    r'x/fred\n' +
                    r'Will request that 1 files \(1\.0 GB\) be copied from node x to group bar\.\n' +
                    r'Updating 1 existing requests and inserting 0 new ones\.\n$',
                    result.output, re.DOTALL)

    ## by default, the cleaned copy should be marked as 'maybe wanted'
    copy_request = (ar.ArchiveFileCopyRequest
                    .select()
                    .join(ac.ArchiveFile)
                    .where(ac.ArchiveFile.name == 'fred')).get()
    assert copy_request.node_from.name == 'x'
    assert copy_request.group_to.name == 'bar'
    assert not copy_request.completed
    assert not copy_request.cancelled
    assert copy_request.n_requests == 2


def test_status(fixtures):
    """Test the output of the 'status' command"""
    runner = CliRunner()

    help_result = runner.invoke(cli.status, ['--help'])
    assert help_result.exit_code == 0
    assert 'Summarise the status of alpenhorn storage nodes.' in help_result.output
    assert 'Options:\n  --all   Show the status of all nodes,' in help_result.output

    # we start off with no good file copies, so the output should only contain
    # the header
    result = runner.invoke(cli.status, args=['--all'])
    assert result.exit_code == 0
    output = result.output.splitlines()
    assert len(output) == 2
    assert re.match(r'^Node\s+Files\s+Size \[TB\]\s+', output[0])
    assert re.match(r'^[- ]+$', output[1])

    # now pretend node 'x' has a copy of 'fred'
    file_copy = (ar.ArchiveFileCopy
                 .select()
                 .join(ac.ArchiveFile)
                 .where(ac.ArchiveFile.name == 'fred')
                 .get())
    file_copy.has_file = 'Y'
    file_copy.file.size_b = 1
    file_copy.save()
    file_copy.file.save()

    result = runner.invoke(cli.status, args=['--all'])
    assert result.exit_code == 0
    output = result.output.splitlines()
    assert len(output) == 3
    assert re.match(r'x\s+1\s+0\.0\s+33\.3\s+100\.0\s+foo.example.com:', output[2])


def test_verify(fixtures):
    """Test the output of the 'verify' command"""
    tmpdir = fixtures['root']

    runner = CliRunner()
    result = runner.invoke(cli.verify, ['x'])
    assert result.exit_code == 0
    assert re.match(r'.*\n=== Summary ===\n' +
                    '  0 total files\n' +
                    '  0 missing files\n' +
                    '  0 corrupt files',
                    result.output, re.DOTALL)

    # now pretend node 'x' has a copy of 'fred'
    file_copy = (ar.ArchiveFileCopy
                 .select()
                 .join(ac.ArchiveFile)
                 .where(ac.ArchiveFile.name == 'fred')
                 .get())
    file_copy.has_file = 'Y'
    file_copy.save()
    result = runner.invoke(cli.verify, ['x'])
    assert result.exit_code == 2
    assert re.match(r'.*\n=== Missing files ===\n' +
                    str(tmpdir.join(file_copy.file.acq.name, file_copy.file.name)),
                    result.output, re.DOTALL)
    assert re.match(r'.*\n=== Summary ===\n' +
                    '  1 total files\n' +
                    '  1 missing files\n' +
                    '  0 corrupt files',
                    result.output, re.DOTALL)

    ## now add a known file ('fred')
    tmpdir.join('x', 'fred').write('')
    result = runner.invoke(cli.verify, ['--md5', 'x'])
    assert result.exit_code == 1
    assert re.match(r'.*\n=== Corrupt files ===\n' +
                    '/.*/ROOT/x/fred\n'
                    '.*\n=== Summary ===\n' +
                    '  1 total files\n' +
                    '  0 missing files\n' +
                    '  1 corrupt files',
                    result.output, re.DOTALL)


def test_clean(fixtures):
    """Test the 'clean' command"""
    runner = CliRunner()

    help_result = runner.invoke(cli.clean, ['--help'])
    assert help_result.exit_code == 0
    assert 'Clean up NODE by marking older files as potentially removable.' in help_result.output
    assert 'Options:\n  -d, --days INTEGER     Clean files older than <days>.' in help_result.output

    ## pretend 'fred' is 1 GB in size
    f = ac.ArchiveFile.get(ac.ArchiveFile.name == 'fred')
    f.size_b = 1073741824.0
    f.save()

{'fraction_non_alphanumeric': 0.13370437483820866, 'fraction_numerical': 0.009772197773750971, 'mean_word_length': 2.3033347584437793, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6499105', 'n_tokens_mistral': 4985, 'n_tokens_neox': 4706, 'n_words': 1285}