<commit_before># --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

import argparse
import os
import re
from collections import OrderedDict
from datetime import datetime

from azure.cli.core._config import az_config
from azure.cli.core._profile import CLOUD
from azure.cli.core._util import CLIError
from azure.cli.core.commands.client_factory import get_mgmt_service_client
from azure.cli.core.commands.validators import validate_key_value_pairs
from azure.mgmt.storage import StorageManagementClient
from azure.mgmt.storage.models import CustomDomain
from azure.storage.blob import Include, PublicAccess
from azure.storage.blob.baseblobservice import BaseBlobService
from azure.storage.blob.models import ContentSettings as BlobContentSettings
from azure.storage.file import FileService
from azure.storage.file.models import ContentSettings as FileContentSettings
from azure.storage.models import ResourceTypes, Services
from azure.storage.table import TablePermissions, TablePayloadFormat

from ._factory import get_storage_data_service_client
from .util import glob_files_locally

storage_account_key_options = {'primary': 'key1', 'secondary': 'key2'}

# region PARAMETER VALIDATORS


def validate_accept(namespace):
    if namespace.accept:
        formats = {
            'none': TablePayloadFormat.JSON_NO_METADATA,
            'minimal': TablePayloadFormat.JSON_MINIMAL_METADATA,
            'full': TablePayloadFormat.JSON_FULL_METADATA
        }
        namespace.accept = formats[namespace.accept.lower()]


def validate_client_parameters(namespace):
    """ Retrieves storage connection parameters from environment variables and parses out
    connection string into account name and key """
    n = namespace

    if not n.connection_string:
        n.connection_string = az_config.get('storage', 'connection_string', None)

    # if connection string supplied or in environment variables, extract account key and name
    if n.connection_string:
        conn_dict = validate_key_value_pairs(n.connection_string)
        n.account_name = conn_dict['AccountName']
        n.account_key = conn_dict['AccountKey']

    # otherwise, simply try to retrieve the remaining variables from environment variables
    if not n.account_name:
        n.account_name = az_config.get('storage', 'account', None)
    if not n.account_key:
        n.account_key = az_config.get('storage', 'key', None)
    if not n.sas_token:
        n.sas_token = az_config.get('storage', 'sas_token', None)

    # if account name is specified but no key, attempt to query
    if n.account_name and not n.account_key:
        scf = get_mgmt_service_client(StorageManagementClient)
        acc = next((x for x in scf.storage_accounts.list() if x.name == n.account_name), None)
        if acc:
            from azure.cli.core.commands.arm import parse_resource_id
            rg = parse_resource_id(acc.id)['resource_group']
            n.account_key = \
                scf.storage_accounts.list_keys(rg, n.account_name).keys[0].value  # pylint: disable=no-member
        else:
            raise ValueError("Storage account '{}' not found.".format(n.account_name))


def validate_source_uri(namespace):
    usage_string = 'invalid usage: supply only one of the following argument sets:' + \
                   '\n\t   --source-uri' + \
                   '\n\tOR --source-container --source-blob [--source-snapshot] [--source-sas]' + \
                   '\n\tOR --source-share --source-path [--source-sas]'
    ns = vars(namespace)
    validate_client_parameters(namespace)  # must run first to resolve storage account
    storage_acc = ns.get('account_name', None) or az_config.get('storage', 'account', None)
    uri = ns.get('copy_source', None)
    container = ns.pop('source_container', None)
    blob = ns.pop('source_blob', None)
    sas = ns.pop('source_sas', None)
    snapshot = ns.pop('source_snapshot', None)
    share = ns.pop('source_share', None)
    path = ns.pop('source_path', None)<commit_msg>Fix #2004: not to query key when sas presents (#2063)<commit_after># --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

import argparse
import os
import re
from collections import OrderedDict
from datetime import datetime

from azure.cli.core._config import az_config
from azure.cli.core._profile import CLOUD
from azure.cli.core._util import CLIError
from azure.cli.core.commands.client_factory import get_mgmt_service_client
from azure.cli.core.commands.validators import validate_key_value_pairs
from azure.mgmt.storage import StorageManagementClient
from azure.mgmt.storage.models import CustomDomain
from azure.storage.blob import Include, PublicAccess
from azure.storage.blob.baseblobservice import BaseBlobService
from azure.storage.blob.models import ContentSettings as BlobContentSettings
from azure.storage.file import FileService
from azure.storage.file.models import ContentSettings as FileContentSettings
from azure.storage.models import ResourceTypes, Services
from azure.storage.table import TablePermissions, TablePayloadFormat

from ._factory import get_storage_data_service_client
from .util import glob_files_locally

storage_account_key_options = {'primary': 'key1', 'secondary': 'key2'}

# region PARAMETER VALIDATORS


def validate_accept(namespace):
    if namespace.accept:
        formats = {
            'none': TablePayloadFormat.JSON_NO_METADATA,
            'minimal': TablePayloadFormat.JSON_MINIMAL_METADATA,
            'full': TablePayloadFormat.JSON_FULL_METADATA
        }
        namespace.accept = formats[namespace.accept.lower()]


def validate_client_parameters(namespace):
    """ Retrieves storage connection parameters from environment variables and parses out
    connection string into account name and key """
    n = namespace

    if not n.connection_string:
        n.connection_string = az_config.get('storage', 'connection_string', None)

    # if connection string supplied or in environment variables, extract account key and name
    if n.connection_string:
        conn_dict = validate_key_value_pairs(n.connection_string)
        n.account_name = conn_dict['AccountName']
        n.account_key = conn_dict['AccountKey']

    # otherwise, simply try to retrieve the remaining variables from environment variables
    if not n.account_name:
        n.account_name = az_config.get('storage', 'account', None)
    if not n.account_key:
        n.account_key = az_config.get('storage', 'key', None)
    if not n.sas_token:
        n.sas_token = az_config.get('storage', 'sas_token', None)

    # strip the '?' from sas token. the portal and command line are returns sas token in different
    # forms
    if n.sas_token:
        n.sas_token = n.sas_token.lstrip('?')

    # if account name is specified but no key, attempt to query
    if n.account_name and not n.account_key and not n.sas_token:
        scf = get_mgmt_service_client(StorageManagementClient)
        acc = next((x for x in scf.storage_accounts.list() if x.name == n.account_name), None)
        if acc:
            from azure.cli.core.commands.arm import parse_resource_id
            rg = parse_resource_id(acc.id)['resource_group']
            n.account_key = \
                scf.storage_accounts.list_keys(rg, n.account_name).keys[0].value  # pylint: disable=no-member
        else:
            raise ValueError("Storage account '{}' not found.".format(n.account_name))


def validate_source_uri(namespace):
    usage_string = 'invalid usage: supply only one of the following argument sets:' + \
                   '\n\t   --source-uri' + \
                   '\n\tOR --source-container --source-blob [--source-snapshot] [--source-sas]' + \
                   '\n\tOR --source-share --source-path [--source-sas]'
    ns = vars(namespace)
    validate_client_parameters(namespace)  # must run first to resolve storage account
    storage_acc = ns.get('account_name', None) or az_config.get('storage', 'account', None)
    uri = ns.get('copy_source', None)
    container = ns.pop('source_container', None)
    blob = ns.pop('source_blob', None)
    sas = ns.pop('source_sas', None)
    snapshot = ns.pop('source_snapshot', None)
    share = ns.pop('source_share', None)
    path = ns.pop('source_path', None)
{'fraction_non_alphanumeric': 0.13067665758401453, 'fraction_numerical': 0.001589464123524069, 'mean_word_length': 3.0059117780809457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3440818', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2408, 'n_words': 668}