<commit_before># !/usr/bin/env python
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Ambari Agent

"""
import \
  ambari_simplejson as json  # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
import grp
import os
import pwd
import re
import time
from urlparse import urlparse
from resource_management.core import shell
from resource_management.core import sudo
from resource_management.core.base import Fail
from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
from resource_management.core.providers import Provider
from resource_management.core.resources.system import Execute
from resource_management.core.resources.system import File
from resource_management.libraries.functions import format
from resource_management.libraries.functions import namenode_ha_utils
from resource_management.libraries.functions.get_user_call_output import get_user_call_output
from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs

JSON_PATH = '/var/lib/ambari-agent/tmp/hdfs_resources_{timestamp}.json'
JAR_PATH = '/var/lib/ambari-agent/lib/fast-hdfs-resource.jar'

RESOURCE_TO_JSON_FIELDS = {
  'target': 'target',
  'type': 'type',
  'action': 'action',
  'source': 'source',
  'owner': 'owner',
  'group': 'group',
  'mode': 'mode',
  'recursive_chown': 'recursiveChown',
  'recursive_chmod': 'recursiveChmod',
  'change_permissions_for_parents': 'changePermissionforParents',
  'manage_if_exists': 'manageIfExists',
  'dfs_type': 'dfs_type'
}

EXCEPTIONS_TO_RETRY = {
  # ("ExceptionName"): ("required text fragment", try_count, try_sleep_seconds)

  # Happens when multiple nodes try to put same file at the same time.
  # Needs a longer retry time, to wait for other nodes success.
  "FileNotFoundException": (" does not have any open files", 6, 30),

  "LeaseExpiredException": ("", 20, 6),
  "RetriableException": ("", 20, 6),
}

class HdfsResourceJar:
  """
  This is slower than HdfsResourceWebHDFS implementation of HdfsResouce, but it works in any cases on any DFS types.
  
  The idea is to put all the files/directories/copyFromLocals/copyToLocals we have to create/delete into a json file.
  And then perform them with ONLY ONE expensive hadoop call to our custom jar fast-hdfs-resource.jar which grabs this json.
  
  'create_and_execute', 'delete_on_execute' and "download_on_execute do nothing except add actions to this json,
  while execute does all the expensive creating/deleting work executing the jar with the json as parameter.
  """
  def action_delayed(self, action_name, main_resource):
    dfs_type = main_resource.resource.dfs_type

    if main_resource.resource.nameservices is None: # all nameservices
      nameservices = namenode_ha_utils.get_nameservices(main_resource.resource.hdfs_site)
    else:
      nameservices = main_resource.resource.nameservices

    # non-federated cluster
    if not nameservices:
      self.action_delayed_for_nameservice(None, action_name, main_resource)
    else:
      default_fs_protocol = urlparse(main_resource.resource.default_fs).scheme

      if not default_fs_protocol or default_fs_protocol == "viewfs":
        protocol = dfs_type.lower()
      else:
        protocol = default_fs_protocol

      for nameservice in nameservices:
        try:
          nameservice = protocol + "://" + nameservice
          self.action_delayed_for_nameservice(nameservice, action_name, main_resource)
        except namenode_ha_utils.NoActiveNamenodeException as ex:
          # one of ns can be down (during initial start forexample) no need to worry for federated cluster<commit_msg>AMBARI-24623. hdfsResource fails to using nameservices for filesystems which does not support that (aonishuk)<commit_after># !/usr/bin/env python
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Ambari Agent

"""
import \
  ambari_simplejson as json  # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
import grp
import os
import pwd
import re
import time
from urlparse import urlparse
from resource_management.core import shell
from resource_management.core import sudo
from resource_management.core.base import Fail
from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
from resource_management.core.providers import Provider
from resource_management.core.resources.system import Execute
from resource_management.core.resources.system import File
from resource_management.libraries.functions import format
from resource_management.libraries.functions import namenode_ha_utils
from resource_management.libraries.functions.get_user_call_output import get_user_call_output
from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs

JSON_PATH = '/var/lib/ambari-agent/tmp/hdfs_resources_{timestamp}.json'
JAR_PATH = '/var/lib/ambari-agent/lib/fast-hdfs-resource.jar'

RESOURCE_TO_JSON_FIELDS = {
  'target': 'target',
  'type': 'type',
  'action': 'action',
  'source': 'source',
  'owner': 'owner',
  'group': 'group',
  'mode': 'mode',
  'recursive_chown': 'recursiveChown',
  'recursive_chmod': 'recursiveChmod',
  'change_permissions_for_parents': 'changePermissionforParents',
  'manage_if_exists': 'manageIfExists',
  'dfs_type': 'dfs_type'
}

EXCEPTIONS_TO_RETRY = {
  # ("ExceptionName"): ("required text fragment", try_count, try_sleep_seconds)

  # Happens when multiple nodes try to put same file at the same time.
  # Needs a longer retry time, to wait for other nodes success.
  "FileNotFoundException": (" does not have any open files", 6, 30),

  "LeaseExpiredException": ("", 20, 6),
  "RetriableException": ("", 20, 6),
}

class HdfsResourceJar:
  """
  This is slower than HdfsResourceWebHDFS implementation of HdfsResouce, but it works in any cases on any DFS types.
  
  The idea is to put all the files/directories/copyFromLocals/copyToLocals we have to create/delete into a json file.
  And then perform them with ONLY ONE expensive hadoop call to our custom jar fast-hdfs-resource.jar which grabs this json.
  
  'create_and_execute', 'delete_on_execute' and "download_on_execute do nothing except add actions to this json,
  while execute does all the expensive creating/deleting work executing the jar with the json as parameter.
  """
  def action_delayed(self, action_name, main_resource):
    dfs_type = main_resource.resource.dfs_type

    if main_resource.resource.nameservices is None: # all nameservices
      nameservices = namenode_ha_utils.get_nameservices(main_resource.resource.hdfs_site)
    else:
      nameservices = main_resource.resource.nameservices

    # non-federated cluster
    if not nameservices or len(nameservices) < 2:
      self.action_delayed_for_nameservice(None, action_name, main_resource)
    else:
      default_fs_protocol = urlparse(main_resource.resource.default_fs).scheme

      if not default_fs_protocol or default_fs_protocol == "viewfs":
        protocol = dfs_type.lower()
      else:
        protocol = default_fs_protocol

      for nameservice in nameservices:
        try:
          nameservice = protocol + "://" + nameservice
          self.action_delayed_for_nameservice(nameservice, action_name, main_resource)
        except namenode_ha_utils.NoActiveNamenodeException as ex:
          # one of ns can be down (during initial start forexample) no need to worry for federated cluster
{'fraction_non_alphanumeric': 0.07167773151794757, 'fraction_numerical': 0.004050860807921683, 'mean_word_length': 4.72680412371134, 'pattern_counts': {'":': 10, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '537968', 'n_tokens_mistral': 2574, 'n_tokens_neox': 2504, 'n_words': 871}