<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.hive.hcatalog.templeton.tool;

import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Tool;
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.templeton.AppConfig;
import org.apache.hive.hcatalog.templeton.SecureProxySupport;
import org.apache.hive.hcatalog.templeton.UgiFactory;
import org.apache.thrift.TException;

/**
 * A Map Reduce job that will start another job.
 *
 * We have a single Mapper job that starts a child MR job.  The parent
 * monitors the child child job and ends when the child job exits.  In
 * addition, we
 *
 * - write out the parent job id so the caller can record it.
 * - run a keep alive thread so the job doesn't end.
 * - Optionally, store the stdout, stderr, and exit value of the child
 *   in hdfs files.
 *
 * A note on security.  When jobs are submitted through WebHCat that use HCatalog, it means that
 * metastore access is required.  Hive queries, of course, need metastore access.  This in turn
 * requires delegation token to be obtained for metastore in a <em>secure cluster</em>.  Since we
 * can't usually parse the job to find out if it is using metastore, we require 'usehcatalog'
 * parameter supplied in the REST call.  WebHcat takes care of cancelling the token when the job
 * is complete.
 */
@InterfaceAudience.Private
public class TempletonControllerJob extends Configured implements Tool, JobSubmissionConstants {
  private static final Log LOG = LogFactory.getLog(TempletonControllerJob.class);
  private final boolean secureMetastoreAccess;
  private final AppConfig appConf;

  /**
   * @param secureMetastoreAccess - if true, a delegation token will be created
   *                              and added to the job
   */
  public TempletonControllerJob(boolean secureMetastoreAccess, AppConfig conf) {
    super();
    this.secureMetastoreAccess = secureMetastoreAccess;
    this.appConf = conf;
  }

  @Override
  public Configuration getConf() {
    return appConf;
  }

  private JobID submittedJobId;

  public String getSubmittedId() {
    if (submittedJobId == null) {
      return null;
    } else {
      return submittedJobId.toString();
    }
  }
<commit_msg>HIVE-10564 : webhcat should use webhcat-site.xml properties for controller job submission HIVE-10564.2.patch (<NAME>, reviewed by <NAME>)<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.hive.hcatalog.templeton.tool;

import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Tool;
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.templeton.AppConfig;
import org.apache.hive.hcatalog.templeton.SecureProxySupport;
import org.apache.hive.hcatalog.templeton.UgiFactory;
import org.apache.thrift.TException;

/**
 * A Map Reduce job that will start another job.
 *
 * We have a single Mapper job that starts a child MR job.  The parent
 * monitors the child child job and ends when the child job exits.  In
 * addition, we
 *
 * - write out the parent job id so the caller can record it.
 * - run a keep alive thread so the job doesn't end.
 * - Optionally, store the stdout, stderr, and exit value of the child
 *   in hdfs files.
 *
 * A note on security.  When jobs are submitted through WebHCat that use HCatalog, it means that
 * metastore access is required.  Hive queries, of course, need metastore access.  This in turn
 * requires delegation token to be obtained for metastore in a <em>secure cluster</em>.  Since we
 * can't usually parse the job to find out if it is using metastore, we require 'usehcatalog'
 * parameter supplied in the REST call.  WebHcat takes care of cancelling the token when the job
 * is complete.
 */
@InterfaceAudience.Private
public class TempletonControllerJob extends Configured implements Tool, JobSubmissionConstants {
  private static final Log LOG = LogFactory.getLog(TempletonControllerJob.class);
  private final boolean secureMetastoreAccess;
  private final AppConfig appConf;

  /**
   * @param secureMetastoreAccess - if true, a delegation token will be created
   *                              and added to the job
   */
  public TempletonControllerJob(boolean secureMetastoreAccess, AppConfig conf) {
    super(new Configuration(conf));
    this.secureMetastoreAccess = secureMetastoreAccess;
    this.appConf = conf;
  }

  private JobID submittedJobId;

  public String getSubmittedId() {
    if (submittedJobId == null) {
      return null;
    } else {
      return submittedJobId.toString();
    }
  }

{'fraction_non_alphanumeric': 0.07924290220820189, 'fraction_numerical': 0.0023974763406940064, 'mean_word_length': 4.16015625, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3442291', 'n_tokens_mistral': 2236, 'n_tokens_neox': 2201, 'n_words': 783}