def _update_spark_env(self):
    try:
        node_num = re.search(u'node(\\d+)', self.name).group(1)
    except AttributeError:
        node_num = 0
    conf_file = os.path.join(self.get_path(), 'resources', 'spark', 'conf', 'spark-env.sh')
    env = self.get_env()
    content = []
    with open(conf_file, 'r') as f:
        for line in f.readlines():
            for spark_var in env.keys():
                if (line.startswith(('export %s=' % spark_var)) or line.startswith(('export %s=' % spark_var), 2)):
                    line = ('export %s=%s\n' % (spark_var, env[spark_var]))
                    break
            content.append(line)
    with open(conf_file, 'w') as f:
        f.writelines(content)
    if (self.cluster.version() > '5.0'):
        print_('Writing shuffle')
        defaults_file = os.path.join(self.get_path(), 'resources', 'spark', 'conf', 'spark-defaults.conf')
        with open(defaults_file, 'a') as f:
            port_num = (7737 + int(node_num))
            f.write(('\nspark.shuffle.service.port %s\n' % port_num))
def _update_spark_env(self):
    try:
        node_num = re.search(u'node(\\d+)', self.name).group(1)
    except AttributeError:
        node_num = 0
    conf_file = os.path.join(self.get_path(), 'resources', 'spark', 'conf', 'spark-env.sh')
    env = self.get_env()
    content = []
    with open(conf_file, 'r') as f:
        for line in f.readlines():
            for spark_var in env.keys():
                if (line.startswith(('export %s=' % spark_var)) or line.startswith(('export %s=' % spark_var), 2)):
                    line = ('export %s=%s\n' % (spark_var, env[spark_var]))
                    break
            content.append(line)
    with open(conf_file, 'w') as f:
        f.writelines(content)
    if (self.cluster.version() > '5.0'):
        print_('Writing shuffle')
        defaults_file = os.path.join(self.get_path(), 'resources', 'spark', 'conf', 'spark-defaults.conf')
        with open(defaults_file, 'a') as f:
            port_num = (7737 + int(node_num))
            f.write(('\nspark.shuffle.service.port %s\n' % port_num))