<commit_before>#
# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
# (the "License"). You may not use this work except in compliance with the License, which is
# available at www.apache.org/licenses/LICENSE-2.0
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied, as more fully set forth in the License.
#
# See the NOTICE file distributed with this work for information regarding copyright ownership.
#

# May get overridden by System Property

log4j.rootLogger=INFO, ${alluxio.logger.type}, ${alluxio.remote.logger.type}

log4j.category.alluxio.logserver=INFO, ${alluxio.logserver.logger.type}
log4j.additivity.alluxio.logserver=false

log4j.logger.AUDIT_LOG=INFO, ${alluxio.master.audit.logger.type}
log4j.logger.JOB_MASTER_AUDIT_LOG=INFO, ${alluxio.job.master.audit.logger.type}
log4j.additivity.AUDIT_LOG=false

# Configures an appender whose name is "" (empty string) to be NullAppender.
# By default, if a Java class does not specify a particular appender, log4j will
# use "" as the appender name, then it will use Null appender.
log4j.appender.=org.apache.log4j.varia.NullAppender

log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.Target=System.out
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# The ParquetWriter logs for every row group which is not noisy for large row group size,
# but very noisy for small row group size.
log4j.logger.org.apache.parquet.hadoop.InternalParquetRecordWriter=WARN
log4j.logger.org.apache.parquet.hadoop.InternalParquetRecordReader=WARN

# Appender for Job Master
log4j.appender.JOB_MASTER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.JOB_MASTER_LOGGER.File=${alluxio.logs.dir}/job_master.log
log4j.appender.JOB_MASTER_LOGGER.MaxFileSize=10MB
log4j.appender.JOB_MASTER_LOGGER.MaxBackupIndex=100
log4j.appender.JOB_MASTER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.JOB_MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n

# Appender for Job Workers
log4j.appender.JOB_WORKER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.JOB_WORKER_LOGGER.File=${alluxio.logs.dir}/job_worker.log
log4j.appender.JOB_WORKER_LOGGER.MaxFileSize=10MB
log4j.appender.JOB_WORKER_LOGGER.MaxBackupIndex=100
log4j.appender.JOB_WORKER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.JOB_WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n

# Appender for Master
log4j.appender.MASTER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.MASTER_LOGGER.File=${alluxio.logs.dir}/master.log
log4j.appender.MASTER_LOGGER.MaxFileSize=10MB
log4j.appender.MASTER_LOGGER.MaxBackupIndex=100
log4j.appender.MASTER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Master
log4j.appender.SECONDARY_MASTER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.SECONDARY_MASTER_LOGGER.File=${alluxio.logs.dir}/secondary_master.log
log4j.appender.SECONDARY_MASTER_LOGGER.MaxFileSize=10MB
log4j.appender.SECONDARY_MASTER_LOGGER.MaxBackupIndex=100
log4j.appender.SECONDARY_MASTER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.SECONDARY_MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Master audit
log4j.appender.MASTER_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.MASTER_AUDIT_LOGGER.File=${alluxio.logs.dir}/master_audit.log
log4j.appender.MASTER_AUDIT_LOGGER.MaxFileSize=10MB
log4j.appender.MASTER_AUDIT_LOGGER.MaxBackupIndex=100
log4j.appender.MASTER_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n

# Appender for Job Master audit
log4j.appender.JOB_MASTER_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.JOB_MASTER_AUDIT_LOGGER.File=${alluxio.logs.dir}/job_master_audit.log
log4j.appender.JOB_MASTER_AUDIT_LOGGER.MaxFileSize=10MB
log4j.appender.JOB_MASTER_AUDIT_LOGGER.MaxBackupIndex=100
log4j.appender.JOB_MASTER_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.JOB_MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n

# Appender for Hub Agent
log4j.appender.HUB_AGENT_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.HUB_AGENT_LOGGER.File=${alluxio.logs.dir}/hub_agent.log
log4j.appender.HUB_AGENT_LOGGER.MaxFileSize=10MB
log4j.appender.HUB_AGENT_LOGGER.MaxBackupIndex=100
log4j.appender.HUB_AGENT_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.HUB_AGENT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Hub Manager
log4j.appender.HUB_MANAGER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.HUB_MANAGER_LOGGER.File=${alluxio.logs.dir}/hub_manager.log
log4j.appender.HUB_MANAGER_LOGGER.MaxFileSize=10MB
log4j.appender.HUB_MANAGER_LOGGER.MaxBackupIndex=100
log4j.appender.HUB_MANAGER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.HUB_MANAGER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n<commit_msg>Update log4j configuration for performance<commit_after>#
# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
# (the "License"). You may not use this work except in compliance with the License, which is
# available at www.apache.org/licenses/LICENSE-2.0
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied, as more fully set forth in the License.
#
# See the NOTICE file distributed with this work for information regarding copyright ownership.
#

# May get overridden by System Property

log4j.rootLogger=INFO, ${alluxio.logger.type}, ${alluxio.remote.logger.type}

log4j.category.alluxio.logserver=INFO, ${alluxio.logserver.logger.type}
log4j.additivity.alluxio.logserver=false

log4j.logger.AUDIT_LOG=INFO, ${alluxio.master.audit.logger.type}
log4j.logger.JOB_MASTER_AUDIT_LOG=INFO, ${alluxio.job.master.audit.logger.type}
log4j.additivity.AUDIT_LOG=false

# Configures an appender whose name is "" (empty string) to be NullAppender.
# By default, if a Java class does not specify a particular appender, log4j will
# use "" as the appender name, then it will use Null appender.
log4j.appender.=org.apache.log4j.varia.NullAppender

log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.Target=System.out
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# The ParquetWriter logs for every row group which is not noisy for large row group size,
# but very noisy for small row group size.
log4j.logger.org.apache.parquet.hadoop.InternalParquetRecordWriter=WARN
log4j.logger.org.apache.parquet.hadoop.InternalParquetRecordReader=WARN

# Appender for Job Master
log4j.appender.JOB_MASTER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.JOB_MASTER_LOGGER.File=${alluxio.logs.dir}/job_master.log
log4j.appender.JOB_MASTER_LOGGER.MaxFileSize=10MB
log4j.appender.JOB_MASTER_LOGGER.MaxBackupIndex=100
log4j.appender.JOB_MASTER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.JOB_MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Job Workers
log4j.appender.JOB_WORKER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.JOB_WORKER_LOGGER.File=${alluxio.logs.dir}/job_worker.log
log4j.appender.JOB_WORKER_LOGGER.MaxFileSize=10MB
log4j.appender.JOB_WORKER_LOGGER.MaxBackupIndex=100
log4j.appender.JOB_WORKER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.JOB_WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Master
log4j.appender.MASTER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.MASTER_LOGGER.File=${alluxio.logs.dir}/master.log
log4j.appender.MASTER_LOGGER.MaxFileSize=10MB
log4j.appender.MASTER_LOGGER.MaxBackupIndex=100
log4j.appender.MASTER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Master
log4j.appender.SECONDARY_MASTER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.SECONDARY_MASTER_LOGGER.File=${alluxio.logs.dir}/secondary_master.log
log4j.appender.SECONDARY_MASTER_LOGGER.MaxFileSize=10MB
log4j.appender.SECONDARY_MASTER_LOGGER.MaxBackupIndex=100
log4j.appender.SECONDARY_MASTER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.SECONDARY_MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Master audit
log4j.appender.MASTER_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.MASTER_AUDIT_LOGGER.File=${alluxio.logs.dir}/master_audit.log
log4j.appender.MASTER_AUDIT_LOGGER.MaxFileSize=10MB
log4j.appender.MASTER_AUDIT_LOGGER.MaxBackupIndex=100
log4j.appender.MASTER_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Job Master audit
log4j.appender.JOB_MASTER_AUDIT_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.JOB_MASTER_AUDIT_LOGGER.File=${alluxio.logs.dir}/job_master_audit.log
log4j.appender.JOB_MASTER_AUDIT_LOGGER.MaxFileSize=10MB
log4j.appender.JOB_MASTER_AUDIT_LOGGER.MaxBackupIndex=100
log4j.appender.JOB_MASTER_AUDIT_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.JOB_MASTER_AUDIT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Hub Agent
log4j.appender.HUB_AGENT_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.HUB_AGENT_LOGGER.File=${alluxio.logs.dir}/hub_agent.log
log4j.appender.HUB_AGENT_LOGGER.MaxFileSize=10MB
log4j.appender.HUB_AGENT_LOGGER.MaxBackupIndex=100
log4j.appender.HUB_AGENT_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.HUB_AGENT_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n

# Appender for Hub Manager
log4j.appender.HUB_MANAGER_LOGGER=org.apache.log4j.RollingFileAppender
log4j.appender.HUB_MANAGER_LOGGER.File=${alluxio.logs.dir}/hub_manager.log
log4j.appender.HUB_MANAGER_LOGGER.MaxFileSize=10MB
log4j.appender.HUB_MANAGER_LOGGER.MaxBackupIndex=100
log4j.appender.HUB_MANAGER_LOGGER.layout=org.apache.log4j.PatternLayout
log4j.appender.HUB_MANAGER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n
{'fraction_non_alphanumeric': 0.10722743203561617, 'fraction_numerical': 0.034005872880553185, 'mean_word_length': 7.267815191855912, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2091175', 'n_tokens_mistral': 4271, 'n_tokens_neox': 4268, 'n_words': 535}