<commit_before>#!/usr/bin/python

import re
import json
import time
import logging
import zlib
import base64

import classad

AUTO_ATTRS = {
    "date_attrs": re.compile(r"^.*Date$"),
    "provisioned_attrs": re.compile(r"^.*Provisioned$"),
    "resource_request_attrs": re.compile(r"^Request[A-Z].*$"),
    "target_boolean_attrs": re.compile(r"^(Want|Has|Is)[A-Z_].*$"),
}

# TEXT_ATTRS should only contain attrs that we want full text search on,
# otherwise strings are stored as keywords.
TEXT_ATTRS = {} or set()

INDEXED_KEYWORD_ATTRS = {
    "AccountingGroup",
    "AcctGroup",
    "AcctGroupUser",
    "AutoClusterId",
    "BatchQueue",
    "CloudLabelNames",
    "ConcurrencyLimits",
    "CondorPlatform",
    "CondorVersion",
    "DAGNodeName",
    "DAGParentNodeNames",
    "DockerImage",
    "GLIDEIN_Entry_Name",
    "GlideinClient",
    "GlideinEntryName",
    "GlideinFactory",
    "GlideinFrontendName",
    "GlideinName",
    "GlobalJobId",
    "GridJobId",
    "GridJobStatus",
    "GridResource",
    "JobBatchName",
    "JobDescription",
    "JobKeyword",
    "JobState",
    "LastRemoteHost",
    "LastRemotePool",
    "MATCH_EXP_JOB_GLIDECLIENT_Name",
    "MATCH_EXP_JOB_GLIDEIN_ClusterId",
    "MATCH_EXP_JOB_GLIDEIN_Entry_Name",
    "MATCH_EXP_JOB_GLIDEIN_Factory",
    "MATCH_EXP_JOB_GLIDEIN_Name",
    "MATCH_EXP_JOB_GLIDEIN_SEs",
    "MATCH_EXP_JOB_GLIDEIN_Schedd",
    "MATCH_EXP_JOB_GLIDEIN_Site",
    "MATCH_EXP_JOB_GLIDEIN_SiteWMS",
    "MATCH_EXP_JOB_GLIDEIN_SiteWMS_JobId",
    "MATCH_EXP_JOB_GLIDEIN_SiteWMS_Queue",
    "MATCH_EXP_JOB_GLIDEIN_SiteWMS_Slot",
    "MyType",
    "NTDomain",
    "Owner",
    "ProjectName",
    "RemoteHost",
    "RemotePool",
    "ScheddName",
    "ShouldTransferFiles",
    "StartdName",
    "StartdSlot",
    "Status",
    "SubmitterGlobalJobId",
    "SubmitterGroup",
    "SubmitterNegotiatingGroup",
    "TargetType",
    "Universe",
    "User",
    "WhenToTransferOutput",
    "x509UserProxyEmail",
    "x509UserProxyFQAN",
    "x509UserProxyFirstFQAN",
    "x509UserProxySubject",
    "x509UserProxyVOName",
}

NOINDEX_KEYWORD_ATTRS = {
    "AllRemoteHosts",
    "Args",
    "Arguments",
    "Cmd",
    "DAGManNodesLog",
    "DAGManNodesMask",
    "Err",
    "ExitReason",
    "HoldReason",
    "In",
    "Iwd",
    "LastHoldReason",
    "LastRejMatchReason",
    "OtherJobRemoveRequirements",
    "Out",
    "OutputDestination",
    "PostCmd",
    "PreCmd",
    "ReleaseReason",
    "RemoteIwd",
    "RemoveReason",
    "Requirements",
    "RootDir",
    "StartdIpAddr",
    "StartdPrincipal",
    "StarterIpAddr",
    "StarterPrincipal",
    "SubmitEventNotes",
    "TransferCheckpoint",
    "TransferInput",
    "TransferOutput",
    "TransferOutputRemaps",
    "TransferPlugins",
    "UserLog",
}

FLOAT_ATTRS = {
    "CPUsUsage",
    "JobDuration",
    "NetworkInputMb",
    "NetworkOutputMb",
    "Rank",
}

INT_ATTRS = {
    "AutoClusterId",
    "BlockReadKbytes",
    "BlockReads",
    "BlockWriteKbytes",
    "BlockWrites",
    "BytesRecvd",
    "BytesSent",
    "ClusterId",
    "CommittedSlotTime",
    "CommittedSuspensionTime",
    "CommittedTime",
    "CoreSize",
    "CpusProvisioned",
    "CumulativeRemoteSysCpu",
    "CumulativeRemoteUserCpu",
    "CumulativeSlotTime",
    "CumulativeSuspensionTime",
    "CumulativeTransferTime",
    "CurrentHosts",
    "DAGManJobId",
    "DataLocationsCount",
    "DelegatedProxyExpiration",
    "DiskProvisioned",
    "DiskUsage",
    "DiskUsage_RAW",
    "ErrSize",
    "ExecutableSize",
    "ExecutableSize_RAW",
    "ExitCode",
    "ExitSignal",
    "ExitStatus",
    "GpusProvisioned",
    "HoldReasonCode",
    "HoldReasonSubCode",
    "ImageSize",
    "ImageSize_RAW",
    "IOWait",
    "JobLeaseDuration",
    "JobMaxRetries",
    "JobMaxVacateTime",
    "JobPid",
    "JobPrio",
    "JobRunCount",
    "JobStatus",
    "JobSuccessExitCode",
    "JobUniverse",
    "KeepClaimIdle",
    "KillSig",
    "LastHoldReasonCode",
    "LastHoldReasonSubCode",
    "LastJobStatus",
    "LocalSysCpu",
    "LocalUserCpu",
    "MATCH_EXP_JOB_GLIDEIN_Job_Max_Time",
    "MATCH_EXP_JOB_GLIDEIN_MaxMemMBs",
    "MATCH_EXP_JOB_GLIDEIN_Max_Walltime",
    "MATCH_EXP_JOB_GLIDEIN_Memory",
    "MATCH_EXP_JOB_GLIDEIN_ProcId",
    "MATCH_EXP_JOB_GLIDEIN_ToDie",
    "MATCH_EXP_JOB_GLIDEIN_ToRetire",
    "MaxHosts",
    "MaxJobRetirementTime",
    "MaxTransferInputMB",<commit_msg>Add a few more well known attrs to ES push script HTCONDOR-148<commit_after>#!/usr/bin/python

import re
import json
import time
import logging
import zlib
import base64

import classad

AUTO_ATTRS = {
    "date_attrs": re.compile(r"^.*Date$"),
    "provisioned_attrs": re.compile(r"^.*Provisioned$"),
    "resource_request_attrs": re.compile(r"^Request[A-Z].*$"),
    "target_boolean_attrs": re.compile(r"^(Want|Has|Is)[A-Z_].*$"),
}

# TEXT_ATTRS should only contain attrs that we want full text search on,
# otherwise strings are stored as keywords.
TEXT_ATTRS = {} or set()

INDEXED_KEYWORD_ATTRS = {
    "AccountingGroup",
    "AcctGroup",
    "AcctGroupUser",
    "AutoClusterId",
    "BatchQueue",
    "CloudLabelNames",
    "ConcurrencyLimits",
    "CondorPlatform",
    "CondorVersion",
    "DAGNodeName",
    "DAGParentNodeNames",
    "DockerImage",
    "GLIDEIN_Entry_Name",
    "GlideinClient",
    "GlideinEntryName",
    "GlideinFactory",
    "GlideinFrontendName",
    "GlideinName",
    "GlobalJobId",
    "GridJobId",
    "GridJobStatus",
    "GridResource",
    "JobBatchName",
    "JobDescription",
    "JobKeyword",
    "JobState",
    "LastRemoteHost",
    "LastRemotePool",
    "MATCH_EXP_JOBGLIDEIN_ResourceName",
    "MATCH_EXP_JOB_GLIDECLIENT_Name",
    "MATCH_EXP_JOB_GLIDEIN_ClusterId",
    "MATCH_EXP_JOB_GLIDEIN_Entry_Name",
    "MATCH_EXP_JOB_GLIDEIN_Factory",
    "MATCH_EXP_JOB_GLIDEIN_Name",
    "MATCH_EXP_JOB_GLIDEIN_SEs",
    "MATCH_EXP_JOB_GLIDEIN_Schedd",
    "MATCH_EXP_JOB_GLIDEIN_Site",
    "MATCH_EXP_JOB_GLIDEIN_SiteWMS",
    "MATCH_EXP_JOB_GLIDEIN_SiteWMS_JobId",
    "MATCH_EXP_JOB_GLIDEIN_SiteWMS_Queue",
    "MATCH_EXP_JOB_GLIDEIN_SiteWMS_Slot",
    "MyType",
    "NTDomain",
    "Owner",
    "ProjectName",
    "RemoteHost",
    "RemotePool",
    "ScheddName",
    "ShouldTransferFiles",
    "StartdName",
    "StartdSlot",
    "Status",
    "SubmitterGlobalJobId",
    "SubmitterGroup",
    "SubmitterNegotiatingGroup",
    "TargetType",
    "Universe",
    "User",
    "WhenToTransferOutput",
    "x509UserProxyEmail",
    "x509UserProxyFQAN",
    "x509UserProxyFirstFQAN",
    "x509UserProxySubject",
    "x509UserProxyVOName",
}

NOINDEX_KEYWORD_ATTRS = {
    "AllRemoteHosts",
    "Args",
    "Arguments",
    "Cmd",
    "DAGManNodesLog",
    "DAGManNodesMask",
    "Err",
    "ExitReason",
    "HoldReason",
    "In",
    "Iwd",
    "JOBGLIDEIN_ResourceName",
    "LastHoldReason",
    "LastRejMatchReason",
    "OtherJobRemoveRequirements",
    "Out",
    "OutputDestination",
    "PostCmd",
    "PreCmd",
    "ReleaseReason",
    "RemoteIwd",
    "RemoveReason",
    "Requirements",
    "RootDir",
    "StartdIpAddr",
    "StartdPrincipal",
    "StarterIpAddr",
    "StarterPrincipal",
    "SubmitEventNotes",
    "TransferCheckpoint",
    "TransferInput",
    "TransferOutput",
    "TransferOutputRemaps",
    "TransferPlugins",
    "UserLog",
}

FLOAT_ATTRS = {
    "CPUsUsage",
    "JobBatchId",
    "JobDuration",
    "NetworkInputMb",
    "NetworkOutputMb",
    "Rank",
}

INT_ATTRS = {
    "AutoClusterId",
    "BlockReadKbytes",
    "BlockReads",
    "BlockWriteKbytes",
    "BlockWrites",
    "BytesRecvd",
    "BytesSent",
    "ClusterId",
    "CommittedSlotTime",
    "CommittedSuspensionTime",
    "CommittedTime",
    "CoreSize",
    "CpusProvisioned",
    "CumulativeRemoteSysCpu",
    "CumulativeRemoteUserCpu",
    "CumulativeSlotTime",
    "CumulativeSuspensionTime",
    "CumulativeTransferTime",
    "CurrentHosts",
    "DAGManJobId",
    "DataLocationsCount",
    "DelegatedProxyExpiration",
    "DiskProvisioned",
    "DiskUsage",
    "DiskUsage_RAW",
    "ErrSize",
    "ExecutableSize",
    "ExecutableSize_RAW",
    "ExitCode",
    "ExitSignal",
    "ExitStatus",
    "GpusProvisioned",
    "HoldReasonCode",
    "HoldReasonSubCode",
    "ImageSize",
    "ImageSize_RAW",
    "IOWait",
    "JobLeaseDuration",
    "JobMaxRetries",
    "JobMaxVacateTime",
    "JobPid",
    "JobPrio",
    "JobRunCount",
    "JobStatus",
    "JobSuccessExitCode",
    "JobUniverse",
    "KeepClaimIdle",
    "KillSig",
    "LastHoldReasonCode",
    "LastHoldReasonSubCode",
    "LastJobStatus",
    "LocalSysCpu",
    "LocalUserCpu",
    "MachineAttrCpus0",
    "MachineAttrSlotWeight0",
    "MATCH_EXP_JOB_GLIDEIN_Job_Max_Time",
    "MATCH_EXP_JOB_GLIDEIN_MaxMemMBs",
    "MATCH_EXP_JOB_GLIDEIN_Max_Walltime",
    "MATCH_EXP_JOB_GLIDEIN_Memory",
    "MATCH_EXP_JOB_GLIDEIN_ProcId",
    "MATCH_EXP_JOB_GLIDEIN_ToDie",
    "MATCH_EXP_JOB_GLIDEIN_ToRetire",
    "MaxHosts",
    "MaxJobRetirementTime",
    "MaxTransferInputMB",
{'fraction_non_alphanumeric': 0.13275420425437132, 'fraction_numerical': 0.004343468092215168, 'mean_word_length': 3.7362869198312234, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5379882', 'n_tokens_mistral': 3446, 'n_tokens_neox': 3361, 'n_words': 443}