<commit_before>package lychi;

import java.io.*;
import java.util.*;
import java.util.logging.Logger;
import java.util.logging.Level;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
import java.security.MessageDigest;

import chemaxon.formats.MolImporter;
import chemaxon.formats.MolFormatException;
import chemaxon.struc.MolAtom;
import chemaxon.struc.MolBond;
import chemaxon.struc.Molecule;
import chemaxon.struc.MoleculeGraph;
import chemaxon.struc.RgMolecule;
import chemaxon.util.MolHandler;
import chemaxon.struc.DPoint3;
import lychi.tautomers.*;
import lychi.util.*;

/**
 * A new version of CanonicalForm but without using Reactor class;
 * Please keep a detailed log of the changes in VERSION here:
 *
 * 07.22.2010: 0x05: Didn't previously handle hydrogen isotopes.
 *     Make changes to makeHydrogensImplicit from MolAtom.ALL_H to
 *     MolAtom.ALL_H & ~MolAtom.ISOTOPE_H.  Also make changes to 
 *     hashKeyArray to preserve isotopes in the last layer.
 *
 * 09.01.2010: 0x06: Fix all standardization errors based on PubChem
 *     run.  Changed NeutralizedRules, Deprotonator, 
 *     fixed NormalizedRules 0 & 35.  
 *
 * 10.28.2010: 0x07: Fix hashKeyArray to use the new AtomIterator
 *     so that a canonical ordering is guaranteed whereas the previous 
 *     version that used ChemUtil.graphInvariantOrder on the skeleton 
 *     and atom labels to break ties doesn't work too well.
 */
public class LyChIStandardizer {
    private static Logger logger = Logger.getLogger
	(LyChIStandardizer.class.getName());

    /**
     * This static version value must be updated if any changes is made
     * to this class that would be imcompatible with earlier results!!!
     */
    public static final int VERSION = 0x10;

    static final private boolean DEBUG;
    static final private boolean UNMEX = true; // apply UNM extra rules
    static {
        boolean debug = false;
	try {
	    debug = Boolean.getBoolean("lychi-debug");
	}
	catch (Exception ex) {
            ex.printStackTrace();
	}
        DEBUG = debug;
    }


    public static String[] NeutralizedRules = new String[] {
	"[*-;$([!#6;!$([-]~[+]);!$([v4;B,Fe,Al,Au,Ga,In]);!$([v5;S,Se,Te,Sn,Si,Zr]);!$([v6;P,As,Sb,Ta]);!$([v2;Ag]);!$([v3;Hg,Zn]);!$([v2;Cl,Br]);!$([v4;Cl,Br]);!$([v6;Cl,Br]);!$([v7;Te]);!$([v8;Cl,Br])]):1]>>[*+0:1][H]"
    };

    /**
     * Please be VERY CAREFUL about changing the order of this array!
     * Any rearrangement of this array will certainly produce different
     * hash keys!  Please also update the VERSION value above if any
     * changes is made here.
     */
    public static String[] NormalizedRules = new String[] {
	"[*+;$([!#8]);H0:1]([O-:2])>>[*:1](=[O:2])",
	"[N;v4:1][O-:2]>>[N+:1][O-:2]",
	//"[N;v5:1][N;v3:2]>>[N+:1][N:2]",
	"[O+:1][N-:2]>>[O:1][N:2]",
	"[O;X1:1][#6:2]=[#7;X2;v4:3]=[*:4]>>[O:1]=[#6:2][#7:3]=[*:4]",
	//"[O:1]=[#6:2][#6:3]=[#7;X3;v5:4]>>[O:1][#6:2]=[#6:3][#7+:4]",
	"[#6][NH2+:1][O;X1-:2]>>[#6][N:1]=[O:2]",
	"[#6;X3+:1]-[#7;X3:2]>>[C:1]=[N+:2]",
	"[C;X2+:1]=[N;X2:2]>>[C:1]#[N+:2]",<commit_msg>don't use unm rules by default as they generate lots of incorrect (e.g., valence errors) transforms<commit_after>package lychi;

import java.io.*;
import java.util.*;
import java.util.logging.Logger;
import java.util.logging.Level;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
import java.security.MessageDigest;

import chemaxon.formats.MolImporter;
import chemaxon.formats.MolFormatException;
import chemaxon.struc.MolAtom;
import chemaxon.struc.MolBond;
import chemaxon.struc.Molecule;
import chemaxon.struc.MoleculeGraph;
import chemaxon.struc.RgMolecule;
import chemaxon.util.MolHandler;
import chemaxon.struc.DPoint3;
import lychi.tautomers.*;
import lychi.util.*;

/**
 * A new version of CanonicalForm but without using Reactor class;
 * Please keep a detailed log of the changes in VERSION here:
 *
 * 07.22.2010: 0x05: Didn't previously handle hydrogen isotopes.
 *     Make changes to makeHydrogensImplicit from MolAtom.ALL_H to
 *     MolAtom.ALL_H & ~MolAtom.ISOTOPE_H.  Also make changes to 
 *     hashKeyArray to preserve isotopes in the last layer.
 *
 * 09.01.2010: 0x06: Fix all standardization errors based on PubChem
 *     run.  Changed NeutralizedRules, Deprotonator, 
 *     fixed NormalizedRules 0 & 35.  
 *
 * 10.28.2010: 0x07: Fix hashKeyArray to use the new AtomIterator
 *     so that a canonical ordering is guaranteed whereas the previous 
 *     version that used ChemUtil.graphInvariantOrder on the skeleton 
 *     and atom labels to break ties doesn't work too well.
 */
public class LyChIStandardizer {
    private static Logger logger = Logger.getLogger
	(LyChIStandardizer.class.getName());

    /**
     * This static version value must be updated if any changes is made
     * to this class that would be imcompatible with earlier results!!!
     */
    public static final int VERSION = 0x10;

    static final private boolean DEBUG;
    static final private boolean UNMEX; // apply UNM extra rules
    static {
        boolean debug = false;
        boolean extra = false;
	try {
	    debug = Boolean.getBoolean("lychi-debug");
            extra = Boolean.getBoolean("lychi-unm-extra");
	}
	catch (Exception ex) {
            ex.printStackTrace();
	}
        DEBUG = debug;
        UNMEX = extra;
    }


    public static String[] NeutralizedRules = new String[] {
	"[*-;$([!#6;!$([-]~[+]);!$([v4;B,Fe,Al,Au,Ga,In]);!$([v5;S,Se,Te,Sn,Si,Zr]);!$([v6;P,As,Sb,Ta]);!$([v2;Ag]);!$([v3;Hg,Zn]);!$([v2;Cl,Br]);!$([v4;Cl,Br]);!$([v6;Cl,Br]);!$([v7;Te]);!$([v8;Cl,Br])]):1]>>[*+0:1][H]"
    };

    /**
     * Please be VERY CAREFUL about changing the order of this array!
     * Any rearrangement of this array will certainly produce different
     * hash keys!  Please also update the VERSION value above if any
     * changes is made here.
     */
    public static String[] NormalizedRules = new String[] {
	"[*+;$([!#8]);H0:1]([O-:2])>>[*:1](=[O:2])",
	"[N;v4:1][O-:2]>>[N+:1][O-:2]",
	//"[N;v5:1][N;v3:2]>>[N+:1][N:2]",
	"[O+:1][N-:2]>>[O:1][N:2]",
	"[O;X1:1][#6:2]=[#7;X2;v4:3]=[*:4]>>[O:1]=[#6:2][#7:3]=[*:4]",
	//"[O:1]=[#6:2][#6:3]=[#7;X3;v5:4]>>[O:1][#6:2]=[#6:3][#7+:4]",
	"[#6][NH2+:1][O;X1-:2]>>[#6][N:1]=[O:2]",
	"[#6;X3+:1]-[#7;X3:2]>>[C:1]=[N+:2]",
	"[C;X2+:1]=[N;X2:2]>>[C:1]#[N+:2]",
{'fraction_non_alphanumeric': 0.18665831506420294, 'fraction_numerical': 0.04008769182586909, 'mean_word_length': 4.1466559226430295, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5374132', 'n_tokens_mistral': 2591, 'n_tokens_neox': 2439, 'n_words': 544}