<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace AsmTools {

    /// <summary>
    /// Operand Type: reg, mem, imm, UNKNOWN
    /// </summary>
    [Flags]
    public enum Ot : byte {
        reg     = 1 << 0,
        mem     = 1 << 1,
        imm     = 1 << 2,
        UNKNOWN = 1 << 3
    }

    /// <summary>
    /// Operand Type tuple (OperandType x OperandType)
    /// </summary>
    [Flags]
    public enum Ot2 : byte {
        reg_reg = Ot.reg | (Ot.reg << 4),
        reg_mem = Ot.reg | (Ot.mem << 4),
        reg_imm = Ot.reg | (Ot.imm << 4),
        reg_UNKNOWN = Ot.reg | (Ot.UNKNOWN << 4),

        mem_reg = Ot.mem | (Ot.reg << 4),
        mem_mem = Ot.mem | (Ot.mem << 4),
        mem_imm = Ot.mem | (Ot.imm << 4),
        mem_UNKNOWN = Ot.mem | (Ot.UNKNOWN << 4),

        imm_reg = Ot.imm | (Ot.reg << 4),
        imm_mem = Ot.imm | (Ot.mem << 4),
        imm_imm = Ot.imm | (Ot.imm << 4),
        imm_UNKNOWN = Ot.imm | (Ot.UNKNOWN << 4),

        UNKNOWN_reg = Ot.UNKNOWN | (Ot.reg << 4),
        UNKNOWN_mem = Ot.UNKNOWN | (Ot.mem << 4),
        UNKNOWN_imm = Ot.UNKNOWN | (Ot.imm << 4),
        UNKNOWN_UNKNOWN = Ot.UNKNOWN | (Ot.UNKNOWN << 4),
    }

    public static partial class AsmSourceTools {
        public static Tuple<Ot, Ot> splitOt(Ot2 operandTuple) {
            switch (operandTuple) {
                case Ot2.reg_reg: return new Tuple<Ot, Ot>(Ot.reg, Ot.reg);
                case Ot2.reg_mem: return new Tuple<Ot, Ot>(Ot.reg, Ot.mem);
                case Ot2.reg_imm: return new Tuple<Ot, Ot>(Ot.reg, Ot.imm);
                case Ot2.reg_UNKNOWN: return new Tuple<Ot, Ot>(Ot.reg, Ot.UNKNOWN);
                case Ot2.mem_reg: return new Tuple<Ot, Ot>(Ot.mem, Ot.reg);
                case Ot2.mem_mem: return new Tuple<Ot, Ot>(Ot.mem, Ot.mem);
                case Ot2.mem_imm: return new Tuple<Ot, Ot>(Ot.mem, Ot.imm);
                case Ot2.mem_UNKNOWN: return new Tuple<Ot, Ot>(Ot.mem, Ot.UNKNOWN);
                case Ot2.imm_reg: return new Tuple<Ot, Ot>(Ot.imm, Ot.reg);
                case Ot2.imm_mem: return new Tuple<Ot, Ot>(Ot.imm, Ot.mem);
                case Ot2.imm_imm: return new Tuple<Ot, Ot>(Ot.imm, Ot.imm);
                case Ot2.imm_UNKNOWN: return new Tuple<Ot, Ot>(Ot.imm, Ot.UNKNOWN);
                case Ot2.UNKNOWN_reg: return new Tuple<Ot, Ot>(Ot.UNKNOWN, Ot.reg);
                case Ot2.UNKNOWN_mem: return new Tuple<Ot, Ot>(Ot.UNKNOWN, Ot.mem);
                case Ot2.UNKNOWN_imm: return new Tuple<Ot, Ot>(Ot.UNKNOWN, Ot.imm);
                case Ot2.UNKNOWN_UNKNOWN:
                default:
                    return new Tuple<Ot, Ot>(Ot.UNKNOWN, Ot.UNKNOWN);
            }
        }
        public static Ot2 mergeOt(Ot ot1, Ot ot2) {
            return (Ot2)(((byte)ot1) | (((byte)ot2) << 4));
        }
    }
}
<commit_msg>added Enum Ot3<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace AsmTools {

    /// <summary>
    /// Operand Type: reg, mem, imm, UNKNOWN
    /// </summary>
    [Flags]
    public enum Ot : byte {
        reg     = 1 << 0,
        mem     = 1 << 1,
        imm     = 1 << 2,
        UNKNOWN = 1 << 3
    }

    /// <summary>
    /// Operand Type tuple (OperandType x OperandType)
    /// </summary>
    [Flags]
    public enum Ot2 : byte {
        reg_reg = Ot.reg | (Ot.reg << 4),
        reg_mem = Ot.reg | (Ot.mem << 4),
        reg_imm = Ot.reg | (Ot.imm << 4),
        reg_UNKNOWN = Ot.reg | (Ot.UNKNOWN << 4),

        mem_reg = Ot.mem | (Ot.reg << 4),
        mem_mem = Ot.mem | (Ot.mem << 4),
        mem_imm = Ot.mem | (Ot.imm << 4),
        mem_UNKNOWN = Ot.mem | (Ot.UNKNOWN << 4),

        imm_reg = Ot.imm | (Ot.reg << 4),
        imm_mem = Ot.imm | (Ot.mem << 4),
        imm_imm = Ot.imm | (Ot.imm << 4),
        imm_UNKNOWN = Ot.imm | (Ot.UNKNOWN << 4),

        UNKNOWN_reg = Ot.UNKNOWN | (Ot.reg << 4),
        UNKNOWN_mem = Ot.UNKNOWN | (Ot.mem << 4),
        UNKNOWN_imm = Ot.UNKNOWN | (Ot.imm << 4),
        UNKNOWN_UNKNOWN = Ot.UNKNOWN | (Ot.UNKNOWN << 4),
    }
    /// <summary>
    ///  Operand Type tuple (OperandType x OperandType x OperandType)
    /// </summary>
    [Flags]
    public enum Ot3 : short {
        reg_reg_reg = Ot.reg | (Ot.reg << 4) | (Ot.reg << 8),
        reg_mem_reg = Ot.reg | (Ot.mem << 4) | (Ot.reg << 8),
        reg_imm_reg = Ot.reg | (Ot.imm << 4) | (Ot.reg << 8),
        reg_UNKNOWN_reg = Ot.reg | (Ot.UNKNOWN << 4) | (Ot.reg << 8),

        mem_reg_reg = Ot.mem | (Ot.reg << 4) | (Ot.reg << 8),
        mem_mem_reg = Ot.mem | (Ot.mem << 4) | (Ot.reg << 8),
        mem_imm_reg = Ot.mem | (Ot.imm << 4) | (Ot.reg << 8),
        mem_UNKNOWN_reg = Ot.mem | (Ot.UNKNOWN << 4) | (Ot.reg << 8),

        imm_reg_reg = Ot.imm | (Ot.reg << 4) | (Ot.reg << 8),
        imm_mem_reg = Ot.imm | (Ot.mem << 4) | (Ot.reg << 8),
        imm_imm_reg = Ot.imm | (Ot.imm << 4) | (Ot.reg << 8),
        imm_UNKNOWN_reg = Ot.imm | (Ot.UNKNOWN << 4) | (Ot.reg << 8),

        UNKNOWN_reg_reg = Ot.UNKNOWN | (Ot.reg << 4) | (Ot.reg << 8),
        UNKNOWN_mem_reg = Ot.UNKNOWN | (Ot.mem << 4) | (Ot.reg << 8),
        UNKNOWN_imm_reg = Ot.UNKNOWN | (Ot.imm << 4) | (Ot.reg << 8),
        UNKNOWN_UNKNOWN_reg = Ot.UNKNOWN | (Ot.UNKNOWN << 4) | (Ot.reg << 8),
        ///
        reg_rem_mem = Ot.reg | (Ot.reg << 4) | (Ot.mem << 8),
        reg_mem_mem = Ot.reg | (Ot.mem << 4) | (Ot.mem << 8),
        reg_imm_mem = Ot.reg | (Ot.imm << 4) | (Ot.mem << 8),
        reg_UNKNOWN_mem = Ot.reg | (Ot.UNKNOWN << 4) | (Ot.mem << 8),

        mem_reg_mem = Ot.mem | (Ot.reg << 4) | (Ot.mem << 8),
        mem_mem_mem = Ot.mem | (Ot.mem << 4) | (Ot.mem << 8),
        mem_imm_mem = Ot.mem | (Ot.imm << 4) | (Ot.mem << 8),
        mem_UNKNOWN_mem = Ot.mem | (Ot.UNKNOWN << 4) | (Ot.mem << 8),

        imm_reg_mem = Ot.imm | (Ot.reg << 4) | (Ot.mem << 8),
        imm_mem_mem = Ot.imm | (Ot.mem << 4) | (Ot.mem << 8),
        imm_imm_mem = Ot.imm | (Ot.imm << 4) | (Ot.mem << 8),
        imm_UNKNOWN_mem = Ot.imm | (Ot.UNKNOWN << 4) | (Ot.mem << 8),

        UNKNOWN_reg_mem = Ot.UNKNOWN | (Ot.reg << 4) | (Ot.mem << 8),
        UNKNOWN_mem_mem = Ot.UNKNOWN | (Ot.mem << 4) | (Ot.mem << 8),
        UNKNOWN_imm_mem = Ot.UNKNOWN | (Ot.imm << 4) | (Ot.mem << 8),
        UNKNOWN_UNKNOWN_mem = Ot.UNKNOWN | (Ot.UNKNOWN << 4) | (Ot.mem << 8),
        ///
        reg_reg_imm = Ot.reg | (Ot.reg << 4) | (Ot.imm << 8),
        reg_mem_imm = Ot.reg | (Ot.mem << 4) | (Ot.imm << 8),
        reg_imm_imm = Ot.reg | (Ot.imm << 4) | (Ot.imm << 8),
        reg_UNKNOWN_imm = Ot.reg | (Ot.UNKNOWN << 4) | (Ot.imm << 8),

        mem_reg_imm = Ot.mem | (Ot.reg << 4) | (Ot.imm << 8),
        mem_mem_imm = Ot.mem | (Ot.mem << 4) | (Ot.imm << 8),
        mem_imm_imm = Ot.mem | (Ot.imm << 4) | (Ot.imm << 8),
        mem_UNKNOWN_imm = Ot.mem | (Ot.UNKNOWN << 4) | (Ot.imm << 8),

        imm_reg_imm = Ot.imm | (Ot.reg << 4) | (Ot.imm << 8),
        imm_mem_imm = Ot.imm | (Ot.mem << 4) | (Ot.imm << 8),
        imm_imm_imm = Ot.imm | (Ot.imm << 4) | (Ot.imm << 8),
        imm_UNKNOWN_imm = Ot.imm | (Ot.UNKNOWN << 4) | (Ot.imm << 8),

        UNKNOWN_reg_imm = Ot.UNKNOWN | (Ot.reg << 4) | (Ot.imm << 8),
        UNKNOWN_mem_imm = Ot.UNKNOWN | (Ot.mem << 4) | (Ot.imm << 8),
        UNKNOWN_imm_imm = Ot.UNKNOWN | (Ot.imm << 4) | (Ot.imm << 8),
        UNKNOWN_UNKNOWN_imm = Ot.UNKNOWN | (Ot.UNKNOWN << 4) | (Ot.imm << 8),
        ///
        reg_reg_UNKNOWN = Ot.reg | (Ot.reg << 4) | (Ot.UNKNOWN << 8),
        reg_mem_UNKNOWN = Ot.reg | (Ot.mem << 4) | (Ot.UNKNOWN << 8),
        reg_imm_UNKNOWN = Ot.reg | (Ot.imm << 4) | (Ot.UNKNOWN << 8),
        reg_UNKNOWN_UNKNOWN = Ot.reg | (Ot.UNKNOWN << 4) | (Ot.UNKNOWN << 8),

        mem_reg_UNKNOWN = Ot.mem | (Ot.reg << 4) | (Ot.UNKNOWN << 8),
        mem_mem_UNKNOWN = Ot.mem | (Ot.mem << 4) | (Ot.UNKNOWN << 8),
        mem_imm_UNKNOWN = Ot.mem | (Ot.imm << 4) | (Ot.UNKNOWN << 8),
        mem_UNKNOWN_UNKNOWN = Ot.mem | (Ot.UNKNOWN << 4) | (Ot.UNKNOWN << 8),

        imm_reg_UNKNOWN = Ot.imm | (Ot.reg << 4) | (Ot.UNKNOWN << 8),
        imm_mem_UNKNOWN = Ot.imm | (Ot.mem << 4) | (Ot.UNKNOWN << 8),
        imm_imm_UNKNOWN = Ot.imm | (Ot.imm << 4) | (Ot.UNKNOWN << 8),
        imm_UNKNOWN_UNKNOWN = Ot.imm | (Ot.UNKNOWN << 4) | (Ot.UNKNOWN << 8),

        UNKNOWN_reg_UNKNOWN = Ot.UNKNOWN | (Ot.reg << 4) | (Ot.UNKNOWN << 8),
        UNKNOWN_mem_UNKNOWN = Ot.UNKNOWN | (Ot.mem << 4) | (Ot.UNKNOWN << 8),
        UNKNOWN_imm_UNKNOWN = Ot.UNKNOWN | (Ot.imm << 4) | (Ot.UNKNOWN << 8),
        UNKNOWN_UNKNOWN_UNKNOWN = Ot.UNKNOWN | (Ot.UNKNOWN << 4) | (Ot.UNKNOWN << 8),

    }


    public static partial class AsmSourceTools {
        public static Tuple<Ot, Ot> splitOt(Ot2 operandTuple) {
            switch (operandTuple) {
                case Ot2.reg_reg: return new Tuple<Ot, Ot>(Ot.reg, Ot.reg);
                case Ot2.reg_mem: return new Tuple<Ot, Ot>(Ot.reg, Ot.mem);
                case Ot2.reg_imm: return new Tuple<Ot, Ot>(Ot.reg, Ot.imm);
                case Ot2.reg_UNKNOWN: return new Tuple<Ot, Ot>(Ot.reg, Ot.UNKNOWN);
                case Ot2.mem_reg: return new Tuple<Ot, Ot>(Ot.mem, Ot.reg);
                case Ot2.mem_mem: return new Tuple<Ot, Ot>(Ot.mem, Ot.mem);
                case Ot2.mem_imm: return new Tuple<Ot, Ot>(Ot.mem, Ot.imm);
                case Ot2.mem_UNKNOWN: return new Tuple<Ot, Ot>(Ot.mem, Ot.UNKNOWN);
                case Ot2.imm_reg: return new Tuple<Ot, Ot>(Ot.imm, Ot.reg);
                case Ot2.imm_mem: return new Tuple<Ot, Ot>(Ot.imm, Ot.mem);
                case Ot2.imm_imm: return new Tuple<Ot, Ot>(Ot.imm, Ot.imm);
                case Ot2.imm_UNKNOWN: return new Tuple<Ot, Ot>(Ot.imm, Ot.UNKNOWN);
                case Ot2.UNKNOWN_reg: return new Tuple<Ot, Ot>(Ot.UNKNOWN, Ot.reg);
                case Ot2.UNKNOWN_mem: return new Tuple<Ot, Ot>(Ot.UNKNOWN, Ot.mem);
                case Ot2.UNKNOWN_imm: return new Tuple<Ot, Ot>(Ot.UNKNOWN, Ot.imm);
                case Ot2.UNKNOWN_UNKNOWN:
                default:
                    return new Tuple<Ot, Ot>(Ot.UNKNOWN, Ot.UNKNOWN);
            }
        }
        public static Ot2 mergeOt(Ot ot1, Ot ot2) {
            return (Ot2)(((byte)ot1) | (((byte)ot2) << 4));
        }
        public static Ot2 mergeOt(Ot ot1, Ot ot2, Ot ot3) {
            return (Ot2)(((byte)ot1) | (((byte)ot2) << 4) | (((byte)ot3) << 8));
        }
    }
}

{'fraction_non_alphanumeric': 0.17954567056363463, 'fraction_numerical': 0.02262142381902861, 'mean_word_length': 1.8986225895316804, 'pattern_counts': {'":': 0, '<': 391, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5375770', 'n_tokens_mistral': 4331, 'n_tokens_neox': 3966, 'n_words': 918}