def __init__(self, model_type, layers_setting, embed_layers_setting, input_dim, max_length, max_size, type_size, position_type, segment_type, sketch_embed_type, embed_pool_type, attention_norm_type, inter_activation, attention_dropout_prob, hidden_dropout_prob, output_dropout_prob):
    super(SketchTransformerModel, self).__init__()
    self.layers_setting = layers_setting
    self.num_hidden_layers = len(layers_setting)
    self.embed_pool_type = embed_pool_type
    assert (sketch_embed_type in ['linear', 'discrete'])
    if (sketch_embed_type == 'linear'):
        self.embedding = SketchEmbedding(input_dim, embed_layers_setting[0])
    elif (sketch_embed_type == 'discrete'):
        self.embedding = SketchDiscreteEmbedding(max_size, type_size, embed_layers_setting[0], embed_pool_type)
    assert (position_type in ['sin', 'learn', 'none'])
    if (position_type == 'sin'):
        self.pos_embedding = SketchSinPositionEmbedding(max_length, embed_layers_setting[0])
    elif (position_type == 'learn'):
        self.pos_embedding = SketchLearnPositionEmbedding(max_length, embed_layers_setting[0])
    else:
        self.pos_embedding = None
    if (segment_type == 'learn'):
        self.segment_embedding = SketchLearnPositionEmbedding(max_length, embed_layers_setting[0])
    else:
        self.segment_embedding = None
    self.embed_refine_net = SketchEmbeddingRefineNetwork(layers_setting[0][1], embed_layers_setting)
    assert (model_type in ['albert', 'bert'])
    if (model_type == 'albert'):
        self.encoder = SketchALEncoder(layers_setting, attention_norm_type, inter_activation, attention_dropout_prob, hidden_dropout_prob, output_dropout_prob)
    elif (model_type == 'bert'):
        self.encoder = SketchEncoder(layers_setting, attention_norm_type, inter_activation, attention_dropout_prob, hidden_dropout_prob, output_dropout_prob)
def __init__(self, model_type, layers_setting, embed_layers_setting, input_dim, max_length, max_size, type_size, position_type, segment_type, sketch_embed_type, embed_pool_type, attention_norm_type, inter_activation, attention_dropout_prob, hidden_dropout_prob, output_dropout_prob):
    super(SketchTransformerModel, self).__init__()
    self.layers_setting = layers_setting
    self.num_hidden_layers = len(layers_setting)
    self.embed_pool_type = embed_pool_type
    assert (sketch_embed_type in ['linear', 'discrete'])
    if (sketch_embed_type == 'linear'):
        self.embedding = SketchEmbedding(input_dim, embed_layers_setting[0])
    elif (sketch_embed_type == 'discrete'):
        self.embedding = SketchDiscreteEmbedding(max_size, type_size, embed_layers_setting[0], embed_pool_type)
    assert (position_type in ['sin', 'learn', 'none'])
    if (position_type == 'sin'):
        self.pos_embedding = SketchSinPositionEmbedding(max_length, embed_layers_setting[0])
    elif (position_type == 'learn'):
        self.pos_embedding = SketchLearnPositionEmbedding(max_length, embed_layers_setting[0])
    else:
        self.pos_embedding = None
    if (segment_type == 'learn'):
        self.segment_embedding = SketchLearnPositionEmbedding(max_length, embed_layers_setting[0])
    else:
        self.segment_embedding = None
    self.embed_refine_net = SketchEmbeddingRefineNetwork(layers_setting[0][1], embed_layers_setting)
    assert (model_type in ['albert', 'bert'])
    if (model_type == 'albert'):
        self.encoder = SketchALEncoder(layers_setting, attention_norm_type, inter_activation, attention_dropout_prob, hidden_dropout_prob, output_dropout_prob)
    elif (model_type == 'bert'):
        self.encoder = SketchEncoder(layers_setting, attention_norm_type, inter_activation, attention_dropout_prob, hidden_dropout_prob, output_dropout_prob)