# Path: third_party/bi_att_flow/my/tensorflow/general.py
# def add_wd(wd, scope=None):
#   scope = scope or tf.get_variable_scope().name
#   variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope)
#   with tf.name_scope('weight_decay'):
#     for var in variables:
#       weight_decay = tf.mul(
#           tf.nn.l2_loss(var), wd, name='{}/wd'.format(var.op.name))
#       tf.add_to_collection('losses', weight_decay)
# 
# Path: third_party/bi_att_flow/my/tensorflow/general.py
# def exp_mask(val, mask, name=None):
#   """Give very negative number to unmasked elements in val.
# 
#     For example, [-3, -2, 10], [True, True, False] -> [-3, -2, -1e9].
#     Typically, this effectively masks in exponential space (e.g. softmax)
#     Args:
#         val: values to be masked
#         mask: masking boolean tensor, same shape as tensor
#         name: name for output tensor
# 
#     Returns:
#         Same shape as val, where some elements are very small (exponentially
#         zero)
#     """
#   if name is None:
#     name = 'exp_mask'
#   return tf.add(
#       val, (1 - tf.cast(mask, 'float')) * VERY_NEGATIVE_NUMBER, name=name)
# 
# Path: third_party/bi_att_flow/my/tensorflow/general.py
# def flatten(tensor, keep):
#   fixed_shape = tensor.get_shape().as_list()
#   start = len(fixed_shape) - keep
#   left = reduce(mul,
#                 [fixed_shape[i] or tf.shape(tensor)[i] for i in range(start)])
#   out_shape = [left] + [
#       fixed_shape[i] or tf.shape(tensor)[i]
#       for i in range(start, len(fixed_shape))
#   ]
#   flat = tf.reshape(tensor, out_shape)
#   return flat
# 
# Path: third_party/bi_att_flow/my/tensorflow/general.py
# def reconstruct(tensor, ref, keep):
#   ref_shape = ref.get_shape().as_list()
#   tensor_shape = tensor.get_shape().as_list()
#   ref_stop = len(ref_shape) - keep
#   tensor_start = len(tensor_shape) - keep
#   pre_shape = [ref_shape[i] or tf.shape(ref)[i] for i in range(ref_stop)]
#   keep_shape = [
#       tensor_shape[i] or tf.shape(tensor)[i]
#       for i in range(tensor_start, len(tensor_shape))
#   ]
#   # pre_shape = [tf.shape(ref)[i] for i in range(len(ref.get_shape().as_list()[:-keep]))]
#   # keep_shape = tensor.get_shape().as_list()[-keep:]
#   target_shape = pre_shape + keep_shape
#   out = tf.reshape(tensor, target_shape)
#   return out


import tensorflow as tf
from third_party.bi_att_flow.my.tensorflow.general import add_wd
from third_party.bi_att_flow.my.tensorflow.general import exp_mask
from third_party.bi_att_flow.my.tensorflow.general import flatten
from third_party.bi_att_flow.my.tensorflow.general import reconstruct
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.util import nest
                         size,
                         bias,
                         bias_start=0.0,
                         scope=None,
                         mask=None,
                         wd=0.0,
                         input_keep_prob=1.0,
                         is_train=None):
  with tf.variable_scope(scope or "Double_Linear_Logits"):
    first = tf.tanh(
        linear(
            args,
            size,
            bias,
            bias_start=bias_start,
            scope="first",
            wd=wd,
            input_keep_prob=input_keep_prob,
            is_train=is_train))
    second = linear(
        first,
        1,
        bias,
        bias_start=bias_start,
        squeeze=True,
        scope="second",
        wd=wd,
        input_keep_prob=input_keep_prob,
        is_train=is_train)
    if mask is not None:

# Path: third_party/bi_att_flow/my/tensorflow/general.py
# def add_wd(wd, scope=None):
#   scope = scope or tf.get_variable_scope().name
#   variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope)
#   with tf.name_scope('weight_decay'):
#     for var in variables:
#       weight_decay = tf.mul(
#           tf.nn.l2_loss(var), wd, name='{}/wd'.format(var.op.name))
#       tf.add_to_collection('losses', weight_decay)
# 
# Path: third_party/bi_att_flow/my/tensorflow/general.py
# def exp_mask(val, mask, name=None):
#   """Give very negative number to unmasked elements in val.
# 
#     For example, [-3, -2, 10], [True, True, False] -> [-3, -2, -1e9].
#     Typically, this effectively masks in exponential space (e.g. softmax)
#     Args:
#         val: values to be masked
#         mask: masking boolean tensor, same shape as tensor
#         name: name for output tensor
# 
#     Returns:
#         Same shape as val, where some elements are very small (exponentially
#         zero)
#     """
#   if name is None:
#     name = 'exp_mask'
#   return tf.add(
#       val, (1 - tf.cast(mask, 'float')) * VERY_NEGATIVE_NUMBER, name=name)
# 
# Path: third_party/bi_att_flow/my/tensorflow/general.py
# def flatten(tensor, keep):
#   fixed_shape = tensor.get_shape().as_list()
#   start = len(fixed_shape) - keep
#   left = reduce(mul,
#                 [fixed_shape[i] or tf.shape(tensor)[i] for i in range(start)])
#   out_shape = [left] + [
#       fixed_shape[i] or tf.shape(tensor)[i]
#       for i in range(start, len(fixed_shape))
#   ]
#   flat = tf.reshape(tensor, out_shape)
#   return flat
# 
# Path: third_party/bi_att_flow/my/tensorflow/general.py
# def reconstruct(tensor, ref, keep):
#   ref_shape = ref.get_shape().as_list()
#   tensor_shape = tensor.get_shape().as_list()
#   ref_stop = len(ref_shape) - keep
#   tensor_start = len(tensor_shape) - keep
#   pre_shape = [ref_shape[i] or tf.shape(ref)[i] for i in range(ref_stop)]
#   keep_shape = [
#       tensor_shape[i] or tf.shape(tensor)[i]
#       for i in range(tensor_start, len(tensor_shape))
#   ]
#   # pre_shape = [tf.shape(ref)[i] for i in range(len(ref.get_shape().as_list()[:-keep]))]
#   # keep_shape = tensor.get_shape().as_list()[-keep:]
#   target_shape = pre_shape + keep_shape
#   out = tf.reshape(tensor, target_shape)
#   return out


# Path: third_party/bi_att_flow/my/tensorflow/nn.py
import tensorflow as tf
from third_party.bi_att_flow.my.tensorflow.general import add_wd
from third_party.bi_att_flow.my.tensorflow.general import exp_mask
from third_party.bi_att_flow.my.tensorflow.general import flatten
from third_party.bi_att_flow.my.tensorflow.general import reconstruct
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.util import nest

                         size,
                         bias,
                         bias_start=0.0,
                         scope=None,
                         mask=None,
                         wd=0.0,
                         input_keep_prob=1.0,
                         is_train=None):
  with tf.variable_scope(scope or "Double_Linear_Logits"):
    first = tf.tanh(
        linear(
            args,
            size,
            bias,
            bias_start=bias_start,
            scope="first",
            wd=wd,
            input_keep_prob=input_keep_prob,
            is_train=is_train))
    second = linear(
        first,
        1,
        bias,
        bias_start=bias_start,
        squeeze=True,
        scope="second",
        wd=wd,
        input_keep_prob=input_keep_prob,
        is_train=is_train)
    if mask is not None:

      second = exp_mask(second, mask)