<jupyter_start><jupyter_text>ERROR ANALYSIS<jupyter_code># load preprocessed ref file
ref = _load_predictions(preds_dir/'..'/f'{split}.tsv')
ref.loc[:, 'answer_coordinates'] = ref["answer_coordinates"].apply(eval)
ref.loc[:,'answer_coordinates'] = ref["answer_coordinates"].apply(lambda x:set(map(eval,x)))
ref = ref.rename(columns={'answer_coordinates':'answer_coordinates_gold'})
ref = ref[['id','answer_coordinates_gold']]
print('Total tf preprocessed refs:',len(ref))
ref['answer_coordinates_gold'][0]
# load predictions marked by official evaluator
marks = _load_predictions(base_dir/'formatted_predictions'/f'{split}_marked.tsv')
print('Total', len(marks))
errlen = len(marks[marks['mark']=='false'])
print('Accuracy',_format_fl(1-errlen/len(marks)))
print('Total Error', errlen, _format_fl(errlen/len(marks)))
#marks.loc[:,'answer'] = marks['answer'].apply(eval)
# load original prediction file with prediction details
preds = _load_predictions(preds_dir/f'{split}.tsv')
missing = len(marks)-len(preds)
print('Errors that have no output:', missing, _format_fl(missing/errlen),'of all errors')
# merge the marks with the ones that are predicted
combined = pd.merge(preds,marks,left_on='id',right_on='id',how='inner')
combined = pd.merge(combined,ref,left_on='id',right_on='id',how='inner')
combined = combined.sort_values(by='id',ascending=True, key=lambda y:y.map(lambda x:int(x.split('-')[1])))
combined.loc[:, 'answers'] = combined["answers"].apply(eval)
combined.loc[:, 'answer_coordinates'] = combined["answer_coordinates"].apply(eval)
combined.loc[:,'answer_coordinates'] = combined["answer_coordinates"].apply(lambda x:set(map(eval,x)))
print('Has prediction:', len(combined))
combined['answer_coordinates'][0]
# Analyze Error Subset with predictions
combined = combined[combined['mark']=='false']
print('Errors that have final output:', len(combined), f', {_format_fl(len(combined)/errlen)} of all errors')
# Coordinate Prediction Errors
# Didn't produce answer coordinates
empt_coor = len(combined[combined['answers'].map(len)==0]["answers"])
nonempt = combined[combined['answers'].map(len)!=0]
assert len(nonempt)+empt_coor == len(combined)
assert empt_coor==len(combined[combined['answer_coordinates'].map(len)==0]["answer_coordinates"])
print('Errors wo ans coordinate predictions:',empt_coor,_format_fl(empt_coor/errlen))
noempt_coor_err= len(nonempt[nonempt['answer_coordinates']!= nonempt['answer_coordinates_gold']])
all_coor_err=len(combined[combined['answer_coordinates']!= combined['answer_coordinates_gold']])
print('Total Coordinate Errors', all_coor_err,_format_fl(all_coor_err/errlen))
print('Errors w coor pred but wrong', noempt_coor_err,_format_fl(noempt_coor_err/errlen))
55/errlen
# Aggregation Errors
print('Total Aggregation Errors', len(combined[combined['pred_aggr']!= combined['gold_aggr']]))
wrong_op = len(nonempt[nonempt['pred_aggr']!= nonempt['gold_aggr']])
print('Aggregation Errors among the ans w coor predictions', wrong_op,_format_fl(wrong_op/errlen))
combined['answer_coordinates'][2]
cols = combined.columns.difference(['id'])
combined[cols]
d = combined.set_index('id')[['pred_aggr','answer_coordinates','answer','mark']].to_dict(orient='index')<jupyter_output><empty_output><jupyter_text>View Preprocessed TF Record<jupyter_code>import tensorflow as tf
raw_ds = tf.data.TFRecordDataset('results/wtq/tf_examples/random-split-1-dev.tfrecord', compression_type="GZIP")<jupyter_output><empty_output><jupyter_text>Read and Dump interactions to json for flask<jupyter_code>from tapas.protos import interaction_pb2
from google.protobuf.json_format import MessageToDict
import tensorflow as tf
def _to_df(dic):
    columns = [f"{v}-{k}" for d in dic['columns'] for k,v in d.items()]
    rows = [{columns[i]: ' '.join(list(d.values())) for i, d in enumerate(row['cells'])} for row in dic['rows']]
    df = pd.DataFrame(rows, columns=columns)
    return df
# view interaction protobuf
dics = {}
split = 'random-split-1-dev'
for value in tf.data.TFRecordDataset(f'results/wtq/interactions/{split}.tfrecord'):
    interaction = interaction_pb2.Interaction()
    interaction.ParseFromString(value.numpy())
    d = MessageToDict(interaction)
    d['table_id'] = d['table']['tableId']
    d['table'] = _to_df(d['table']).to_dict()
    dics[d['id'].rsplit('-',1)[0]] = d
d['table'].keys()
import json

with open(f'results/wtq/for_flask/{split}.json','w') as f:
    json.dump(dics,f)
print(len(dics))
_to_df(dics[0])<jupyter_output><empty_output><jupyter_text>Analyze Preprocessing Errors<jupyter_code>import json
with open ('results/wtq/tf_examples/random-split-1-dev_errors.json') as f:
    errors = json.load(f)
s=set()
t = 0
for k,v in errors.items():
    if k=='Invalid answer':
        s|=set(v) 
    print(k,len(v))
    t+=len(v)
print('total',t)
print(len(s))
# Invalid Answers Breakdown
with open('results/wtq/interactions/err_ids.json') as f:
    bkdns = json.load(f)
se= set()
split = 'random-split-1-dev.tsv'
for k,v in bkdns[split].items():
    v = list(map(lambda x:x.split('_')[0],v))
    bkdns[split][k] = v
    print(k)
    print(len(v))
    se|=set(v)
print(f'{split} total',len(se))
print(se.difference(s))
'nt-11953-0' in errors["Too many rows"] and 'nt-3899-0' in errors["Too many rows"]<jupyter_output>{'nt-3899-0', 'nt-11953-0'}<jupyter_text>Isolate the float value conversion errorsSee if we can still convert them to tf examples without float values<jupyter_code>err_key = "Cannot parse answer: [float_value: Unable to convert value to float]"
maj_err = set(bkdns['random-split-1-dev.tsv'][err_key])
from tapas.protos import interaction_pb2
import tensorflow.compat.v1 as tf
# view interaction protobuf
todo = []
split = 'random-split-1-dev'
corr = []

for value in tf.python_io.tf_record_iterator(f'results/wtq/interactions/{split}.tfrecord'):
    interaction = interaction_pb2.Interaction()
    interaction.ParseFromString(value)
    if interaction.id in maj_err:
        todo.append(interaction)
    elif len(corr)<3:
        corr.append(interaction)
# eager execution
# for value in tf.data.TFRecordDataset([f'results/wtq/interactions/{split}.tfrecord']):
#     interaction = interaction_pb2.Interaction()
#     interaction.ParseFromString(value.numpy())
#     if interaction.id in maj_err:
#         todo.append(interaction)
#     elif len(corr)<3:
#         corr.append(interaction)
        
#     d = MessageToDict(interaction)
#     d['table_id'] = d['table']['tableId']
#     d['table'] = _to_df(d['table']).to_dict()
#     dics[d['id'].rsplit('-',1)[0]] = d
from tapas.utils import tf_example_utils
from tapas.utils import pruning_utils
token_selector = pruning_utils.HeuristicExactMatchTokenSelector(
    'tapas_wtq_wikisql_sqa_inter_masklm_large_reset/vocab.txt', 
    512, 
    pruning_utils.SelectionType.COLUMN,
    # Only relevant for SQA where questions come in sequence
    use_previous_answer=True,
    use_previous_questions=True,
  )
config = tf_example_utils.ClassifierConversionConfig(
    vocab_file='tapas_wtq_wikisql_sqa_inter_masklm_large_reset/vocab.txt', 
    max_seq_length=512, 
    max_column_id=512, 
    max_row_id=512, 
    strip_column_names=False, 
    cell_trim_length=-1, 
    add_aggregation_candidates=False, 
    expand_entity_descriptions=False, 
    use_entity_title=False, 
    entity_descriptions_sentence_limit=5, 
    use_document_title=False, 
    update_answer_coordinates=False, 
    drop_rows_to_fit=False, 
    use_context_title=False, 
    trim_question_ids=False, 
    label_sampling_rate={}
)

converter = tf_example_utils.ToClassifierTensorflowExample(config)
examples = []
for ex in todo:
    ex = token_selector.annotated_interaction(ex)
    if len(ex.questions[0].text)>0:
        print(ex.questions[0].text)
    examples.append(converter.convert(ex,0))
features = MessageToDict(examples[0].features)
features['feature'].keys()
type(examples[0].features.feature['input_ids'])
from tapas.models import tapas_classifier_model
from tapas.models.bert import modeling
from tapas.utils import tasks, hparam_utils
from tapas.utils import text_utils


task = tasks.Task.WTQ
hparams = hparam_utils.get_hparams(task)
checkpoint_dir = 'results/wtq/model'
print(tf.train.latest_checkpoint(
    checkpoint_dir
))

bert_config_file = 'tapas_wtq_wikisql_sqa_inter_masklm_large_reset/bert_config.json'
bert_config = modeling.BertConfig.from_json_file(bert_config_file)
tapas_config = tapas_classifier_model.TapasClassifierConfig(
    bert_config= bert_config, 
    init_checkpoint='tapas_wtq_wikisql_sqa_inter_masklm_large_reset/model.ckpt', 
    learning_rate=1.93581e-05, 
    num_train_steps=50000, 
    num_warmup_steps=6448, 
    use_tpu=False, 
    positive_weight=10.0, 
    num_aggregation_labels=4, 
    num_classification_labels=0, 
    aggregation_loss_importance=1.0, 
    use_answer_as_supervision=True, 
    answer_loss_importance=1.0, 
    use_normalized_answer_loss=False, 
    huber_loss_delta=0.121194, 
    temperature=0.0352513, 
    agg_temperature=1.0, 
    use_gumbel_for_cells=False, 
    use_gumbel_for_agg=False, 
    average_approximation_function=tapas_classifier_model.AverageApproximationFunction.RATIO, 
    cell_select_pref=0.207951, 
    answer_loss_cutoff=0.664694, 
    grad_clipping=10.0, 
    max_num_rows=64, 
    max_num_columns=32, 
    average_logits_per_cell=False, 
    select_one_column=True, 
    allow_empty_column_selection=False, 
    disabled_features=[], 
    init_cell_selection_weights_to_zero=True, 
    disable_position_embeddings=False, 
    reset_position_index_per_cell=False, 
    disable_per_token_loss=False, 
    span_prediction=tapas_classifier_model.SpanPredictionMode(
          hparams.get('span_prediction',
                      tapas_classifier_model.SpanPredictionMode.NONE)),
    proj_value_length=None,
    reset_output_cls=False,
    classification_label_weight=None,
    mask_examples_without_labels=False,
    table_pruning_config_file=None)
model_fn = tapas_classifier_model.model_fn_builder(tapas_config)
from tapas.run_task_main import _predict
file_patterns='results/wtq/tf_examples/random-split-1-dev.tfrecord'
data_format='tfrecord'
compression_type='GZIP'
is_training=False
max_seq_length=512
max_predictions_per_seq=20
add_aggregation_function_id=True
add_classification_labels=False
add_answer=True
include_id=False
add_candidate_answers=False,
max_num_candidates=0
feature_types = {
      "input_ids":
          tf.io.FixedLenFeature([max_seq_length], tf.int64),
      "input_mask":
          tf.io.FixedLenFeature([max_seq_length], tf.int64),
      "segment_ids":
          tf.io.FixedLenFeature([max_seq_length], tf.int64),
      "column_ids":
          tf.io.FixedLenFeature([max_seq_length], tf.int64),
      "row_ids":
          tf.io.FixedLenFeature([max_seq_length], tf.int64),
      "prev_label_ids":
          tf.io.FixedLenFeature([max_seq_length],
                             tf.int64,
                             default_value=[0] * max_seq_length),
      "column_ranks":
          tf.io.FixedLenFeature(
              [max_seq_length],
              tf.int64,
              default_value=[0] * max_seq_length,
          ),
      "inv_column_ranks":
          tf.io.FixedLenFeature(
              [max_seq_length],
              tf.int64,
              default_value=[0] * max_seq_length,
          ),
      "numeric_relations":
          tf.io.FixedLenFeature([max_seq_length],
                             tf.int64,
                             default_value=[0] * max_seq_length),
  }
feature_types.update({
        "label_ids":
            tf.io.FixedLenFeature(
                [max_seq_length],
                tf.int64,
                default_value=[0] * max_seq_length,
            ),
    })
feature_types.update({
        "question_id_ints":
            tf.io.FixedLenFeature([text_utils.DEFAULT_INTS_LENGTH],
                               tf.int64,
                               default_value=[0] *
                               text_utils.DEFAULT_INTS_LENGTH),
    })

if add_aggregation_function_id:
      feature_types.update({
          "aggregation_function_id": tf.io.FixedLenFeature([1], tf.int64),
      })
if add_classification_labels:
      feature_types.update({
          "classification_class_index": tf.io.FixedLenFeature([1], tf.int64),
      })
    # Features for the weakly supervised setting.
if add_answer:
      feature_types.update({
          "numeric_values":
              tf.io.FixedLenFeature(
                  [max_seq_length],
                  tf.float32,
                  default_value=[0] * max_seq_length,
              ),
          "numeric_values_scale":
              tf.io.FixedLenFeature(
                  [max_seq_length],
                  tf.float32,
                  default_value=[0] * max_seq_length,
              ),
          "answer":
              tf.io.FixedLenFeature(
                  [1],
                  tf.float32,
                  default_value=[0],
              ),
      })
  
def build_parser_function(feature_types,
                          params):
  """Returns a parse function that can be used by read_dataset."""
  del params

  def parse_examples(serialized_examples):
    features = tf.io.parse_single_example(serialized_examples, feature_types)
    # tf.Example only supports tf.int64, but the TPU only supports tf.int32.
    # So cast all int64 to int32.
    for name in list(features.keys()):
      t = features[name]
      if t.dtype == tf.int64:
        t = tf.cast(t, tf.int32)
        features[name] = t
    return features

  return parse_examples

def _parse_fn(serialized_example):
    features = dict(
        build_parser_function(feature_types,
                                      params)(serialized_example))
    return features
example = examples[0]
ex_str = example.SerializeToString()
def cast_int(fts):
    for name in list(fts.keys()):
        t = fts[name]
        if t.dtype == tf.int64:
            t = tf.cast(t, tf.int32)
            fts[name] = t
    return fts
serialized = []
for example in examples:
    serialized.append(example.SerializeToString())
#fts = tf.io.parse_single_example(ex_str, feature_types)
fts = cast_int(tf.io.parse_example(serialized, feature_types))

parsed = _parse_fn(ex_str)
def input_fn(features, params):
    parse_fn = _parse_fn
    dataset = tf.data.Dataset.from_tensor_slices(features)
    return dataset
import functools
predict_input_fn = functools.partial(input_fn,features=fts)
# To get a single element from tapas's V1 dataset
#('tensorflow.python.data.ops.dataset_ops.DatasetV1Adapter')
# tf.data.experimental.get_single_element(
#     dataset
# )

# predict_input_fn = some sort of input fn that iterates through the dataset<jupyter_output><empty_output><jupyter_text>taken from Traceback:```    tapas/models/tapas_classifier_model.py:1170        tapas/datasets/table_dataset.py:190            tapas/datasets/dataset.py:78```<jupyter_code>ds = tf.data.Dataset.from_tensor_slices(fts)
run_config = tf.compat.v1.estimator.tpu.RunConfig(
      cluster=None,
      master=None,
      model_dir='results/wtq/model',
      tf_random_seed=None,
      save_checkpoints_steps=1000,
      keep_checkpoint_max=5,
      keep_checkpoint_every_n_hours=4.0,
      tpu_config=tf.compat.v1.estimator.tpu.TPUConfig(
          iterations_per_loop=1000,
          num_shards=8,
          per_host_input_for_training=3))
estimator = tf.compat.v1.estimator.tpu.TPUEstimator(
      params={'gradient_accumulation_steps': 1},
      use_tpu=False,
      model_fn=model_fn,
      config=run_config,
      train_batch_size=512 // 1,
      eval_batch_size=None,
      predict_batch_size=32)
checkpoint = estimator.latest_checkpoint()
def test_input_fn(params):
    return tf.data.Dataset.from_tensor_slices(fts)
next(estimator.predict(input_fn=test_input_fn))
%debug
for k,v in ds[0].items():
    print(k,v.shape)
expd = {k: v[None,:] for k,v in ds[0].items()}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12382986743733115, 'fraction_numerical': 0.015141044166614312, 'mean_word_length': 4.075892857142857, 'pattern_counts': {'":': 16, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '606354', 'license': 'Apache-2.0', 'n_tokens_mistral': 5608, 'n_tokens_neox': 5378, 'n_words': 833, 'path': 'visualize.ipynb', 'repo_name': 'jeffersonHsieh/tapas', 'size': '15917'}