Files changed (2) hide show
  1. naacl_demo/demo_utils.py +21 -21
  2. naacl_demo/main.py +3 -3
naacl_demo/demo_utils.py CHANGED
@@ -80,20 +80,20 @@ def prompt_boolq(passage, question, pattern):
80
 
81
  def advantage_text(advantage):
82
  model_type = (
83
- """<span style="color: #4B0082">Head</span>"""
84
  if advantage < 0
85
- else """<span style="color: #daa520">Prompting</span>"""
86
  )
87
- return f"""<b>{model_type}</b> advantage: <b>{abs(advantage):.2f}</b> data points"""
88
 
89
 
90
  def average_advantage_text(advantage):
91
  model_type = (
92
- """<span style="color: #4B0082">head</span>"""
93
  if advantage < 0
94
- else """<span style="color: #daa520">prompting</span>"""
95
  )
96
- return f"""<b>Average {model_type}</b> advantage: <b>{abs(advantage):.2f}</b> data points"""
97
 
98
 
99
  def naming_convention(task, seed, pvp_index=None, neutral=False):
@@ -296,13 +296,13 @@ def plot_polygons_bokeh(task, training_points, clf_results, pvp_results, clf_col
296
  middle_y = (full_range[0] + full_range[1]) / 2
297
 
298
  fig = figure(plot_height=400, plot_width=800, max_height=400, max_width=800,
299
- x_axis_type="log" if x_log_scale else "linear", title="Performance over training subset sizes of head and prompting methods")
300
 
301
- fig.circle(training_points, clf_results, color=clf_colors[0], legend="head run")
302
- fig.circle(training_points, pvp_results, color=pvp_colors[0], legend="prompting run")
303
  fig.line(training_points, clf_results, color=clf_colors[0], alpha=1)
304
  fig.line(training_points, pvp_results, color=pvp_colors[0], alpha=1)
305
- fig.xaxis.axis_label = "training subset size"
306
  fig.yaxis.axis_label = task_metrics[task]
307
  fig.patch(
308
  [training_points[0], training_points[0], training_points[-1], training_points[-1]],
@@ -310,7 +310,7 @@ def plot_polygons_bokeh(task, training_points, clf_results, pvp_results, clf_col
310
  color="black",
311
  fill_alpha=0,
312
  line_width=0,
313
- legend="comparison region",
314
  hatch_alpha=0.14,
315
  hatch_scale=40,
316
  hatch_pattern="/",
@@ -356,7 +356,7 @@ def plot_polygons_bokeh(task, training_points, clf_results, pvp_results, clf_col
356
  location=training_points[-1], dimension="height", line_color="black", line_width=2.5, line_dash="dashed"
357
  )
358
  end_label = Label(
359
- x=training_points[-1], y=middle_y, text="End of dataset", angle=90, angle_units="deg", text_align="center"
360
  )
361
  fig.renderers.extend([vline, end_label])
362
 
@@ -374,12 +374,12 @@ def plot_three_polygons_bokeh(
374
  middle_y = (full_range[0] + full_range[1]) / 2
375
 
376
  fig = figure(plot_height=400, plot_width=800, max_height=400, max_width=800,
377
- x_axis_type="log" if x_log_scale else "linear", title="Performance over training subset sizes of head, prompting and prompting with a null verbalizer")
378
- fig.xaxis.axis_label = "training subset size"
379
  fig.yaxis.axis_label = task_metrics[task]
380
- fig.circle(training_points, clf_results, color=clf_colors[0], legend="head run")
381
- fig.circle(training_points, pvp_results, color=pvp_colors[0], legend="prompting run")
382
- fig.circle(training_points, ctl_results, color=ctl_colors[0], legend="null verbalizer run")
383
  fig.line(training_points, clf_results, color=clf_colors[0], alpha=1)
384
  fig.line(training_points, pvp_results, color=pvp_colors[0], alpha=1)
385
  fig.line(training_points, ctl_results, color=ctl_colors[0], alpha=1)
@@ -390,7 +390,7 @@ def plot_three_polygons_bokeh(
390
  color="black",
391
  fill_alpha=0,
392
  line_width=0,
393
- legend="comparison region",
394
  hatch_alpha=0.14,
395
  hatch_scale=40,
396
  hatch_pattern="/",
@@ -447,7 +447,7 @@ def plot_three_polygons_bokeh(
447
  location=training_points[-1], dimension="height", line_color="black", line_width=2.5, line_dash="dashed"
448
  )
449
  end_label = Label(
450
- x=training_points[-1], y=middle_y, text="End of dataset", angle=90, angle_units="deg", text_align="center"
451
  )
452
  fig.renderers.extend([vline, end_label])
453
 
@@ -458,7 +458,7 @@ def plot_three_polygons_bokeh(
458
 
459
  def pattern_graph(task):
460
  fig = figure(plot_height=400, plot_width=800, max_height=400, max_width=800, x_axis_type="log", title="Performance over training subset sizes of different prompt patterns")
461
- fig.xaxis.axis_label = "training subset size"
462
  fig.yaxis.axis_label = task_metrics[task]
463
  url = f"https://raw.githubusercontent.com/TevenLeScao/pet/master/exported_results/{task.lower()}/wandb_export.csv"
464
  df = pd.read_csv(url)
@@ -488,7 +488,7 @@ def pattern_graph(task):
488
  y_max = list([np.max(training_point_df[task_metrics[task]]) for training_point, training_point_df in gby_training_points])
489
  y_min = list([np.min(training_point_df[task_metrics[task]]) for training_point, training_point_df in gby_training_points])
490
  y = list([np.median(training_point_df[task_metrics[task]]) for training_point, training_point_df in gby_training_points])
491
- fig.circle(x, y, color=pattern_colors[i], alpha=1, legend=f"Pattern {i}")
492
  fig.line(x, y, color=pattern_colors[i], alpha=1)
493
  fig.varea(x=x, y1=y_max, y2=y_min, color=pattern_colors[i], alpha=0.11)
494
  # source = ColumnDataSource(data=dict(base=x, lower=y_min, upper=y_max))
 
80
 
81
  def advantage_text(advantage):
82
  model_type = (
83
+ """<span style="color: #4B0082">分类头法</span>"""
84
  if advantage < 0
85
+ else """<span style="color: #daa520">提示法</span>"""
86
  )
87
+ return f"""<b>{model_type}</b> 优势: <b>{abs(advantage):.2f}</b> 条样本"""
88
 
89
 
90
  def average_advantage_text(advantage):
91
  model_type = (
92
+ """<span style="color: #4B0082">分类头法</span>"""
93
  if advantage < 0
94
+ else """<span style="color: #daa520">提示法</span>"""
95
  )
96
+ return f"""<b>Average {model_type}</b> 优势: <b>{abs(advantage):.2f}</b> 条样本"""
97
 
98
 
99
  def naming_convention(task, seed, pvp_index=None, neutral=False):
 
296
  middle_y = (full_range[0] + full_range[1]) / 2
297
 
298
  fig = figure(plot_height=400, plot_width=800, max_height=400, max_width=800,
299
+ x_axis_type="log" if x_log_scale else "linear", title="分类头法及提示法在各规模的训练子集上的性能")
300
 
301
+ fig.circle(training_points, clf_results, color=clf_colors[0], legend="分类头法")
302
+ fig.circle(training_points, pvp_results, color=pvp_colors[0], legend="提示法")
303
  fig.line(training_points, clf_results, color=clf_colors[0], alpha=1)
304
  fig.line(training_points, pvp_results, color=pvp_colors[0], alpha=1)
305
+ fig.xaxis.axis_label = "训练子集规模"
306
  fig.yaxis.axis_label = task_metrics[task]
307
  fig.patch(
308
  [training_points[0], training_points[0], training_points[-1], training_points[-1]],
 
310
  color="black",
311
  fill_alpha=0,
312
  line_width=0,
313
+ legend="比较区域",
314
  hatch_alpha=0.14,
315
  hatch_scale=40,
316
  hatch_pattern="/",
 
356
  location=training_points[-1], dimension="height", line_color="black", line_width=2.5, line_dash="dashed"
357
  )
358
  end_label = Label(
359
+ x=training_points[-1], y=middle_y, text="数据集总大小", angle=90, angle_units="deg", text_align="center"
360
  )
361
  fig.renderers.extend([vline, end_label])
362
 
 
374
  middle_y = (full_range[0] + full_range[1]) / 2
375
 
376
  fig = figure(plot_height=400, plot_width=800, max_height=400, max_width=800,
377
+ x_axis_type="log" if x_log_scale else "linear", title="分类头法、提示法以及空言语器提示法在各规模的训练子集上的性能")
378
+ fig.xaxis.axis_label = "训练子集规模"
379
  fig.yaxis.axis_label = task_metrics[task]
380
+ fig.circle(training_points, clf_results, color=clf_colors[0], legend="分类头法")
381
+ fig.circle(training_points, pvp_results, color=pvp_colors[0], legend="提示法")
382
+ fig.circle(training_points, ctl_results, color=ctl_colors[0], legend="空言语器提示法")
383
  fig.line(training_points, clf_results, color=clf_colors[0], alpha=1)
384
  fig.line(training_points, pvp_results, color=pvp_colors[0], alpha=1)
385
  fig.line(training_points, ctl_results, color=ctl_colors[0], alpha=1)
 
390
  color="black",
391
  fill_alpha=0,
392
  line_width=0,
393
+ legend="比较区域",
394
  hatch_alpha=0.14,
395
  hatch_scale=40,
396
  hatch_pattern="/",
 
447
  location=training_points[-1], dimension="height", line_color="black", line_width=2.5, line_dash="dashed"
448
  )
449
  end_label = Label(
450
+ x=training_points[-1], y=middle_y, text="数据集总大小", angle=90, angle_units="deg", text_align="center"
451
  )
452
  fig.renderers.extend([vline, end_label])
453
 
 
458
 
459
  def pattern_graph(task):
460
  fig = figure(plot_height=400, plot_width=800, max_height=400, max_width=800, x_axis_type="log", title="Performance over training subset sizes of different prompt patterns")
461
+ fig.xaxis.axis_label = "训练子集规模"
462
  fig.yaxis.axis_label = task_metrics[task]
463
  url = f"https://raw.githubusercontent.com/TevenLeScao/pet/master/exported_results/{task.lower()}/wandb_export.csv"
464
  df = pd.read_csv(url)
 
488
  y_max = list([np.max(training_point_df[task_metrics[task]]) for training_point, training_point_df in gby_training_points])
489
  y_min = list([np.min(training_point_df[task_metrics[task]]) for training_point, training_point_df in gby_training_points])
490
  y = list([np.median(training_point_df[task_metrics[task]]) for training_point, training_point_df in gby_training_points])
491
+ fig.circle(x, y, color=pattern_colors[i], alpha=1, legend=f"模式 {i}")
492
  fig.line(x, y, color=pattern_colors[i], alpha=1)
493
  fig.varea(x=x, y1=y_max, y2=y_min, color=pattern_colors[i], alpha=0.11)
494
  # source = ColumnDataSource(data=dict(base=x, lower=y_min, upper=y_max))
naacl_demo/main.py CHANGED
@@ -100,7 +100,7 @@ advantage_tabs = []
100
  advantage_all_figures = Tabs(tabs=advantage_tabs)
101
 
102
  advantage_box = Div(
103
- text="Click within the comparison region to compute the data advantage for a performance level",
104
  width=text_width,
105
  style=box_style,
106
  sizing_mode="scale_width",
@@ -200,13 +200,13 @@ def on_integrate_click():
200
  advantage_box.text = average_advantage_text(average_advantage)
201
 
202
 
203
- integrate = Button(width=175, max_width=175, label="Integrate over the whole region!")
204
  integrate.align = "center"
205
  integrate.on_click(on_integrate_click)
206
 
207
 
208
  def on_tab_change(attr, old, new):
209
- advantage_box.text = "Click within the comparison region to compute the data advantage for a performance level"
210
 
211
 
212
  advantage_all_figures.on_change('active', on_tab_change)
 
100
  advantage_all_figures = Tabs(tabs=advantage_tabs)
101
 
102
  advantage_box = Div(
103
+ text="在比较区域内点击某点以计算该点对应的性能点上的数据优势",
104
  width=text_width,
105
  style=box_style,
106
  sizing_mode="scale_width",
 
200
  advantage_box.text = average_advantage_text(average_advantage)
201
 
202
 
203
+ integrate = Button(width=175, max_width=175, label="对整个区域进行积分!")
204
  integrate.align = "center"
205
  integrate.on_click(on_integrate_click)
206
 
207
 
208
  def on_tab_change(attr, old, new):
209
+ advantage_box.text = "在比较区域内点击某点以计算该点对应的性能点上的数据优势"
210
 
211
 
212
  advantage_all_figures.on_change('active', on_tab_change)