README update

README.md

@@ -25,11 +25,12 @@ Note that this metric is reference-free and do not require training. It is ready
 ## Quick Start
 ### install
 ```bash
-# assumes torch, transformers, pandas, tqdm, fire and datasets are installed
 pip install evaluate
-# pip install -r requirments.txt
 ```
 ### example
+You can clone this space and run <code>python test_harim_score.py [--pretrained_name CKPTNAME_FOR_S2SLM] </code> or try below. <br />
+(running on CPU is possible, but expected to be too slow for use.)
+
 ```python
 import evaluate
 from pprint import pprint
@@ -46,13 +47,8 @@ articles = [art] * len(summaries)
 
 scorer = evaluate.load('NCSOFT/harim_plus')
 scores = scorer.compute(predictions = summaries, references = articles) # use_aggregator=False, bsz=32, return_details=False, tokenwise_score=False)
-pprint(scores)
->>> [1.8230078220367432,
-            1.5361897945404053,
-            1.806436538696289,
-            1.7360382080078125
-      ]
-
+pprint([round(s,4) for s in scores])
+>>> [2.7096, 3.7338, 2.669, 2.4039, 2.3759]
 ```
 
 ## Powering HaRiM+ score with other summarization model checkpoints

harim_plus.py

@@ -51,7 +51,6 @@ HaRiM+ is proved effective for benchmarking summarization systems (system-level
 NOTE that for HaRiM+...
 * predictions = summaries (List[str])
 * references = articles (List[str])
-
 """
 
 _KWARGS_DESCRIPTION = """

test_harim_score.py

@@ -81,8 +81,9 @@ SUMM_CHUNKS = [
      "lisa mcelroy , 50 , who teaches legal writing at drexel university , reportedly did not send the ` inappropriate ' message on march 31 . when recipients clicked the enclosed link , they were allegedly directed to a video of ' a woman engaging in a sexually explicit act ' . mcelroy appeared on the popular game show in 2010 with then-host meredith vieira but lost the game after reaching just $ 12,500 . along with teaching law , mcelroy is also an accomplished author with a number of published biographies and children 's books . has been placed on leave while school investigates .", # negation
      ]# pos2
 ]
-def test():
-    scorer = evaluate.load('NCSOFT/harim_plus')
+def test(pretrained_name='facebook/bart-large-cnn'):
+    print(f"you can test other ckpts compatible with transformers.AutoModelForSeq2SeqLM class \n\te.g. \n\t\tpython test.py --pretrained_name Yale-LILY/brio-cnndm-cased")
+    scorer = evaluate.load('NCSOFT/harim_plus', pretrained_name=pretrained_name)
     # scorer = Harimplus_Scorer(pretrained_name='facebook/bart-large-cnn')
     table = ['neg_rnn', 'neg_1', 'neg_1_', 'neg2', 'pos1', 'pos2']
     keys_neg = ['ref', 'self-gen', 'summary-infactual', 'ref+wrong subj', 'ref+negation']
@@ -106,7 +107,6 @@ def test():
 
 '''https://arxiv.org/abs/2211.12118
 
-tablename='neg_rnn'
 
 tablename='neg_1_' # main
 {'ref': 1.7218,
@@ -122,6 +122,8 @@ tablename='neg_1'
  'ref+wrong subj': 3.7903,
  'ref+negation': 3.3877,
 
+
+ tablename='neg_rnn'
 {'ref': 2.7096,
 'self-gen': 3.7338,
 'summary-infactual': 2.669}