update

ctc_eval.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 """TODO: Add a description here."""
 
+from typing import final
 import evaluate
 import datasets
 
@@ -89,11 +90,24 @@ class CTC_Eval(evaluate.EvaluationModule):
         def install(package):
             subprocess.check_call([sys.executable, "-m", "pip", "install", package])
         
-        install('ctc-score')
-        from ctc_score import StyleTransferScorer, SummarizationScorer, DialogScorer
-        self.scorer = SummarizationScorer(align='D-cnndm')
+        
+        try:
+            from ctc_score import StyleTransferScorer, SummarizationScorer, DialogScorer
+        except:
+            print('ctc package is not installed. installing...')
+            install('ctc-score')
+
+        if self.config_name == 'default':
+            self.config_name = 'D-cnndm,consistency'
 
-        self.compute(references=['hello world'], predictions=['hi world'])
+        model_name, self.aspect = self.config_name.split(',')
+        if self.aspect in ['consistency, relevance']:
+            self.scorer = SummarizationScorer(align=model_name)
+        elif self.aspect in ['preservation']:
+            self.scorer = StyleTransferScorer(align=model_name)
+        elif self.aspect in ['engagingness', 'groundedness']:
+            self.scorer = DialogScorer(align=model_name)
+        
 
     def _compute(self, predictions, references):
         """Returns the scores"""
@@ -104,5 +118,5 @@ class CTC_Eval(evaluate.EvaluationModule):
         print(references)
         ctc_score = self.scorer.score(doc=references[0], refs=[], hypo=predictions[0], aspect='consistency')
         return {
-            "ctc_score": [ctc_score]
+            "ctc_score": ctc_score
         }