Improve hebrew normalizer

src/visual_eval/evaluator.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+import re
 
 from hebrew import Hebrew
 from jiwer import process_words
@@ -23,7 +24,9 @@ class HebrewTextNormalizer(BasicTextNormalizer):
             {ord(c): None for c in superfluous_chars_to_remove}
         )
 
-        self.quotes_translator = str.maketrans({ord(c): None for c in "\"'"})
+        quotes_str = "\"'״׳"
+        self.quotes_translator = str.maketrans({ord(c): None for c in quotes_str})
+        self.pre_quote_hyphen_removal_pattern = re.compile(f"-[{quotes_str}]")
 
     def __remove_niqqud(self, text: str) -> str:
         return Hebrew(text).no_niqqud().string
@@ -32,6 +35,7 @@ class HebrewTextNormalizer(BasicTextNormalizer):
         return text.translate(self.superfluous_hebrew_unicode_symbols_translator)
 
     def __remove_quotes(self, text: str) -> str:
+        text = self.pre_quote_hyphen_removal_pattern.sub("", text)
         return text.translate(self.quotes_translator)
 
     def __call__(self, text):