added pad token to tokenizer

CERC-AAI · Oct 18, 2023 · 6999b7c · 6999b7c
1 parent fbfb277
commit 6999b7c
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/megatron/tokenizer/tokenizer.py b/megatron/tokenizer/tokenizer.py
@@ -231,6 +231,8 @@ def __init__(self, vocab_file):
         self.eod_id = self.tokenizer.token_to_id("<|endoftext|>")
         self.pad_id = self.tokenizer.token_to_id("<|padding|>")
         self.pad_token_id = self.pad_id
+        self._pad_token = self.pad_id  
+        self.padding_side = "right"
 
     @property
     def vocab_size(self):