Merge branch 'jbarker/data_preprocessing_bug' into 'main'

Fix off by one error in document preprocessing

See merge request ADLR/megatron-lm!705

tools/preprocess_data.py

@@ -95,6 +95,7 @@ def encode(self, json_line):
                     sentence_lens.append(len(sentence_ids))
             if len(doc_ids) > 0 and self.args.append_eod:
                 doc_ids.append(Encoder.tokenizer.eod)
+                sentence_lens[-1] += 1
             ids[key] = doc_ids
             lens[key] = sentence_lens
         return ids, lens, len(json_line)