From 094505ace564e854806a8432e0b73e9c0bace954 Mon Sep 17 00:00:00 2001 From: Sun <53498297+floatingbigcat@users.noreply.github.com> Date: Mon, 2 Oct 2023 16:15:01 +0900 Subject: [PATCH] Update utils.py --- megatron/model/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/megatron/model/utils.py b/megatron/model/utils.py index 5f09dfba8..975080d06 100644 --- a/megatron/model/utils.py +++ b/megatron/model/utils.py @@ -66,7 +66,7 @@ def get_params_groups(module, neox_args): pretrain_no_weight_decay_params = {"params": [], "weight_decay": 0.0} for name, module_ in module.named_modules(): - # Put module into fintune group if name contains any of finetune_groups_key_words + # Grammar Sugar for putting module into fintune group if name contains any of finetune_groups_key_words if sum([kw in name for kw in neox_args.finetune_groups_key_words]): finetune_weight_decay_params, finetune_no_weight_decay_params = update_params_for_weight_decay( module_, finetune_weight_decay_params, finetune_no_weight_decay_params,neox_args.weight_decay