add normalization to enhance numerical stability

DataCanvasIO · Jul 19, 2024 · 01fafb7 · 01fafb7
1 parent d70675a
commit 01fafb7
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 0 deletions.
diff --git a/deeptables/models/config.py b/deeptables/models/config.py
@@ -19,6 +19,7 @@ class ModelConfig(collections.namedtuple('ModelConfig',
                                           'cat_remain_numeric',
                                           'auto_encode_label',
                                           'auto_imputation',
+                                          'auto_scale',
                                           'auto_discrete',
                                           'auto_discard_unique',
                                           'apply_gbm_features',
@@ -68,6 +69,7 @@ def __new__(cls,
                 cat_remain_numeric=True,
                 auto_encode_label=True,
                 auto_imputation=True,
+                auto_scale=False,
                 auto_discrete=False,
                 auto_discard_unique=True,
                 apply_gbm_features=False,
@@ -164,6 +166,7 @@ def __new__(cls,
                                                cat_remain_numeric,
                                                auto_encode_label,
                                                auto_imputation,
+                                               auto_scale,
                                                auto_discrete,
                                                auto_discard_unique,
                                                apply_gbm_features,

diff --git a/deeptables/models/preprocessor.py b/deeptables/models/preprocessor.py
@@ -174,6 +174,8 @@ def fit_transform(self, X, y, copy_data=True):
 
         if self.config.auto_imputation:
             X = self._imputation(X)
+        if self.config.auto_scale:
+            X = self._standard_scale(X)
         if self.config.auto_encode_label:
             X = self._categorical_encoding(X)
         if self.config.auto_discrete:
@@ -393,6 +395,17 @@ def _categorical_encoding(self, X):
         logger.info(f'Categorical encoding taken {time.time() - start}s')
         return X
 
+    def _standard_scale(self, X):
+        start = time.time()
+        logger.info('Standard scale ...')
+        vars = self.get_continuous_columns()
+        ss = self.transformers.MinMaxScalerTransformer(vars)
+        ss.fit(X)
+        X = ss.transform(X)
+        self.X_transformers['standard_scale'] = ss
+        logger.info(f'Standard scale taken {time.time() - start}s')
+        return X
+
     def _discretization(self, X):
         start = time.time()
         logger.info('Data discretization...')