Adding the convert scripts which will now prevent converting models (#59

)

* Adding the convert scripts which will now prevent converting models

in case they will trigger warnigns in the `transformers` side.
Even if the model is perfectly fine, core maintainers fear an influx
of opened issues.

This is perfectly legit.
On the `transformers` side fixes are on the way: huggingface/transformers#20042

We can wait for this PR to hit `main` before communicating super widely.

In the meantime this script of convertion will now prevent converting
models that would trigger such warnings (so the output of the script
**will** depend on the `transformers` freshness.

* Adding a nicer diff for the error when reloading.

bindings/python/convert.py

@@ -2,18 +2,18 @@
 import json
 import os
 import shutil
-from tempfile import TemporaryDirectory
 from collections import defaultdict
 from inspect import signature
-from typing import Optional, List
+from tempfile import TemporaryDirectory
+from typing import Dict, List, Optional
 
 import torch
 
-from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download, get_repo_discussions
+from huggingface_hub import CommitInfo, CommitOperationAdd, Discussion, HfApi, hf_hub_download
 from huggingface_hub.file_download import repo_folder_name
+from safetensors.torch import save_file
 from transformers import AutoConfig
 from transformers.pipelines.base import infer_framework_load_model
-from safetensors.torch import save_file
 
 
 class AlreadyExists(Exception):
@@ -30,15 +30,18 @@ def shared_pointers(tensors):
             failing.append(names)
     return failing
 
+
 def check_file_size(sf_filename: str, pt_filename: str):
     sf_size = os.stat(sf_filename).st_size
     pt_size = os.stat(pt_filename).st_size
 
     if (sf_size - pt_size) / pt_size > 0.01:
-        raise RuntimeError(f"""The file size different is more than 1%:
+        raise RuntimeError(
+            f"""The file size different is more than 1%:
          - {sf_filename}: {sf_size}
          - {pt_filename}: {pt_size}
-         """)
+         """
+        )
 
 
 def rename(pt_filename: str) -> str:
@@ -47,12 +50,13 @@ def rename(pt_filename: str) -> str:
     return local
 
 
-def convert_multi(model_id: str) -> List["CommitOperationAdd"]:
+def convert_multi(model_id: str, folder: str) -> List["CommitOperationAdd"]:
     filename = hf_hub_download(repo_id=model_id, filename="pytorch_model.bin.index.json")
     with open(filename, "r") as f:
         data = json.load(f)
 
     filenames = set(data["weight_map"].values())
+    local_filenames = []
     for filename in filenames:
         cached_filename = hf_hub_download(repo_id=model_id, filename=filename)
         loaded = torch.load(cached_filename)
@@ -71,7 +75,9 @@ def convert_multi(model_id: str) -> List["CommitOperationAdd"]:
         json.dump(newdata, f)
     local_filenames.append(index)
 
-    operations = [CommitOperationAdd(path_in_repo=local.split("/")[-1], path_or_fileobj=local) for local in local_filenames]
+    operations = [
+        CommitOperationAdd(path_in_repo=local.split("/")[-1], path_or_fileobj=local) for local in local_filenames
+    ]
 
     return operations
 
@@ -97,16 +103,34 @@ def convert_single(model_id: str, folder: str) -> List["CommitOperationAdd"]:
     operations = [CommitOperationAdd(path_in_repo=sf_filename, path_or_fileobj=local)]
     return operations
 
+
+def create_diff(pt_infos: Dict[str, List[str]], sf_infos: Dict[str, List[str]]) -> str:
+    errors = []
+    for key in ["missing_keys", "mismatched_keys", "unexpected_keys"]:
+        pt_set = set(pt_infos[key])
+        sf_set = set(sf_infos[key])
+
+        pt_only = pt_set - sf_set
+        sf_only = sf_set - pt_set
+
+        if pt_only:
+            errors.append(f"{key} : PT warnings contain {pt_only} which are not present in SF warnings")
+        if sf_only:
+            errors.append(f"{key} : SF warnings contain {sf_only} which are not present in PT warnings")
+    return "\n".join(errors)
+
+
 def check_final_model(model_id: str, folder: str):
     config = hf_hub_download(repo_id=model_id, filename="config.json")
     shutil.copy(config, os.path.join(folder, "config.json"))
     config = AutoConfig.from_pretrained(folder)
 
-    _, pt_model = infer_framework_load_model(model_id, config)
-    _, sf_model = infer_framework_load_model(folder, config)
+    _, (pt_model, pt_infos) = infer_framework_load_model(model_id, config, output_loading_info=True)
+    _, (sf_model, sf_infos) = infer_framework_load_model(folder, config, output_loading_info=True)
 
-    pt_model = pt_model
-    sf_model = sf_model
+    if pt_infos != sf_infos:
+        error_string = create_diff(pt_infos, sf_infos)
+        raise ValueError(f"Different infos when reloading the model: {error_string}")
 
     pt_params = pt_model.state_dict()
     sf_params = sf_model.state_dict()
@@ -134,7 +158,6 @@ def check_final_model(model_id: str, folder: str):
     if "image" in sig.parameters:
         kwargs["image"] = pixel_values
 
-
     if torch.cuda.is_available():
         pt_model = pt_model.cuda()
         sf_model = sf_model.cuda()
@@ -146,6 +169,7 @@ def check_final_model(model_id: str, folder: str):
     torch.testing.assert_close(sf_logits, pt_logits)
     print(f"Model {model_id} is ok !")
 
+
 def previous_pr(api: "HfApi", model_id: str, pr_title: str) -> Optional["Discussion"]:
     try:
         discussions = api.get_repo_discussions(repo_id=model_id)
@@ -156,7 +180,7 @@ def previous_pr(api: "HfApi", model_id: str, pr_title: str) -> Optional["Discuss
             return discussion
 
 
-def convert(api: "HfApi", model_id: str, force: bool=False) -> Optional["CommitInfo"]:
+def convert(api: "HfApi", model_id: str, force: bool = False) -> Optional["CommitInfo"]:
     pr_title = "Adding `safetensors` variant of this model"
     info = api.model_info(model_id)
     filenames = set(s.rfilename for s in info.siblings)

bindings/python/convert_all.py

@@ -1,18 +1,30 @@
 """Simple utility tool to convert automatically most downloaded models"""
+from convert import AlreadyExists, convert
 from huggingface_hub import HfApi, ModelFilter, ModelSearchArguments
-from convert import convert, AlreadyExists
+from transformers import AutoConfig
 
 
 if __name__ == "__main__":
     api = HfApi()
     args = ModelSearchArguments()
 
-    total = 100
-    models = list(api.list_models(filter=ModelFilter(library=args.library.Transformers), sort="downloads", direction=-1))[:total]
+    total = 50
+    models = list(
+        api.list_models(filter=ModelFilter(library=args.library.Transformers), sort="downloads", direction=-1)
+    )[:total]
 
     correct = 0
     errors = set()
     for model in models:
+        model = api.model_info(model.modelId, files_metadata=True)
+        size = None
+        for sibling in model.siblings:
+            if sibling.rfilename == "pytorch_model.bin":
+                size = sibling.size
+        if size is None or size > 2_000_000_000:
+            print(f"[{model.downloads}] Skipping {model.modelId} (too large {size})")
+            continue
+
         model_id = model.modelId
         print(f"[{model.downloads}] {model.modelId}")
         try:
@@ -22,10 +34,10 @@
             correct += 1
             print(e)
         except Exception as e:
-            errors.add( model_id)
+            config = AutoConfig.from_pretrained(model_id)
+            errors.add(config.__class__.__name__)
             print(e)
 
-
     print(f"Errors: {errors}")
     print(f"File size is difference {len(errors)}")
     print(f"Correct rate {correct}/{total} ({correct/total * 100:.2f}%)")