SAM2 extra init changes

docs/source/en/_toctree.yml

@@ -832,6 +832,8 @@
         title: Perceiver
       - local: model_doc/pix2struct
         title: Pix2Struct
+      - local: model_doc/sam2
+        title: SAM2
       - local: model_doc/sam
         title: Segment Anything
       - local: model_doc/siglip

src/transformers/__init__.py

@@ -67,6 +67,7 @@
         "ToolCollection",
         "launch_gradio_demo",
         "load_tool",
+        "stream_to_gradio",
     ],
     "audio_utils": [],
     "benchmark": [],
@@ -681,6 +682,13 @@
         "SamPromptEncoderConfig",
         "SamVisionConfig",
     ],
+    "models.sam2": [
+        "Sam2Config",
+        "Sam2MaskDecoderConfig",
+        "Sam2Processor",
+        "Sam2PromptEncoderConfig",
+        "Sam2VisionConfig",
+    ],
     "models.seamless_m4t": [
         "SeamlessM4TConfig",
         "SeamlessM4TFeatureExtractor",
@@ -1179,6 +1187,7 @@
     _import_structure["models.pvt"].extend(["PvtImageProcessor"])
     _import_structure["models.rt_detr"].extend(["RTDetrImageProcessor"])
     _import_structure["models.sam"].extend(["SamImageProcessor"])
+    _import_structure["models.sam2"].extend(["Sam2ImageProcessor"])
     _import_structure["models.segformer"].extend(["SegformerFeatureExtractor", "SegformerImageProcessor"])
     _import_structure["models.seggpt"].extend(["SegGptImageProcessor"])
     _import_structure["models.siglip"].append("SiglipImageProcessor")
@@ -3096,6 +3105,12 @@
             "SamPreTrainedModel",
         ]
     )
+    _import_structure["models.sam2"].extend(
+        [
+            "Sam2Model",
+            "Sam2PreTrainedModel",
+        ]
+    )
     _import_structure["models.seamless_m4t"].extend(
         [
             "SeamlessM4TCodeHifiGan",
@@ -4733,6 +4748,7 @@
         ToolCollection,
         launch_gradio_demo,
         load_tool,
+        stream_to_gradio,
     )
     from .configuration_utils import PretrainedConfig
 
@@ -5913,6 +5929,7 @@
         from .models.pvt import PvtImageProcessor
         from .models.rt_detr import RTDetrImageProcessor
         from .models.sam import SamImageProcessor
+        from .models.sam2 import Sam2ImageProcessor
         from .models.segformer import SegformerFeatureExtractor, SegformerImageProcessor
         from .models.seggpt import SegGptImageProcessor
         from .models.siglip import SiglipImageProcessor
@@ -7465,6 +7482,10 @@
             SamModel,
             SamPreTrainedModel,
         )
+        from .models.sam2 import (
+            Sam2Model,
+            Sam2PreTrainedModel,
+        )
         from .models.seamless_m4t import (
             SeamlessM4TCodeHifiGan,
             SeamlessM4TForSpeechToSpeech,

src/transformers/models/__init__.py

@@ -201,6 +201,7 @@
     rt_detr,
     rwkv,
     sam,
+    sam2,
     seamless_m4t,
     seamless_m4t_v2,
     segformer,

src/transformers/models/auto/configuration_auto.py

@@ -223,6 +223,7 @@
         ("rt_detr_resnet", "RTDetrResNetConfig"),
         ("rwkv", "RwkvConfig"),
         ("sam", "SamConfig"),
+        ("sam2", "Sam2Config"),
         ("seamless_m4t", "SeamlessM4TConfig"),
         ("seamless_m4t_v2", "SeamlessM4Tv2Config"),
         ("segformer", "SegformerConfig"),
@@ -517,6 +518,7 @@
         ("rt_detr_resnet", "RT-DETR-ResNet"),
         ("rwkv", "RWKV"),
         ("sam", "SAM"),
+        ("sam2", "SAM2"),
         ("seamless_m4t", "SeamlessM4T"),
         ("seamless_m4t_v2", "SeamlessM4Tv2"),
         ("segformer", "SegFormer"),

src/transformers/models/auto/image_processing_auto.py

@@ -120,6 +120,7 @@
             ("resnet", ("ConvNextImageProcessor",)),
             ("rt_detr", "RTDetrImageProcessor"),
             ("sam", ("SamImageProcessor",)),
+            ("sam2", ("Sam2ImageProcessor",)),
             ("segformer", ("SegformerImageProcessor",)),
             ("seggpt", ("SegGptImageProcessor",)),
             ("siglip", ("SiglipImageProcessor",)),

src/transformers/models/auto/modeling_auto.py

@@ -208,6 +208,7 @@
         ("rt_detr", "RTDetrModel"),
         ("rwkv", "RwkvModel"),
         ("sam", "SamModel"),
+        ("sam2", "Sam2Model"),
         ("seamless_m4t", "SeamlessM4TModel"),
         ("seamless_m4t_v2", "SeamlessM4Tv2Model"),
         ("segformer", "SegformerModel"),
@@ -1280,6 +1281,7 @@
 MODEL_FOR_MASK_GENERATION_MAPPING_NAMES = OrderedDict(
     [
         ("sam", "SamModel"),
+        ("sam2", "Sam2Model"),
     ]
 )
 

src/transformers/models/auto/processing_auto.py

@@ -83,6 +83,7 @@
         ("pix2struct", "Pix2StructProcessor"),
         ("pop2piano", "Pop2PianoProcessor"),
         ("sam", "SamProcessor"),
+        ("sam2", "Sam2Processor"),
         ("seamless_m4t", "SeamlessM4TProcessor"),
         ("sew", "Wav2Vec2Processor"),
         ("sew-d", "Wav2Vec2Processor"),

src/transformers/utils/dummy_pt_objects.py

@@ -7719,6 +7719,20 @@ def __init__(self, *args, **kwargs):
         requires_backends(self, ["torch"])
 
 
+class Sam2Model(metaclass=DummyObject):
+    _backends = ["torch"]
+
+    def __init__(self, *args, **kwargs):
+        requires_backends(self, ["torch"])
+
+
+class Sam2PreTrainedModel(metaclass=DummyObject):
+    _backends = ["torch"]
+
+    def __init__(self, *args, **kwargs):
+        requires_backends(self, ["torch"])
+
+
 class SeamlessM4TCodeHifiGan(metaclass=DummyObject):
     _backends = ["torch"]
 

src/transformers/utils/dummy_vision_objects.py

@@ -527,6 +527,13 @@ def __init__(self, *args, **kwargs):
         requires_backends(self, ["vision"])
 
 
+class Sam2ImageProcessor(metaclass=DummyObject):
+    _backends = ["vision"]
+
+    def __init__(self, *args, **kwargs):
+        requires_backends(self, ["vision"])
+
+
 class SegformerFeatureExtractor(metaclass=DummyObject):
     _backends = ["vision"]