diff --git a/project_milestone_spent_hours/models/project_milestone.py b/project_milestone_spent_hours/models/project_milestone.py
index 03932817ab..37b05e57c5 100644
--- a/project_milestone_spent_hours/models/project_milestone.py
+++ b/project_milestone_spent_hours/models/project_milestone.py
@@ -7,6 +7,7 @@
 class ProjectMilestone(models.Model):
     _inherit = "project.milestone"
 
+    active = fields.Boolean("Active", default=True)
     total_hours = fields.Float(
         compute="_compute_total_hours",
         string="Total Hours",
@@ -14,6 +15,17 @@ class ProjectMilestone(models.Model):
         store=True,
     )
 
+    def write(self, vals):
+        res = super(ProjectMilestone, self).write(vals)
+        if "project_id" in vals:
+            self._remove_task_milestones(vals["project_id"])
+        return res
+
+    def _remove_task_milestones(self, project_id):
+        self.with_context(active_test=False).mapped("project_task_ids").filtered(
+            lambda milestone: not project_id or milestone.project_id.id != project_id
+        ).write({"milestone_id": False})
+
     @api.depends(
         "project_task_ids",
         "project_task_ids.active",