[experiments] Improve on job parent searching to avoid double-assignment

beat/web/experiments/models.py

@@ -1020,11 +1020,21 @@ class Block(models.Model):
 
         from ..backend.models import Job
 
-        # search for other jobs with similar outputs and has no children yet
-        similar = Job.objects.select_for_update().filter(\
-            block__outputs__in=self.outputs.all(), child=None).first()
-
-        Job(block=self, parent=similar).save()
+        # search for other jobs with similar outputs that have no children yet
+        # do this carefully, as other experiments may be scheduled at the same
+        # time, invalidating our "parent" choice
+        parent = Job.objects.filter(block__outputs__in=self.outputs.all(),
+            child=None).first()
+        if parent is not None: #(candidate only) try to lock it
+            while True:
+                parent = Job.objects.select_for_update().get(pk=parent.pk)
+                if parent.child_ is not None: #was taken meanwhile, retry
+                    parent = parent.child
+                    continue
+                Job(block=self, parent=parent).save()
+                break
+        else:
+            Job(block=self).save()
 
         # checks if the job is immediately runnable - if so, tries to
         # make it runnable (check caches and other)