CI: adding much needed lock mechanism when creating VM.

  -- should prevent VM-creation crashes during test stages
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>
(cherry picked from commit eeffdfa200fe834e53f13b8692632baac6ee2dfb)
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/buildOnVM.sh

@@ -94,14 +94,21 @@ function build_on_vm {
         echo "############################################################"
         echo "Creating VM ($VM_NAME) on Ubuntu Cloud Image base"
         echo "############################################################"
+        acquire_vm_create_lock
         uvt-kvm create $VM_NAME release=xenial --memory $VM_MEMORY --cpu $VM_CPU --unsafe-caching --template ci-scripts/template-host.xml
-    fi
+        echo "Waiting for VM to be started"
+        uvt-kvm wait $VM_NAME --insecure
 
-    echo "Waiting for VM to be started"
-    uvt-kvm wait $VM_NAME --insecure
+        VM_IP_ADDR=`uvt-kvm ip $VM_NAME`
+        echo "$VM_NAME has for IP addr = $VM_IP_ADDR"
+        release_vm_create_lock
+    else
+        echo "Waiting for VM to be started"
+        uvt-kvm wait $VM_NAME --insecure
 
-    VM_IP_ADDR=`uvt-kvm ip $VM_NAME`
-    echo "$VM_NAME has for IP addr = $VM_IP_ADDR"
+        VM_IP_ADDR=`uvt-kvm ip $VM_NAME`
+        echo "$VM_NAME has for IP addr = $VM_IP_ADDR"
+    fi
 
     echo "############################################################"
     echo "Copying GIT repo into VM ($VM_NAME)" 

ci-scripts/createVM.sh

@@ -49,6 +49,35 @@ function create_usage {
     echo ""
 }
 
+function acquire_vm_create_lock {
+    local FlockFile="/tmp/vmclone.lck"
+    local unlocked="0"
+    touch ${FlockFile} 2>/dev/null
+    if [[ $? -ne 0 ]]
+    then
+        echo "Cannot access lock file ${FlockFile}"
+        exit 2
+    fi
+    while [ $unlocked -eq 0 ]
+    do
+        exec 5>${FlockFile}
+        flock -nx 5
+        if [[ $? -ne 0 ]]
+        then
+            echo "Another instance of VM creation is running"
+            sleep 10
+        else
+            unlocked="1"
+        fi
+    done
+    chmod 666 ${FlockFile} 2>/dev/null
+}
+
+function release_vm_create_lock {
+    local FlockFile="/tmp/vmclone.lck"
+    rm -Rf ${FlockFile}
+}
+
 function create_vm {
     echo "############################################################"
     echo "OAI CI VM script"
@@ -60,10 +89,12 @@ function create_vm {
     echo "############################################################"
     echo "Creating VM ($VM_NAME) on Ubuntu Cloud Image base"
     echo "############################################################"
+    acquire_vm_create_lock
     uvt-kvm create $VM_NAME release=xenial --memory $VM_MEMORY --cpu $VM_CPU --unsafe-caching --template ci-scripts/template-host.xml
     echo "Waiting for VM to be started"
     uvt-kvm wait $VM_NAME --insecure
 
     VM_IP_ADDR=`uvt-kvm ip $VM_NAME`
     echo "$VM_NAME has for IP addr = $VM_IP_ADDR"
+    release_vm_create_lock
 }

ci-scripts/runTestOnVM.sh

@@ -441,10 +441,16 @@ function run_test_on_vm {
             echo "############################################################"
             echo "Creating test EPC VM ($EPC_VM_NAME) on Ubuntu Cloud Image base"
             echo "############################################################"
+            acquire_vm_create_lock
             uvt-kvm create $EPC_VM_NAME release=xenial --unsafe-caching
+            echo "Waiting for VM to be started"
+            uvt-kvm wait $EPC_VM_NAME --insecure
+            release_vm_create_lock
+        else
+            echo "Waiting for VM to be started"
+            uvt-kvm wait $EPC_VM_NAME --insecure
         fi
 
-        uvt-kvm wait $EPC_VM_NAME --insecure
         EPC_VM_IP_ADDR=`uvt-kvm ip $EPC_VM_NAME`
         echo "$EPC_VM_NAME has for IP addr = $EPC_VM_IP_ADDR"
         scp -o StrictHostKeyChecking=no /etc/apt/apt.conf.d/01proxy ubuntu@$EPC_VM_IP_ADDR:/home/ubuntu