CI: bracket testing added

Signed-off-by: ismail <mohammed.ismail@openairinterface.org>

ci-scripts/Jenkinsfile-Bracket-Test

+#!/bin/groovy
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+//-------------------------------------------------------------------------------
+
+// Location of the CN executor node
+def cn_ci_host = params.Host_CN_CI_Server
+
+// for lock
+def cn_ci_resource = params.DockerContainers
+def ds_tester_ci_resource = params.DsTester
+
+// Location of the 2nd CN executor
+def new_host_flag = false
+def new_host = ""
+def new_host_user = ""
+
+// Location of the CN tester
+def dsT_host_flag = false
+def dsT_host = ""
+def dsT_host_user = ""
+def dsT_host_ip_addr = ""
+
+// Flags
+def scmEvent = false
+def upstreamEvent = false
+def deployed = true
+
+// Default tag  --> could be passed on by upstream job or by PR content
+def nrfTag = params.nrfTag
+
+//-------------------------------------------------------------------------------
+// Pipeline start
+pipeline {
+  agent {
+    label cn_ci_host
+  }
+  options {
+    disableConcurrentBuilds()
+    timestamps()
+    ansiColor('xterm')
+    lock(cn_ci_resource)
+  }
+  stages {
+    stage ('Verify Parameters') {
+      steps {
+        script {
+          echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'
+
+          JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
+          JOB_TIMESTAMP = JOB_TIMESTAMP.trim()
+
+          if (params.Host_CN_CI_2nd_Server_Flag != null) {
+            new_host_flag = params.Host_CN_CI_2nd_Server_Flag
+            if (new_host_flag) {
+              new_host = params.Host_CN_CI_2nd_Server
+              new_host_user = params.Host_CN_CI_2nd_Server_Login
+              echo "1st Node   is ${NODE_NAME}"
+              echo "2nd Node   is ${new_host}"
+            } else {
+              echo "Node       is ${NODE_NAME}"
+            }
+          } else {
+            echo "Node       is ${NODE_NAME}"
+          }
+          if (params.DS_Tester_Server_Flag != null) {
+            dsT_host_flag = params.DS_Tester_Server_Flag
+            if (dsT_host_flag) {
+              def allParametersPresent = true
+              if (params.DS_Tester_Server_Name == null) {
+                allParametersPresent = false
+              } else {
+                dsT_host = params.DS_Tester_Server_Name
+              }
+              if (params.DS_Tester_Server_Login == null) {
+                allParametersPresent = false
+              } else {
+                dsT_host_user = params.DS_Tester_Server_Login
+              }
+              if (params.DS_Tester_Server_IP_Addr == null) {
+                allParametersPresent = false
+              } else {
+                dsT_host_ip_addr = params.DS_Tester_Server_IP_Addr
+              }
+              if (allParametersPresent) {
+                echo "DS Tester  is on ${dsT_host}"
+              } else {
+                echo "Some DS Tester parameters are missing!"
+                sh "./ci-scripts/fail.sh"
+              }
+            }
+          }
+
+          // Find out the cause of the trigger
+          for (cause in currentBuild.getBuildCauses()) {
+            if (cause.toString() ==~ /.*UpstreamCause.*/) {
+              upstreamEvent = true
+            //} else {
+            //  scmEvent = true
+            }
+          }
+          if (upstreamEvent) {
+            if (params.NRF_TAG != null) {
+              nrfTag = params.NRF_TAG
+              echo "Upstream Job passed NRF_TAG to use: ${nrfTag}"
+            }
+            sh "git clean -x -d -f > /dev/null 2>&1"
+            sh "git fetch --prune > /dev/null 2>&1"
+          }
+          sh "mkdir -p archives DS-TEST-RESULTS"
+          // Verify that the images are available
+          try {
+            sh 'echo "OAI_NRF_TAG: oai-nrf:' + nrfTag +'" > archives/oai_nrf_image_info.log'
+            sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
+            sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log'
+          } catch (Exception e) {
+            error "OAI NRF Image tag to test does not exist!"
+          }
+        }
+      }
+    }
+    stage ('Deploy NRF-U18') {
+      steps {
+        script {
+          echo '\u2705 \u001B[32mDeploy NRF using Docker-Compose\u001B[0m'
+          // Prepare all needed files for docker-compose
+          // First put all correct tags to test
+          sh 'sed -e "s#NRF_IMAGE_TAG#' + nrfTag + '#" ci-scripts/dsTesterDockerCompose/docker-compose.tplt > ci-scripts/dsTesterDockerCompose/docker-compose.yml'
+          dir('ci-scripts/dsTesterDockerCompose') {
+            sh 'docker-compose up -d  > ../../archives/compose_nrf_up.log 2>&1'
+            sh 'sleep 60'
+            // Do a check on number of healthy containers
+            // 1 == nrf
+            ret = sh returnStdout: true, script: 'docker-compose ps -a | grep -v unhealthy | grep -c healthy || true'
+            ret = ret.trim()
+            if (ret != '1') {
+              error "Deployment went wrong!"
+            }          
+          }
+        }
+      }
+      post {
+        always {
+          script {
+            // Do docker logs to recover the configuration results
+            try {
+              sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-nrf > archives/nrf_config.log'
+            } catch (Exception e) {
+              sh 'echo "STATUS: KO" >> archives/nrf_config.log'
+            }
+          }
+        }
+        success {
+          script {
+            sh 'echo "DEPLOYMENT: OK" > archives/deployment_status.log'
+            sh 'python3 ./ci-scripts/routeCheck.py --mode=Add --userName=' + dsT_host_user + ' --hostName=' + dsT_host
+          }
+        }
+        unsuccessful {
+          script {
+            sh 'echo "DEPLOYMENT: KO" > archives/deployment_status.log'
+            deployed = false
+          }
+        }
+      }
+    }
+    stage ('Bracket Testing with dsTester') {
+      when { expression {dsT_host_flag} }
+      steps {
+        lock (ds_tester_ci_resource) {
+          script {
+            echo '\u2705 \u001B[32mTesting with DS Tester\u001B[0m'
+            if (fileExists("dstester")) {
+              sh "rm -Rf dstester > /dev/null 2>&1"
+            }
+            sh "mkdir -p dstester"
+            dir ('dstester') {
+              withCredentials([
+                [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.dsTesterGitLabRepository_Credentials}", usernameVariable: 'git_username', passwordVariable: 'git_token']
+              ]) {
+                sh "git clone https://${git_username}:${git_token}@github.com/OPENAIRINTERFACE/chasseur.git . > ../git_clone.log 2>&1"
+                sh "git checkout edge >> ../git_clone.log 2>&1"
+                dir ('jenkins') {
+                  try {
+                    sh "python3 ./dogmatix-agent.py -f ./suits/dc/nrf-bracket.yaml -d true | tee ../../DS-TEST-RESULTS/dsTester_Summary.txt"
+                  } catch (Exception e) {
+                    currentBuild.result = 'FAILURE'
+                    echo "dsTester Running FAILED"
+                  }
+                }
+              }
+            }
+            sh "python3 ./ci-scripts/toCheckDSTesterResult.py"
+          }  
+        }
+      }
+      post {
+        always {
+          script {
+            // Copy the pcap and log from the container
+            sh "mkdir -p archives/pcaps archives/logs"
+            try {
+              sh 'docker cp cicd-oai-nrf:/tmp/nrf.pcap archives/pcaps/oai_nrf.pcap'
+              sh 'docker cp cicd-oai-nrf:/tmp/nrf.log archives/logs/oai_nrf.log'
+            } catch (Exception e) {
+              sh 'echo "Error in copying pcap & log from the container"'
+            }
+          }
+        }
+      }
+    }
+    stage ('Undeploy NRF') {
+      steps {
+        script {
+          echo '\u2705 \u001B[32mUn-Deploy NRF\u001B[0m'
+          sh 'python3 ./ci-scripts/routeCheck.py --mode=Delete --userName=' + dsT_host_user + ' --hostName=' + dsT_host
+          dir('ci-scripts/dsTesterDockerCompose') {
+            sh 'docker-compose down > ../../archives/compose_normal_down.log 2>&1'
+          }
+        }
+      }
+    }
+  }
+  post {
+    always {
+      script {
+        // Get logs if deployment fails
+        if (deployed != true) {
+        sh 'mkdir -p archives/logs'
+        sh 'docker logs cicd-oai-nrf > archives/logs/oai_nrf.log'
+        }
+        // Remove any leftover containers/networks
+        sh 'python3 ./ci-scripts/routeCheck.py --mode=Delete --userName=' + dsT_host_user + ' --hostName=' + dsT_host
+        dir('ci-scripts/dsTesterDockerCompose') {
+          sh 'docker-compose down > ../../archives/compose_l_down.log 2>&1'
+        }
+        // Generating the HTML report
+        sh 'python3 ./ci-scripts/bracketTestHTMLReport.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL
+
+        // Zipping all archived log files
+        sh "zip -r -qq cn5g_nrf_docker_bt_logs.zip archives DS-TEST-RESULTS"
+        if (fileExists('cn5g_nrf_docker_bt_logs.zip')) {
+          archiveArtifacts artifacts: 'cn5g_nrf_docker_bt_logs.zip'
+        }
+        if (fileExists('test_results_oai_nrf_bt.html')) {
+          archiveArtifacts artifacts: 'test_results_oai_nrf_bt.html'
+        }
+      }
+    }
+  }
+}

ci-scripts/Jenkinsfile-GitLab-Docker

@@ -338,6 +338,35 @@ pipeline {
         }
       }
     }
+    stage ('Bracket Testing with DsTester') {
+      steps {
+        script {
+          gitlabCommitStatus(name: "Bracket Test with DsTester") {
+            localStatus = build job: params.NRF_BT_PipelineName,
+              parameters: [
+                string(name: 'NRF_TAG', value: String.valueOf(nrf_tag))
+              ], propagate: false
+            localResult = localStatus.getResult()
+
+            if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
+              echo "Bracket Test Job is OK"
+            } else {
+              echo "Bracket Test Job is KO"
+              sh "ci-scripts/fail.sh"
+            }
+          }
+        }
+      }
+      post {
+        always {
+          script {
+            copyArtifacts(projectName: params.AMF_BT_PipelineName,
+                          filter: '*_results_oai_nrf_bt.html',
+                          selector: lastCompleted())
+          }
+        }
+      }
+    }
     stage ('Testing in CN-5G-FED environment') {
       steps {
         script {
@@ -439,6 +468,10 @@ pipeline {
             sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' *_results_oai_cn5g.html"
             sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' *_results_oai_cn5g.html"
           }
+          if (fileExists('test_results_oai_nrf_bt.html')) {
+            sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' *_results_oai_nrf_bt.html"
+            sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' *_results_oai_nrf_bt.html"
+          }
         } else {
           sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${GIT_BRANCH} --git_src_commit=${GIT_COMMIT}"
         }
@@ -451,6 +484,10 @@ pipeline {
           sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' *_results_oai_cn5g.html"
           archiveArtifacts artifacts: '*_results_oai_cn5g.html'
         }
+        if (fileExists('test_results_oai_nrf_bt.html')) {
+          sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' *_results_oai_nrf_bt.html"
+          archiveArtifacts artifacts: '*_results_oai_nrf_bt.html'
+        }
 
         // Sending email to commiter
         if (params.sendToCommitterEmail != null) {

ci-scripts/dsTesterDockerCompose/docker-compose.tplt

+version: '3.8'
+services:
+    cicd_oai_nrf:
+        container_name: cicd-oai-nrf
+        image: oai-nrf:NRF_IMAGE_TAG
+        ports:
+            - 80
+            - 9090
+        command: >
+            bash -c "nohup tshark -i eth0 -w /tmp/nrf.pcap 2>&1 > /dev/null &
+            /openair-nrf/bin/oai_nrf -c /openair-nrf/etc/nrf.conf -o | tee /tmp/nrf.log 2>&1
+            "
+        cap_add:
+            - NET_ADMIN
+        environment:
+            - TZ=Europe/Paris
+            - NRF_INTERFACE_NAME_FOR_SBI=eth0
+            - NRF_INTERFACE_PORT_FOR_SBI=80
+            - NRF_INTERFACE_HTTP2_PORT_FOR_SBI=9090
+            - NRF_API_VERSION=v1
+            - INSTANCE=0
+            - PID_DIRECTORY=/var/run
+        networks:
+            cicd_public_net:
+                ipv4_address: 192.168.61.195
+        volumes:
+            - ./nrf-healthy-check.sh:/openair-nrf/bin/nrf-healthy-check.sh
+        healthcheck:
+            test: /bin/bash -c "/openair-nrf/bin/nrf-healthy-check.sh"
+            interval: 10s
+            timeout: 5s
+            retries: 5
+
+networks:
+    cicd_public_net:
+        name: cicd-oai-public-net
+        driver: bridge
+        ipam:
+            config:
+                - subnet: 192.168.61.192/26

ci-scripts/dsTesterDockerCompose/nrf-healthy-check.sh

+#!/bin/bash
+
+STATUS=0
+
+RESULT=$(ps aux | grep -v nohup || true)
+SUB='/openair-nrf/bin/oai_nrf -c /openair-nrf/etc/nrf.conf -o'
+if [[ $RESULT =~ $SUB ]]; then
+    STATUS=0
+else
+   STATUS=-1
+fi
+
+exit $STATUS

ci-scripts/routeCheck.py

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *   http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *   contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+
+import os
+import re
+import sys
+import subprocess
+import yaml
+
+class IpRouteCheck():
+	def __init__(self):
+		self.mode = ''
+		self.userName = ''
+		self.hostName = ''
+		self.subnet = ''
+		self.gatwayIp = ''
+		self.interfaceName = ''
+
+	def getSubnet(self):
+		cmd = "egrep 'subnet' ci-scripts/dsTesterDockerCompose/docker-compose.yml"
+		ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
+		if ret.returncode == 0:
+			if ret.stdout is not None:
+				result = re.search("(?:[0-9]{1,3}[.]){3}[0-9]{1,3}/[0-9]{1,2}", ret.stdout.strip())
+				if result is not None:
+					self.subnet = result.group(0)
+					#print("found subnet:", self.subnet)
+			else:
+				print("subnet not found in docker compose")
+				sys.exit(-1)
+		else:
+			print("docker-compose file not found")
+			sys.exit(-1)
+
+	def getGatwayIp(self):
+		cmd = "ifconfig | grep 192.168.18"
+		ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
+		if ret.returncode == 0:
+			result = re.search("inet (?:[0-9]{1,3}[.]){3}[0-9]{1,3}", ret.stdout.strip())
+			self.gatwayIp = result.group(0)
+			#print("Gatway IP Address:", self.gatwayIp)
+		else:
+			print("No Gatway IP address starts with 196.168.18")
+			sys.exit(-1)	
+
+	def routeCheck(self):
+		# Get the interface name
+		cmd = "ip route | grep 192.168.18.0/24 | awk {'print $3'}"
+		ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+		#print('interface name:', ret.stdout.strip())
+		self.interfaceName = ret.stdout.strip()
+		# Check whether or not ip route exist 
+		cmd = f'ip route | grep -c "{self.subnet}"'
+		ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+		if ret.stdout is not None:
+			if ret.stdout.strip() == '1':
+				#print('Route exits')
+				if IPRC.mode == 'Delete':
+					IPRC.routeDel()
+				else:
+					sys.exit(0)
+			else:
+				#print("Route not found")
+				if IPRC.mode == 'Add':
+					IPRC.routeAdd()
+				else:
+					sys.exit(0)
+
+	def routeAdd(self):
+		# Add the route
+		cmd = f"sudo ip route add {self.subnet} via {self.gatwayIp} dev {self.interfaceName}"
+		ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+		print("Added ip route")
+
+	def routeDel(self):
+		# Delete the route
+		cmd = f"sudo ip route del {self.subnet} via {self.gatwayIp} dev {self.interfaceName}"
+		ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8')
+		print("Deleted ip route")
+
+def Usage():
+	print('----------------------------------------------------------------------------------------------------------------------')
+	print('routecheck.py')
+	print('   Add and Delete the ip route on the Server.')
+	print('----------------------------------------------------------------------------------------------------------------------')
+	print('Usage: python3 routecheck.py [options]')
+	print('  --help  Show this help.')
+	print('---------------------------------------------------------------------------------------------- Mandatory Options -----')
+	print('  --mode=[Add/Delete]')
+	print('  --userName=[server userName where to add/delete]')
+	print('  --hostName=[server hostName where to add/delete]')
+	print('------------------------------------------------------------------------------------------------- Actions Syntax -----')
+	print('python3 routeCheck.py --mode=Add [Mandatory Options]')
+	print('python3 routeCheck.py --mode=Delete [Mandatory Options]')	
+
+
+#--------------------------------------------------------------------------------------------------------
+#
+# Start of main
+#
+#--------------------------------------------------------------------------------------------------------
+
+argvs = sys.argv
+
+IPRC = IpRouteCheck()
+
+while len(argvs) > 1:
+	myArgv = argvs.pop(1)
+	if re.match('^\-\-help$', myArgv, re.IGNORECASE):
+		Usage()
+		sys.exit(0)
+	elif re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE):
+		matchReg = re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE)
+		IPRC.mode = matchReg.group(1)
+	elif re.match('^\-\-userName=(.+)$', myArgv, re.IGNORECASE):
+		matchReg = re.match('^\-\-userName=(.+)$', myArgv, re.IGNORECASE)
+		IPRC.userName = matchReg.group(1)
+	elif re.match('^\-\-hostName=(.+)$', myArgv, re.IGNORECASE):
+		matchReg = re.match('^\-\-hostName=(.+)$', myArgv, re.IGNORECASE)
+		IPRC.hostName = matchReg.group(1)
+	else:
+		sys.exit('Invalid Parameter: ' + myArgv)
+
+if IPRC.mode == '' or IPRC.userName == '' or IPRC.hostName == '':
+	sys.exit('Missing Parameter in job description')
+
+IPRC.getSubnet()
+IPRC.getGatwayIp()
+IPRC.routeCheck()

ci-scripts/toCheckDSTesterResult.py

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *   http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *   contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+
+import re
+import sys
+import subprocess
+import yaml
+import os
+import time
+
+locexist = False
+cwd = os.getcwd()
+try:
+    with open(cwd + '/DS-TEST-RESULTS/dsTester_Summary.txt') as f:
+        for line in f:
+            if re.search('Result file is available here', str(line)):
+                result = re.search('(?:\/.+?\/)(.+?)(?:\/.+)', str(line))
+                if result:
+                    result1 = re.search('^(.*/)([^/]*)$', str(result.group(0)))
+                    subprocess.check_output(f'cp {result1.group(1)}* DS-TEST-RESULTS/', stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
+                    result2 = re.search('.*\/(.+)$', str(result.group(0)))
+                    filename = result2.group(1)
+                locexist = True
+except IOError:
+    sys.exit("File not accessible to check DSTester Summary: DS-TEST-RESULTS/dsTester_Summary.txt")
+
+if locexist:
+    try:
+        with open(cwd + f'/DS-TEST-RESULTS/{filename}') as f:
+            data = yaml.load(f)
+            if data["final-result"] == 'fail':
+                sys.exit('DsTester final result FAILED')
+    except IOError:
+        sys.exit(f'File not accessible to check DSTester result: DS-TEST-RESULTS/{filename}')