Commit 6aa12648 authored by Raphael Defosseux's avatar Raphael Defosseux

[CI][Docker] Building docker images in python framework

- only on Ubuntu and Docker
- No back ground computing to paralellize RHEL/Ubuntu
- No image clean up
Signed-off-by: default avatarRaphael Defosseux <raphael.defosseux@eurecom.fr>
parent df334999
Branches unavailable
2024.w43 2024.w42 2024.w41 2024.w40 2024.w39 2024.w38 2024.w36 2024.w35 2024.w34 2024.w33 2024.w32 2024.w31 2024.w30 2024.w29 2024.w28 2024.w27 2024.w26 2024.w25 2024.w24 2024.w23 2024.w22 2024.w21 2024.w18 2024.w17 2024.w16 2024.w15 2024.w14 2024.w13 2024.w12 2024.w11 2024.w10 2024.w09 2024.w08 2024.w06 2024.w05 2024.w04 2024.w03 2024.w02 2024.w01 2023.w51 2023.w50 2023.w49 2023.w48 2023.w47 2023.w45 2023.w43 2023.w42 2023.w41 2023.w40 2023.w39 2023.w38 2023.w37 2023.w36 2023.w34 2023.w33 2023.w32 2023.w31 2023.w30 2023.w29 2023.w28 2023.w27 2023.w26 2023.w25 2023.w24 2023.w23 2023.w22 2023.w21 2023.w20 2023.w19 2023.w18 2023.w18b 2023.w16 2023.w15 2023.w14 2023.w13 2023.w12 2023.w11 2023.w11b 2023.w10 2023.w10b 2023.w09 2023.w08 2023.w08b 2023.w07 2023.w06 2023.w05 2023.w03 2023.w02 2022.42 2022.41 2022.w51 2022.w50 2022.w49 2022.w48 2022.w47 2022.w46 2022.w45 2022.w43 2022.w42 2022.w42b 2022.w41 2022.w40 2022.w39 2022.w38 2022.w37 2022.w37b 2022.w36 2022.w35 2022.w33 2022.w32 2022.w31 2022.w31b 2022.w30 2022.w29 2022.w26 2022.w25 2022.w24 2022.w24b 2022.w23 2022.w22 2022.w21 2022.w20 2022.w19 2022.w18 2022.w17 2022.w15 2022.w15b 2022.w14a 2022.w13 2022.w13b 2022.w13a 2022.w12 2022.w10 2022.w09 2022.w09b 2022.w08 2022.w08b 2022.w07 2022.w07b 2022.w06 2022.w06a 2022.w05 2022.w05b 2022.w03_hotfix 2022.w03_b 2022.w02 2022.w01 2021.wk46 2021.wk14_a 2021.wk13_d 2021.wk13_c 2021.w51_c 2021.w51_a 2021.w50_a 2021.w49_b 2021.w49_a 2021.w48 2021.w47 2021.w46 2021.w46-powder 2021.w45 2021.w45_b 2021.w44 2021.w43 2021.w42 2021.w37 2021.w36 2021.w35 2021.w34 2021.w33 2021.w32 2021.w31 2021.w30 2021.w29 2021.w28 2021.w27 2021.w26 2021.w25 2021.w24 2021.w23 2021.w22 2021.w20 2021.w19 2021.w18_b 2021.w18_a 2021.w17_b 2021.w16 2021.w15 2021.w14 2021.w13_a 2021.w12 2021.w11 2021.w10 2021.w09 2021.w08 2021.w06 2021.w05 2021.w04 2021.w02 2020.w51_2 2020.w51 v2.1.0 v2.0.0 setparam flexran-eol benetel_gnb_rel_2.0 benetel_gnb_rel_1.0 benetel_enb_rel_2.0 benetel_enb_rel_1.0 ARC_1.3
No related merge requests found
...@@ -98,8 +98,9 @@ def AssignParams(params_dict): ...@@ -98,8 +98,9 @@ def AssignParams(params_dict):
def GetParametersFromXML(action): def GetParametersFromXML(action):
if action == 'Build_eNB': if action == 'Build_eNB' or action == 'Build_Image':
RAN.Build_eNB_args=test.findtext('Build_eNB_args') RAN.Build_eNB_args=test.findtext('Build_eNB_args')
RAN.imageKind=test.findtext('kind')
forced_workspace_cleanup = test.findtext('forced_workspace_cleanup') forced_workspace_cleanup = test.findtext('forced_workspace_cleanup')
if (forced_workspace_cleanup is None): if (forced_workspace_cleanup is None):
RAN.Build_eNB_forced_workspace_cleanup=False RAN.Build_eNB_forced_workspace_cleanup=False
...@@ -127,8 +128,9 @@ def GetParametersFromXML(action): ...@@ -127,8 +128,9 @@ def GetParametersFromXML(action):
else: else:
RAN.backgroundBuild=False RAN.backgroundBuild=False
elif action == 'WaitEndBuild_eNB': elif action == 'WaitEndBuild_eNB' or action == 'WaitEndBuild_Image':
RAN.Build_eNB_args=test.findtext('Build_eNB_args') RAN.Build_eNB_args=test.findtext('Build_eNB_args')
RAN.imageKind=test.findtext('kind')
eNB_instance=test.findtext('eNB_instance') eNB_instance=test.findtext('eNB_instance')
if (eNB_instance is None): if (eNB_instance is None):
RAN.eNB_instance=0 RAN.eNB_instance=0
...@@ -640,8 +642,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re ...@@ -640,8 +642,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
break break
if action == 'Build_eNB': if action == 'Build_eNB':
RAN.BuildeNB() RAN.BuildeNB()
elif action == 'Build_Image':
RAN.BuildImage()
elif action == 'WaitEndBuild_eNB': elif action == 'WaitEndBuild_eNB':
RAN.WaitBuildeNBisFinished() RAN.WaitBuildeNBisFinished()
elif action == 'WaitEndBuild_Image':
RAN.WaitBuildImageisFinished()
elif action == 'Initialize_eNB': elif action == 'Initialize_eNB':
check_eNB = False check_eNB = False
check_OAI_UE = False check_OAI_UE = False
......
...@@ -77,6 +77,7 @@ class RANManagement(): ...@@ -77,6 +77,7 @@ class RANManagement():
self.backgroundBuildTestId = ['', '', ''] self.backgroundBuildTestId = ['', '', '']
self.Build_eNB_forced_workspace_cleanup = False self.Build_eNB_forced_workspace_cleanup = False
self.Initialize_eNB_args = '' self.Initialize_eNB_args = ''
self.imageKind = ''
self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main
self.eNB_instance = 0 self.eNB_instance = 0
self.eNB_serverId = ['', '', ''] self.eNB_serverId = ['', '', '']
...@@ -318,6 +319,171 @@ class RANManagement(): ...@@ -318,6 +319,171 @@ class RANManagement():
self.htmlObj.CreateHtmlTabFooter(False) self.htmlObj.CreateHtmlTabFooter(False)
sys.exit(1) sys.exit(1)
def BuildImage(self):
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
if self.eNB_serverId[self.eNB_instance] == '0':
lIpAddr = self.eNBIPAddress
lUserName = self.eNBUserName
lPassWord = self.eNBPassword
lSourcePath = self.eNBSourceCodePath
elif self.eNB_serverId[self.eNB_instance] == '1':
lIpAddr = self.eNB1IPAddress
lUserName = self.eNB1UserName
lPassWord = self.eNB1Password
lSourcePath = self.eNB1SourceCodePath
elif self.eNB_serverId[self.eNB_instance] == '2':
lIpAddr = self.eNB2IPAddress
lUserName = self.eNB2UserName
lPassWord = self.eNB2Password
lSourcePath = self.eNB2SourceCodePath
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
logging.debug('Building on server: ' + lIpAddr)
mySSH = SSH.SSHConnection()
mySSH.open(lIpAddr, lUserName, lPassWord)
imageNames = []
result = re.search('eNB', self.imageKind)
if result is not None:
imageNames.append(('oai-enb', 'eNB'))
else:
result = re.search('gNB', self.imageKind)
if result is not None:
imageNames.append(('oai-gnb', 'gNB'))
else:
result = re.search('all', self.imageKind)
if result is not None:
imageNames.append(('oai-enb', 'eNB'))
imageNames.append(('oai-gnb', 'gNB'))
if len(imageNames) == 0:
imageNames.append(('oai-enb', 'eNB'))
# Workaround for some servers, we need to erase completely the workspace
if self.Build_eNB_forced_workspace_cleanup:
mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
if self.htmlObj is not None:
self.testCase_id = self.htmlObj.testCase_id
else:
self.testCase_id = '000000'
# on RedHat/CentOS .git extension is mandatory
result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
if result is not None:
full_ran_repo_name = self.ranRepository
else:
full_ran_repo_name = self.ranRepository + '.git'
mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
mySSH.command('cd ' + lSourcePath, '\$', 5)
mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
# Raphael: here add a check if git clone or git fetch went smoothly
mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
mySSH.command('mkdir -p cmake_targets/log', '\$', 5)
# if the commit ID is provided use it to point to it
if self.ranCommitID != '':
mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5)
# if the branch is not develop, then it is a merge request and we need to do
# the potential merge. Note that merge conflicts should already been checked earlier
imageTag = 'develop'
if (self.ranAllowMerge):
imageTag = 'ci-temp'
if self.ranTargetBranch == '':
if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
else:
logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
# Let's remove any previous run artifacts if still there
mySSH.command('docker image prune --force', '\$', 5)
mySSH.command('docker image rm ran-build:' + imageTag, '\$', 5)
for image,pattern in imageNames:
mySSH.command('docker image rm ' + image + ':' + imageTag, '\$', 5)
# Build the shared image
mySSH.command('docker build --target ran-build --tag ran-build:' + imageTag + ' --file docker/Dockerfile.ran.ubuntu18 --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 800)
# Build the target image(s)
previousImage='ran-build:' + imageTag
danglingShaOnes=[]
for image,pattern in imageNames:
mySSH.command('docker build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + '.ubuntu18 . > cmake_targets/log/' + image + '.log 2>&1', '\$', 600)
# Retrieving the dangling image(s) for the log collection
mySSH.command('docker images --filter "dangling=true" --filter "since=' + previousImage + '" -q | sed -e "s#^#sha=#"', '\$', 5)
result = re.search('sha=(?P<imageShaOne>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
if result is not None:
danglingShaOnes.append((image, result.group('imageShaOne')))
previousImage=image + ':' + imageTag
imageTag = 'ci-temp'
# First verify if images were properly created.
status = True
mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ran-build:' + imageTag, '\$', 5)
if mySSH.getBefore().count('No such object') != 0:
logging.error('Could not build properly ran-build')
status = False
else:
result = re.search('Size = (?P<size>[0-9\-]+) bytes', mySSH.getBefore())
if result is not None:
imageSize = float(result.group('size'))
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
else:
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
else:
imageSize = imageSize / 1000
logging.debug('\u001B[1m ran-build size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
else:
logging.debug('ran-build size is unknown')
for image,pattern in imageNames:
mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5)
if mySSH.getBefore().count('No such object') != 0:
logging.error('Could not build properly ' + image)
status = False
else:
result = re.search('Size = (?P<size>[0-9\-]+) bytes', mySSH.getBefore())
if result is not None:
imageSize = float(result.group('size'))
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
else:
imageSize = imageSize / 1000
if imageSize < 1000:
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
else:
imageSize = imageSize / 1000
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
else:
logging.debug('ran-build size is unknown')
if not status:
mySSH.close()
logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
if self.htmlObj is not None:
self.htmlObj.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
self.htmlObj.CreateHtmlTabFooter(False)
sys.exit(1)
# Recover build logs, for the moment only possible when build is successful
mySSH.command('docker create --name test ran-build:' + imageTag, '\$', 5)
mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5)
mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. /tmp/CI-eNB/cmake_targets/log/ran-build', '\$', 5)
mySSH.command('docker rm -f test', '\$', 5)
for image,shaone in danglingShaOnes:
mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5)
mySSH.command('docker create --name test ' + shaone, '\$', 5)
mySSH.command('docker cp test:/oai-ran/cmake_targets/log/. /tmp/CI-eNB/cmake_targets/log/' + image, '\$', 5)
mySSH.command('docker rm -f test', '\$', 5)
mySSH.command('docker image prune --force', '\$', 5)
mySSH.close()
logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
if self.htmlObj is not None:
self.htmlObj.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)
def InitializeeNB(self): def InitializeeNB(self):
if self.eNB_serverId[self.eNB_instance] == '0': if self.eNB_serverId[self.eNB_instance] == '0':
lIpAddr = self.eNBIPAddress lIpAddr = self.eNBIPAddress
......
- Build_PhySim - Build_PhySim
- Run_PhySim - Run_PhySim
- Build_eNB - Build_eNB
- Build_Image
- WaitEndBuild_eNB - WaitEndBuild_eNB
- WaitEndBuild_Image
- Initialize_eNB - Initialize_eNB
- Terminate_eNB - Terminate_eNB
- Initialize_UE - Initialize_UE
......
<!--
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The OpenAirInterface Software Alliance licenses this file to You under
the OAI Public License, Version 1.1 (the "License"); you may not use this file
except in compliance with the License.
You may obtain a copy of the License at
http://www.openairinterface.org/?page_id=698
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
For more information about the OpenAirInterface (OAI) Software Alliance:
contact@openairinterface.org
000004
-->
<testCaseList>
<htmlTabRef>build-tab</htmlTabRef>
<htmlTabName>Build</htmlTabName>
<htmlTabIcon>wrench</htmlTabIcon>
<TestCaseRequestedList>
000001
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="000001">
<class>Build_Image</class>
<desc>Build eNB Image</desc>
<kind>all</kind>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
<backgroundBuild>True</backgroundBuild>
</testCase>
<testCase id="000004">
<class>WaitEndBuild_Image</class>
<desc>Wait for end of Build eNB</desc>
<kind>eNB</kind>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
</testCase>
</testCaseList>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment