From 8371109a9369bed0d89dbfe961c2f89d4a021b05 Mon Sep 17 00:00:00 2001 From: Remi Hardy <remi.hardy@openairinterface.org> Date: Wed, 16 Jun 2021 22:22:34 +0200 Subject: [PATCH] adding new class for log mgt/rotation on remote servers --- ci-scripts/cls_log_mgt.py | 103 ++++++++++++++++++++++++++++++++++++ ci-scripts/cls_module_ue.py | 8 +++ 2 files changed, 111 insertions(+) create mode 100644 ci-scripts/cls_log_mgt.py diff --git a/ci-scripts/cls_log_mgt.py b/ci-scripts/cls_log_mgt.py new file mode 100644 index 0000000000..36e7d51a4b --- /dev/null +++ b/ci-scripts/cls_log_mgt.py @@ -0,0 +1,103 @@ +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# +# Required Python Version +# Python 3.x +# +#--------------------------------------------------------------------- + +#USAGE: +# log=Log_Mgt(IPAddress,Password,Path) +# log.LogRotation() + + + + +import re +import subprocess +import logging +import math + +class Log_Mgt: + + def __init__(self,IPAddress,Password,Path): + self.IPAddress=IPAddress + self.Password=Password + self.path=Path + +#-----------------$ +#PRIVATE# Methods$ +#-----------------$ + + + def __CheckAvailSpace(self): + HOST=self.IPAddress + COMMAND="df "+ self.path + ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + result = ssh.stdout.readlines() + s=result[1].decode('utf-8').rstrip()#result[1] is the second line with the results we are looking for + tmp=s.split() + return tmp[3] #return avail space from the line + + def __GetOldestFile(self): + HOST=self.IPAddress + COMMAND="ls -rtl "+ self.path #-rtl will bring oldest file on top + ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + result = ssh.stdout.readlines() + s=result[1].decode('utf-8').rstrip() + tmp=s.split() + return tmp[8]#return filename from the line + + + def __AvgSize(self): + HOST=self.IPAddress + COMMAND="ls -rtl "+ self.path + ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + result = ssh.stdout.readlines() + total_size=0 + for i in range(1,len(result)): + s=result[i].decode('utf-8').rstrip() + tmp=s.split() + total_size+=int(tmp[4]) #get filesize + return math.floor(total_size/(len(result)-1)) #compute average file/artifact size + + +#-----------------$ +#PUBLIC Methods$ +#-----------------$ + + + def LogRotation(self): + avail_space =int(self.__CheckAvailSpace())*1000 #avail space in target folder, initially displayed in Gb + avg_size=self.__AvgSize() #average size of artifacts in the target folder + logging.debug("Avail Space : " + str(avail_space) + " / Artifact Avg Size : " + str(avg_size)) + if avail_space < 2*avg_size: #reserved space is 2x artifact file ; oldest file will be deleted + oldestfile=self.__GetOldestFile() + HOST=self.IPAddress + COMMAND="echo " + self.Password + " | sudo -S rm "+ self.path + "/" + oldestfile + logging.debug(COMMAND) + ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + else: + logging.debug("Still some space left for artifacts storage") + + + + diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index f93cb9f036..7f5c81d988 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -39,6 +39,9 @@ import subprocess from datetime import datetime +#for log rotation mgt +import cls_log_mgt + class Module_UE: def __init__(self,Module): @@ -157,6 +160,11 @@ class Module_UE: now_string = now.strftime("%Y%m%d-%H%M") source='ci_qlog' destination='/opt/ci_qlogs/ci_qlog_'+now_string+'.zip' + #qlog artifact is zipped into the target folder mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S zip -r '+destination+' '+source+' &','\$', 10) mySSH.close() + #post action : log cleaning to make sure enough space is reserved for the next run + Log_Mgt=cls_log_mgt.Log_Mgt(self.HostIPAddress, self.HostPassword, "/opt/ci_qlogs") + Log_Mgt.LogRotation() + return destination -- 2.26.2