Implement runScript in cls_cmd for LocalCmd/RemoteCmd, add test

A lot of CI code is python mixed with bash, e.g.,

    ssh = getConnection(host)
    ssh.run('ls')
    ssh.run('echo')

At least some of this CI code would benefit if it was written in a
simple bash script, returning error codes and potentially other
information either through stdout/stderr or files, to the calling Python
code:

    ssh = runScript(host, script)
    # script does: ls; echo

This commit introduces the possibility to run entire scripts. The idea
is that the executor has a script (on localhost), which is either
executed locally or on a remote host. For the remote host, the script is
not copied but piped into a remotely executed bash. In both cases,
output is either returned like the Cmd.run() function with returncode
and mixed stdout/stderr, or optionally redirected into a file on the
(remote) host, which can be treated further by the Python code in later
steps.

ci-scripts/cls_cmd.py

@@ -36,13 +36,22 @@ import time
 
 SSHTIMEOUT=7
 
+def is_local(host):
+	return host is None or host.lower() in ["", "none", "localhost"]
+
 # helper that returns either LocalCmd or RemoteCmd based on passed host name
 def getConnection(host, d=None):
-	if host is None or host.lower() in ["", "none", "localhost"]:
+	if is_local(host):
 		return LocalCmd(d=d)
 	else:
 		return RemoteCmd(host, d=d)
 
+def runScript(host, path, timeout, parameters=None, redirect=None, silent=False):
+	if is_local(host):
+		return LocalCmd.exec_script(path, timeout, parameters, redirect, silent)
+	else:
+		return RemoteCmd.exec_script(host, path, timeout, parameters, redirect, silent)
+
 # provides a partial interface for the legacy SSHconnection class (getBefore(), command())
 class Cmd(metaclass=abc.ABCMeta):
 	def cd(self, d, silent=False):
@@ -58,6 +67,10 @@ class Cmd(metaclass=abc.ABCMeta):
 		if not silent:
 			logging.debug(f'cd {self.cwd}')
 
+	@abc.abstractmethod
+	def exec_script(path, timeout, parameters=None, redirect=None, silent=False):
+		return
+
 	@abc.abstractmethod
 	def run(self, line, timeout=300, silent=False):
 		return
@@ -99,6 +112,22 @@ class LocalCmd(Cmd):
 			logging.debug(f'Working dir is {self.cwd}')
 		self.cp = sp.CompletedProcess(args='', returncode=0, stdout='')
 
+	def exec_script(path, timeout, parameters=None, redirect=None, silent=False):
+		if redirect and not redirect.startswith("/"):
+			raise ValueError(f"redirect must be absolute, but is {redirect}")
+		c = f"{path} {parameters}" if parameters else path
+		if not redirect:
+			ret = sp.run(c, shell=True, timeout=timeout, stdout=sp.PIPE, stderr=sp.STDOUT)
+			ret.stdout = ret.stdout.decode('utf-8').strip()
+		else:
+			with open(redirect, "w") as f:
+				ret = sp.run(c, shell=True, timeout=timeout, stdout=f, stderr=f)
+			ret.args += f" &> {redirect}"
+			ret.stdout = ""
+		if not silent:
+			logging.info(f"local> {ret.args}")
+		return ret
+
 	def run(self, line, timeout=300, silent=False, reportNonZero=True):
 		if not silent:
 			logging.info(f"local> {line}")
@@ -169,9 +198,7 @@ class RemoteCmd(Cmd):
 	def __init__(self, hostname, d=None):
 		cIdx = 0
 		self.hostname = hostname
-		logging.getLogger('paramiko').setLevel(logging.ERROR) # prevent spamming through Paramiko
-		self.client = paramiko.SSHClient()
-		self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+		self.client = RemoteCmd._ssh_init()
 		cfg = RemoteCmd._lookup_ssh_config(hostname)
 		self.cwd = d
 		self.cp = sp.CompletedProcess(args='', returncode=0, stdout='')
@@ -184,6 +211,12 @@ class RemoteCmd(Cmd):
 				cIdx +=1
 		raise Exception ("Error: max retries, did not connect to host")
 
+	def _ssh_init():
+		logging.getLogger('paramiko').setLevel(logging.ERROR) # prevent spamming through Paramiko
+		client = paramiko.SSHClient()
+		client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+		return client
+
 	def _lookup_ssh_config(hostname):
 		ssh_config = paramiko.SSHConfig()
 		user_config_file = os.path.expanduser("~/.ssh/config")
@@ -204,6 +237,30 @@ class RemoteCmd(Cmd):
 			cfg['sock'] = paramiko.ProxyCommand(ucfg['proxycommand'])
 		return cfg
 
+	def exec_script(host, path, timeout, parameters=None, redirect=None, silent=False):
+		if redirect and not redirect.startswith("/"):
+			raise ValueError(f"redirect must be absolute, but is {redirect}")
+		p = parameters if parameters else ""
+		r = f"> {redirect}" if redirect else ""
+		if not silent:
+			logging.info(f"local> ssh {host} bash -s {p} < {path} {r} # {path} from localhost")
+		client = RemoteCmd._ssh_init()
+		cfg = RemoteCmd._lookup_ssh_config(host)
+		client.connect(**cfg)
+		bash_opt = 'BASH_XTRACEFD=1' # write bash set -x output to stdout, see bash(1)
+		stdin, stdout, stderr = client.exec_command(f"{bash_opt} bash -s {p} {r}", timeout=timeout)
+		# open() the file f at path, read() it and write() it into the stdin of the bash -s cmd
+		with open(path) as f:
+			stdin.write(f.read())
+			stdin.close()
+		cmd = path
+		if parameters: cmd += f" {parameters}"
+		if redirect: cmd += f" &> {redirect}"
+		ret = sp.CompletedProcess(args=cmd, returncode=stdout.channel.recv_exit_status(), stdout=stdout.read(size=None) + stderr.read(size=None))
+		ret.stdout = ret.stdout.decode('utf-8').strip()
+		client.close()
+		return ret
+
 	def run(self, line, timeout=300, silent=False, reportNonZero=True):
 		if not silent:
 			logging.info(f"ssh[{self.hostname}]> {line}")

ci-scripts/tests/cmd.py

@@ -35,6 +35,27 @@ class TestCmd(unittest.TestCase):
             self.assertEqual(ret.returncode, 0)
             self.assertEqual(ret.stdout, "test")
 
+    def test_local_script(self):
+        ret = cls_cmd.runScript("localhost", "tests/scripts/hello-world.sh", 1)
+        self.assertEqual(ret.args, "tests/scripts/hello-world.sh")
+        self.assertEqual(ret.returncode, 0)
+        self.assertEqual(ret.stdout, "+ echo hello, world\nhello, world")
+
+        ret = cls_cmd.runScript("localhost", "tests/scripts/hello-fail.sh", 1, "TESTFAIL")
+        self.assertEqual(ret.args, "tests/scripts/hello-fail.sh TESTFAIL")
+        self.assertEqual(ret.returncode, 1)
+        self.assertEqual(ret.stdout, "TESTFAIL")
+
+        ret = cls_cmd.runScript("localhost", "tests/scripts/hello-fail.sh", 1, "TESTFAIL2", "/tmp/result")
+        self.assertEqual(ret.args, "tests/scripts/hello-fail.sh TESTFAIL2 &> /tmp/result")
+        self.assertEqual(ret.returncode, 1)
+        self.assertEqual(ret.stdout, "")
+        with cls_cmd.getConnection("localhost") as ssh:
+            ret = ssh.run("cat /tmp/result")
+            self.assertEqual(ret.args, "cat /tmp/result")
+            self.assertEqual(ret.returncode, 0)
+            self.assertEqual(ret.stdout, "TESTFAIL2")
+
     @unittest.skip("need to be able to passwordlessly SSH to localhost, also disable stty -ixon")
     def test_remote_cmd(self):
         with cls_cmd.getConnection("127.0.0.1") as cmd:
@@ -52,5 +73,27 @@ class TestCmd(unittest.TestCase):
             self.assertEqual(ret.returncode, 0)
             self.assertEqual(ret.stdout, "test")
 
+    @unittest.skip("need to be able to passwordlessly SSH to localhost, also disable stty -ixon")
+    def test_remote_script(self):
+        ret = cls_cmd.runScript("127.0.0.1", "tests/scripts/hello-world.sh", 1)
+        self.assertEqual(ret.args, "tests/scripts/hello-world.sh")
+        self.assertEqual(ret.returncode, 0)
+        self.assertEqual(ret.stdout, "+ echo hello, world\nhello, world")
+
+        ret = cls_cmd.runScript("127.0.0.1", "tests/scripts/hello-fail.sh", 1, "TESTFAIL")
+        self.assertEqual(ret.args, "tests/scripts/hello-fail.sh TESTFAIL")
+        self.assertEqual(ret.returncode, 1)
+        self.assertEqual(ret.stdout, "TESTFAIL")
+
+        ret = cls_cmd.runScript("127.0.0.1", "tests/scripts/hello-fail.sh", 1, "TESTFAIL2", "/tmp/result")
+        self.assertEqual(ret.args, "tests/scripts/hello-fail.sh TESTFAIL2 &> /tmp/result")
+        self.assertEqual(ret.returncode, 1)
+        self.assertEqual(ret.stdout, "")
+        with cls_cmd.getConnection("localhost") as ssh:
+            ret = ssh.run("cat /tmp/result")
+            self.assertEqual(ret.args, "cat /tmp/result")
+            self.assertEqual(ret.returncode, 0)
+            self.assertEqual(ret.stdout, "TESTFAIL2")
+
 if __name__ == '__main__':
 	unittest.main()