model-storyboard: do not fail if a storyboard-related SSH fails.

1 files changed, 29 insertions, 6 deletions

diff --git a/rumba/model.py b/rumba/model.py
index 094babc..bb7d213 100644
--- a/rumba/model.py
+++ b/rumba/model.py
@@ -31,6 +31,7 @@ import time
 
 import rumba.log as log
 from rumba import ssh_support
+from rumba.ssh_support import SSHException
 
 logger = log.get_logger(__name__)
 
@@ -851,14 +852,24 @@ class ClientProcess(Client):
         opt_str = self.options if self.options is not None else ""
         cmd = "./startup.sh %s %s" % (self.ap, opt_str)
         self.running = True
-        self.pid = self.node.execute_command(cmd)
+        try:
+            self.pid = self.node.execute_command(cmd)
+        except SSHException:
+            logger.warn('Could not start client %s on node %s.',
+                        self.ap, node.name)
+        logger.debug('Client app %s on node %s got pid %s.',
+                     self.ap, self.node.name, self.pid)
 
     def stop(self):
         logger.debug(
             'Killing client %s on node %s.',
             self.ap, self.node.name
         )
-        self.node.execute_command("kill %s" % self.pid)
+        try:
+            self.node.execute_command("kill %s" % self.pid)
+        except SSHException:
+                logger.warn('Could not kill client %s on node %s.',
+                            self.ap, self.node.name)
 
     def check(self):
         """Check if the process should keep running, stop it if not,
@@ -868,6 +879,7 @@ class ClientProcess(Client):
             return False
         if now - self.start_time >= self.duration:
             self.stop()
+            self.running = False
             return False
         return True
 
@@ -942,7 +954,11 @@ class Server:
                 'Starting server %s on node %s with logfile %s.',
                 self.ap, node.name, logfile
             )
-            self.pids[node] = (node.execute_commands(cmds))
+            try:
+                self.pids[node] = (node.execute_commands(cmds))
+            except SSHException:
+                logger.warn('Could not start server %s on node %s.',
+                            self.ap, node.name)
 
     def stop(self):
         for node, pid in self.pids.items():
@@ -950,7 +966,11 @@ class Server:
                 'Killing server %s on node %s.',
                 self.ap, node.name
             )
-            node.execute_command("kill %s" % pid)
+            try:
+                node.execute_command("kill %s" % pid)
+            except SSHException:
+                logger.warn('Could not kill server %s on node %s.',
+                            self.ap, node.name)
 
 
 # Base class for ARCFIRE storyboards
@@ -997,8 +1017,11 @@ class StoryBoard:
                         client_node = random.choice(self.client_nodes)
                         new_client.run(client_node)
                         self.active_clients.append(new_client)
-                self.active_clients = \
-                    [x for x in self.active_clients if x.check()]
+                surviving = []
+                for x in self.active_clients:
+                    if x.check():
+                        surviving.append(x)
+                self.active_clients = surviving
                 time.sleep(self.DEFAULT_INTERVAL)
         finally:
             for client in self.active_clients: