From da4623dd38ed8c44a2a21d330a95d40f675de3e8 Mon Sep 17 00:00:00 2001 From: Josep Puigdemont Date: Tue, 3 May 2016 12:44:07 +0200 Subject: deployment.py: stdout not consumed when deploying changes During the automatic deployment, when the environment is ready to be deployed, the deploy.py script will spawn a shell process that will perform the command "fuel deploy-changes". The standard output of this process is then piped to a "tee" process, which redirects the output to the standard output of the shell process, and to a file named cloud.log. The file is monitored by the deploy script to find out the status of the deployment, and print it to the log file of the automatic deployment script, including percentages for each node being provisioned. However, the deploy script never consumes the standard output of the shell process. If the shell process produces enough output, its standard output buffer will fill up, thus making the tee process block trying to write to its standard output, and the cloud.log file will not be updated. At this point, the deploy process, which is monitoring cloud.log, will not detect any progress in the deployment, and eventually it will time out and assume the deployment failed, although it might have finished fine after that. The solution here is to remove the "tee" process from the shell command, and instead redirect standard output to the cloud.log file. Another solution would be to actually parse the standard output of the shell command from the deploy script itself, but that would require a bit more work, as reading a line at a time might block the script. Finally, with this patch the cloud.log file won't be deleted unless the shell process has already finished. Change-Id: I03a77be42d220b1606e48fc4ca35e22d73a6e583 Signed-off-by: Josep Puigdemont --- deploy/cloud/deployment.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'deploy/cloud') diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py index 306abf006..0127d2a52 100644 --- a/deploy/cloud/deployment.py +++ b/deploy/cloud/deployment.py @@ -101,8 +101,8 @@ class Deployment(object): LOG_FILE = 'cloud.log' log('Starting deployment of environment %s' % self.env_id) - run_proc('fuel --env %s deploy-changes | strings | tee %s' - % (self.env_id, LOG_FILE)) + p = run_proc('fuel --env %s deploy-changes | strings > %s' + % (self.env_id, LOG_FILE)) ready = False for i in range(int(self.deploy_timeout)): @@ -119,7 +119,13 @@ class Deployment(object): break else: time.sleep(SLEEP_TIME) - delete(LOG_FILE) + + p.poll() + if p.returncode == None: + log('The process deploying the changes has not yet finished.') + log('''The file %s won't be deleted''' % LOG_FILE) + else: + delete(LOG_FILE) if ready: log('Environment %s successfully deployed' % self.env_id) -- cgit 1.2.3-korg