Exemple #1
0
 ec2.describeInstances([instance.platformId], function (err, instanceData) {
     if (err) {
         logger.error("Hit some error: ", err);
         return callback(err, null);
     }
     if (instanceData.Reservations.length && instanceData.Reservations[0].Instances.length) {
         logger.debug("ip =>", instanceData.Reservations[0].Instances[0].PublicIpAddress);
         instancesDao.updateInstanceIp(instance._id, instanceData.Reservations[0].Instances[0].PublicIpAddress, function (err, updateCount) {
             if (err) {
                 logger.error("update instance ip err ==>", err);
                 return callback(err, null);
             }
             logger.debug('instance ip updated');
             logger.debug("Exit get() for /instances/%s/startInstance", instance._id);
             callback(null, {
                 instanceCurrentState: state,
                 actionLogId: actionLog._id
             });
             return;
         });
     }
 });
Exemple #2
0
 checkSuccessInstanceAction(logReferenceIds, instanceState, instanceLog, actionCompleteLog, function (err, successData) {
     if (err) {
         callback(err, null);
         return;
     }
     instancesDao.updateInstanceIp(instance._id, vmResponse.ip, function (err, updateCount) {
         if (err) {
             logger.error("update instance ip err ==>", err);
             return callback(err, null);
         }
         logger.debug('instance ip upadated');
     });
     callback(null, {
         instanceCurrentState: instanceState,
         actionLogId: actionLog._id
     });
     fs.unlink('/tmp/' + provider.id + '.json', function (err) {
         if (err) {
             logger.error("Unable to delete json file.");
         }
     });
     return;
 });
Exemple #3
0
                                            azureCloud.waitforserverready(launchparamsazure.VMName, launchparamsazure.username, launchparamsazure.password, function(err, publicip) {

                                                if (!err) {
                                                    logger.debug('Instance Ready....');
                                                    logger.debug(JSON.stringify(data)); // logger.debug(data);
                                                    logger.debug('About to bootstrap Instance');
                                                    //identifying pulic ip

                                                    instancesDao.updateInstanceIp(instance.id, publicip, function(err, updateCount) {
                                                        if (err) {
                                                            logger.error("instancesDao.updateInstanceIp Failed ==>", err);
                                                            return;
                                                        }
                                                        logger.debug('Instance ip Updated');
                                                    });
                                                    instancesDao.updateInstanceState(instance.id, "running", function(err, updateCount) {
                                                        if (err) {
                                                            logger.error("instancesDao.updateInstanceState Failed ==>", err);
                                                            return;
                                                        }
                                                        logger.debug('Instance state Updated');
                                                    });

                                                    logsDao.insertLog({
                                                        referenceId: logsReferenceIds,
                                                        err: false,
                                                        log: "Instance Ready..about to bootstrap",
                                                        timestamp: timestampStarted
                                                    });

                                                    instanceLog.status = "running";
                                                    instanceLog.logs = {
                                                        err: false,
                                                        log: "Instance Ready..about to bootstrap",
                                                        timestamp: new Date().getTime()
                                                    };
                                                    instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                                        if (err) {
                                                            logger.error("Failed to create or update instanceLog: ", err);
                                                        }
                                                    });

                                                    var port = '';

                                                    if (instance.hardware.os === 'windows') {
                                                        port = '5985';
                                                    } else {
                                                        port = '22';
                                                    }
                                                    var repoData = {};
                                                    repoData['projectId'] = launchParams.blueprintData.projectId;
                                                    if (launchParams.blueprintData.nexus.repoName) {
                                                        repoData['repoName'] = launchParams.blueprintData.nexus.repoName;
                                                    } else if (launchParams.blueprintData.docker.image) {
                                                        repoData['repoName'] = launchParams.blueprintData.docker.image;
                                                    }
                                                    launchParams.blueprintData.getCookBookAttributes(instance.instanceIP, repoData, function(err, jsonAttributes) {
                                                        var runlist = instance.runlist;
                                                        logger.debug("launchParams.blueprintData.extraRunlist: ", JSON.stringify(launchParams.blueprintData.extraRunlist));
                                                        if (launchParams.blueprintData.extraRunlist) {
                                                            runlist = launchParams.blueprintData.extraRunlist.concat(instance.runlist);
                                                        }

                                                        logger.debug("runlist: ", JSON.stringify(runlist));
                                                        launchParams.infraManager.bootstrapInstance({
                                                            instanceIp: publicip,
                                                            runlist: runlist,
                                                            instanceUsername: launchparamsazure.username,
                                                            instancePassword: launchparamsazure.password, //should be the encryped file 
                                                            nodeName: launchparamsazure.VMName,
                                                            environment: launchParams.envName,
                                                            instanceOS: instance.hardware.os,
                                                            jsonAttributes: jsonAttributes,
                                                            port: port
                                                        }, function(err, code) {
                                                            fs.unlink(decryptedPemFile, function(err) {
                                                                logger.debug("Deleting decryptedPemFile..");
                                                                if (err) {
                                                                    logger.error("Error in deleting decryptedPemFile..");
                                                                }

                                                                fs.unlink(decryptedKeyFile, function(err) {
                                                                    logger.debug("Deleting decryptedKeyFile ..");
                                                                    if (err) {
                                                                        logger.error("Error in deleting decryptedKeyFile..");
                                                                    }
                                                                });
                                                            });

                                                            if (err) {
                                                                instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {
                                                                    if (err) {
                                                                        logger.error("Unable to set instance bootstarp status. code 0", err);
                                                                    }
                                                                });

                                                                var timestampEnded = new Date().getTime();
                                                                logsDao.insertLog({
                                                                    referenceId: logsReferenceIds,
                                                                    err: true,
                                                                    log: "Bootstrap failed",
                                                                    timestamp: timestampEnded
                                                                });
                                                                instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);
                                                                instanceLog.endedOn = new Date().getTime();
                                                                instanceLog.actionStatus = "failed";
                                                                instanceLog.logs = {
                                                                    err: true,
                                                                    log: "Bootstrap failed",
                                                                    timestamp: new Date().getTime()
                                                                };
                                                                instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                                                    if (err) {
                                                                        logger.error("Failed to create or update instanceLog: ", err);
                                                                    }
                                                                });
                                                                return;
                                                            }

                                                            logger.debug("Azure vm bootstrap code:", code);

                                                            if (code == 0) {
                                                                instancesDao.updateInstanceBootstrapStatus(instance.id, 'success', function(err, updateData) {
                                                                    if (err) {
                                                                        logger.error("Unable to set instance bootstarp status. code 0", err);
                                                                    }

                                                                });

                                                                var timestampEnded = new Date().getTime();

                                                                logsDao.insertLog({
                                                                    referenceId: logsReferenceIds,
                                                                    err: false,
                                                                    log: "Instance Bootstraped successfully",
                                                                    timestamp: timestampEnded
                                                                });
                                                                instancesDao.updateActionLog(instance.id, actionLog._id, true, timestampEnded);
                                                                instanceLog.endedOn = new Date().getTime();
                                                                instanceLog.actionStatus = "success";
                                                                instanceLog.logs = {
                                                                    err: false,
                                                                    log: "Instance Bootstraped successfully",
                                                                    timestamp: new Date().getTime()
                                                                };
                                                                instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                                                    if (err) {
                                                                        logger.error("Failed to create or update instanceLog: ", err);
                                                                    }
                                                                });

                                                                launchParams.infraManager.getNode(instance.chefNodeName, function(err, nodeData) {
                                                                    if (err) {
                                                                        logger.error("Failed chef.getNode", err);
                                                                        return;
                                                                    }
                                                                    instanceLog.platform = nodeData.automatic.platform;
                                                                    instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                                                        if (err) {
                                                                            logger.error("Failed to create or update instanceLog: ", err);
                                                                        }
                                                                    });
                                                                    var hardwareData = {};
                                                                    hardwareData.architecture = nodeData.automatic.kernel.machine;
                                                                    hardwareData.platform = nodeData.automatic.platform;
                                                                    hardwareData.platformVersion = nodeData.automatic.platform_version;
                                                                    hardwareData.memory = {
                                                                        total: 'unknown',
                                                                        free: 'unknown'
                                                                    };
                                                                    if (nodeData.automatic.memory) {
                                                                        hardwareData.memory.total = nodeData.automatic.memory.total;
                                                                        hardwareData.memory.free = nodeData.automatic.memory.free;
                                                                    }
                                                                    hardwareData.os = instance.hardware.os;
                                                                    instancesDao.setHardwareDetails(instance.id, hardwareData, function(err, updateData) {
                                                                        if (err) {
                                                                            logger.error("Unable to set instance hardware details  code (setHardwareDetails)", err);
                                                                        } else {
                                                                            logger.debug("Instance hardware details set successessfully");
                                                                        }
                                                                    });

                                                                });

                                                            } else {

                                                                instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {
                                                                    if (err) {
                                                                        logger.error("Unable to set instance bootstarp status code != 0", err);
                                                                    } else {
                                                                        logger.debug("Instance bootstrap status set to failed");
                                                                    }
                                                                });
                                                                var timestampEnded = new Date().getTime();
                                                                logsDao.insertLog({
                                                                    referenceId: logsReferenceIds,
                                                                    err: false,
                                                                    log: "Bootstrap Failed",
                                                                    timestamp: timestampEnded
                                                                });
                                                                instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);
                                                                instanceLog.endedOn = new Date().getTime();
                                                                instanceLog.actionStatus = "failed";
                                                                instanceLog.logs = {
                                                                    err: false,
                                                                    log: "Bootstrap Failed",
                                                                    timestamp: new Date().getTime()
                                                                };
                                                                instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                                                    if (err) {
                                                                        logger.error("Failed to create or update instanceLog: ", err);
                                                                    }
                                                                });


                                                            }
                                                        }, function(stdOutData) {

                                                            logsDao.insertLog({
                                                                referenceId: logsReferenceIds,
                                                                err: false,
                                                                log: stdOutData.toString('ascii'),
                                                                timestamp: new Date().getTime()
                                                            });

                                                            instanceLog.logs = {
                                                                err: false,
                                                                log: stdOutData.toString('ascii'),
                                                                timestamp: new Date().getTime()
                                                            };
                                                            instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                                                if (err) {
                                                                    logger.error("Failed to create or update instanceLog: ", err);
                                                                }
                                                            });



                                                        }, function(stdErrData) {

                                                            //retrying 4 times before giving up.
                                                            logsDao.insertLog({
                                                                referenceId: logsReferenceIds,
                                                                err: true,
                                                                log: stdErrData.toString('ascii'),
                                                                timestamp: new Date().getTime()
                                                            });

                                                            instanceLog.logs = {
                                                                err: false,
                                                                log: stdErrData.toString('ascii'),
                                                                timestamp: new Date().getTime()
                                                            };
                                                            instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                                                if (err) {
                                                                    logger.error("Failed to create or update instanceLog: ", err);
                                                                }
                                                            });

                                                        });
                                                    });



                                                } else {
                                                    logger.debug('Err Creating Instance:' + err);
                                                    return;
                                                }
                                            });
Exemple #4
0
								ec2.waitForInstanceRunnnigState(instance.platformId, function(err, instanceData) {
									if (err) {
										logsDao.insertLog({
											referenceId: logsReferenceIds,
											err: true,
											log: "Instance ready state wait failed. Unable to bootstrap",
											timestamp: timestampStarted
										});
										logger.error("waitForInstanceRunnnigState returned an error  >>", err);
										return;
									}
									logger.debug("Enter waitForInstanceRunnnigState :", instanceData);
									instance.instanceIP = instanceData.PublicIpAddress || instanceData.PrivateIpAddress;
									instancesDao.updateInstanceIp(instance.id, instance.instanceIP, function(err, updateCount) {
										if (err) {
											logger.error("instancesDao.updateInstanceIp Failed ==>", err);
											return;
										}
										logger.debug('Instance ip upadated');
									});

									instancesDao.updateInstanceState(instance.id, instanceData.State.Name, function(err, updateCount) {
										if (err) {
											logger.error("error(date instance state err ==>", err);
											return;
										}
										logger.debug('instance state upadated');
									});

									logger.debug('waiting for instance');
									logsDao.insertLog({
										referenceId: logsReferenceIds,
										err: false,
										log: "waiting for instance state to be ok",
										timestamp: new Date().getTime()
									});
									ec2.waitForEvent(instanceData.InstanceId, 'instanceStatusOk', function(err) {
										if (err) {
											logsDao.insertLog({
												referenceId: logsReferenceIds,
												err: true,
												log: "Instance ok state wait failed. Unable to bootstrap",
												timestamp: timestampStarted
											});
											logger.error('intance wait failed ==> ', openport, err);
											return;
										}

										logger.debug('intance wait success');


										//decrypting pem file
										var cryptoConfig = appConfig.cryptoSettings;
										var tempUncryptedPemFileLoc = appConfig.tempDir + uuid.v4();
										//cryptography.decryptFile(instance.credentials.pemFileLocation, cryptoConfig.decryptionEncoding, tempUncryptedPemFileLoc, cryptoConfig.encryptionEncoding, function(err) {
										credentialcryptography.decryptCredential(instance.credentials, function(err, decryptedCredentials) {

											if (err) {
												instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {
													if (err) {
														logger.error("Unable to set instance bootstarp status", err);
													} else {
														logger.debug("Instance bootstrap status set to failed");
													}
												});
												var timestampEnded = new Date().getTime();
												logsDao.insertLog({
													referenceId: logsReferenceIds,
													err: true,
													log: "Unable to decrpt pem file. Bootstrap failed",
													timestamp: timestampEnded
												});
												instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);

												if (instance.hardware.os != 'windows')
													return;
											}



											var repoData = {};
											repoData['projectId'] = launchParams.blueprintData.projectId;
											if (launchParams.blueprintData.nexus.repoName) {
												repoData['repoName'] = launchParams.blueprintData.nexus.repoName;
											} else if (launchParams.blueprintData.docker.image) {
												repoData['repoName'] = launchParams.blueprintData.docker.image;
											}


											launchParams.blueprintData.getCookBookAttributes(instance.instanceIP, repoData, function(err, jsonAttributes) {
												//logger.debug("jsonAttributes: ",JSON.stringify(jsonAttributes));
												var runlist = instance.runlist;
												//logger.debug("launchParams.blueprintData.extraRunlist: ", JSON.stringify(launchParams.blueprintData.extraRunlist));
												if (launchParams.blueprintData.extraRunlist) {
													runlist = launchParams.blueprintData.extraRunlist.concat(instance.runlist);
												}

												//logger.debug("runlist: ", JSON.stringify(runlist));
												var bootstrapInstanceParams = {
													instanceIp: instance.instanceIP,
													pemFilePath: decryptedCredentials.pemFileLocation,
													runlist: runlist,
													instanceUsername: instance.credentials.username,
													nodeName: instance.chef.chefNodeName,
													environment: launchParams.envName,
													instanceOS: instance.hardware.os,
													jsonAttributes: jsonAttributes,
													instancePassword: decryptedCredentials.password
												};


												launchParams.infraManager.bootstrapInstance(bootstrapInstanceParams, function(err, code) {
													
                                                    if (decryptedCredentials.pemFileLocation) {
														fileIo.removeFile(decryptedCredentials.pemFileLocation, function(err) {
															if (err) {
																logger.error("Unable to delete temp pem file =>", err);
															} else {
																logger.debug("temp pem file deleted =>", err);
															}
														});
													}


													logger.error('process stopped ==> ', err, code);
													if (err) {
														logger.error("knife launch err ==>", err);
														instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {

														});
														var timestampEnded = new Date().getTime();
														logsDao.insertLog({
															referenceId: logsReferenceIds,
															err: true,
															log: "Bootstrap failed",
															timestamp: timestampEnded
														});
														instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);


													} else {
														if (code == 0) {
															instancesDao.updateInstanceBootstrapStatus(instance.id, 'success', function(err, updateData) {
																if (err) {
																	logger.error("Unable to set instance bootstarp status. code 0", err);
																} else {
																	logger.debug("Instance bootstrap status set to success");
																}
															});
															var timestampEnded = new Date().getTime();
															logsDao.insertLog({
																referenceId: logsReferenceIds,
																err: false,
																log: "Instance Bootstraped successfully",
																timestamp: timestampEnded
															});
															instancesDao.updateActionLog(instance.id, actionLog._id, true, timestampEnded);


															launchParams.infraManager.getNode(instance.chefNodeName, function(err, nodeData) {
																if (err) {
																	logger.error("Failed chef.getNode", err);
																	return;
																}
																var hardwareData = {};
																hardwareData.architecture = nodeData.automatic.kernel.machine;
																hardwareData.platform = nodeData.automatic.platform;
																hardwareData.platformVersion = nodeData.automatic.platform_version;
																hardwareData.memory = {
																	total: 'unknown',
																	free: 'unknown'
																};
																if (nodeData.automatic.memory) {
																	hardwareData.memory.total = nodeData.automatic.memory.total;
																	hardwareData.memory.free = nodeData.automatic.memory.free;
																}
																hardwareData.os = instance.hardware.os;
																instancesDao.setHardwareDetails(instance.id, hardwareData, function(err, updateData) {
																	if (err) {
																		logger.error("Unable to set instance hardware details  code (setHardwareDetails)", err);
																	} else {
																		logger.debug("Instance hardware details set successessfully");
																	}
																});
																//Checking docker status and updating
																var _docker = new Docker();
																_docker.checkDockerStatus(instance.id,
																	function(err, retCode) {
																		if (err) {
																			logger.error("Failed _docker.checkDockerStatus", err);
																			res.send(500);
																			return;
																			//res.end('200');

																		}
																		logger.debug('Docker Check Returned:' + retCode);
																		if (retCode == '0') {
																			instancesDao.updateInstanceDockerStatus(instance.id, "success", '', function(data) {
																				logger.debug('Instance Docker Status set to Success');
																			});

																		}
																	});

															});

														} else {
															instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {
																if (err) {
																	logger.error("Unable to set instance bootstarp status code != 0", err);
																} else {
																	logger.debug("Instance bootstrap status set to failed");
																}
															});
															var timestampEnded = new Date().getTime();
															logsDao.insertLog({
																referenceId: logsReferenceIds,
																err: false,
																log: "Bootstrap Failed",
																timestamp: timestampEnded
															});
															instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);

														}
													}

												}, function(stdOutData) {

													logsDao.insertLog({
														referenceId: logsReferenceIds,
														err: false,
														log: stdOutData.toString('ascii'),
														timestamp: new Date().getTime()
													});

												}, function(stdErrData) {

													//retrying 4 times before giving up.
													logsDao.insertLog({
														referenceId: logsReferenceIds,
														err: true,
														log: stdErrData.toString('ascii'),
														timestamp: new Date().getTime()
													});


												});
											});


										});
									});
								});
Exemple #5
0
                                vmwareCloud.waitforserverready(appConfig.vmware.serviceHost, createserverdata["vm_name"], anImage.userName, anImage.instancePassword, function(err, publicip, vmdata) {
                                    if (err) {
                                        var timestampEnded = new Date().getTime();
                                        logger.error("Instance wait failes", err);
                                        instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {
                                            if (err) {
                                                logger.error("Unable to set instance bootstarp status. code 0", err);
                                            } else {
                                                logger.debug("Instance bootstrap status set to success");
                                            }
                                        });
                                        logsDao.insertLog({
                                            referenceId: logsReferenceIds,
                                            err: true,
                                            log: 'Instance not responding. Bootstrap failed',
                                            timestamp: timestampEnded
                                        });
                                        instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);

                                        return;
                                    }
                                    if (!err) {
                                        logger.debug('Instance Ready....');
                                        logger.debug(JSON.stringify(vmdata)); // logger.debug(data);
                                        logger.debug('About to bootstrap Instance');

                                        instancesDao.updateInstanceIp(instance.id, publicip, function(err, updateCount) {
                                            if (err) {
                                                logger.error("instancesDao.updateInstanceIp Failed ==>", err);
                                                return;
                                            }
                                            logger.debug('Instance ip Updated');
                                        });
                                        instancesDao.updateInstanceState(instance.id, "running", function(err, updateCount) {
                                            if (err) {
                                                logger.error("instancesDao.updateInstanceState Failed ==>", err);
                                                return;
                                            }
                                            logger.debug('Instance state Updated');
                                        });


                                        logsDao.insertLog({
                                            referenceId: logsReferenceIds,
                                            err: false,
                                            log: "Instance Ready..about to bootstrap",
                                            timestamp: timestampStarted
                                        });
                                        var repoData = {
                                            "projectId": launchParams.blueprintData.projectId,
                                            "repoName": launchParams.blueprintData.nexus.repoName
                                        };
                                        var repoData = {};
                                        repoData['projectId'] = launchParams.blueprintData.projectId;
                                        if (launchParams.blueprintData.nexus.repoName) {
                                            repoData['repoName'] = launchParams.blueprintData.nexus.repoName;
                                        } else if (launchParams.blueprintData.docker.image) {
                                            repoData['repoName'] = launchParams.blueprintData.docker.image;
                                        }
                                        launchParams.blueprintData.getCookBookAttributes(instance.instanceIP, repoData, function(err, jsonAttributes) {
                                            var runlist = instance.runlist;
                                            logger.debug("launchParams.blueprintData.extraRunlist: ", JSON.stringify(launchParams.blueprintData.extraRunlist));
                                            if (launchParams.blueprintData.extraRunlist) {
                                                runlist = launchParams.blueprintData.extraRunlist.concat(instance.runlist);
                                            }

                                            logger.debug("runlist: ", JSON.stringify(runlist));
                                            credentialcryptography.decryptCredential(instance.credentials, function(err, decryptedCredentials) {
                                                launchParams.infraManager.bootstrapInstance({
                                                    instanceIp: publicip,
                                                    runlist: runlist,
                                                    instanceUsername: anImage.userName,
                                                    instancePassword: decryptedCredentials.password, 
                                                    nodeName: createserverdata["vm_name"],
                                                    environment: launchParams.envName,
                                                    instanceOS: instance.hardware.os,
                                                    jsonAttributes: jsonAttributes
                                                }, function(err, code) {
                                                    var timestampEnded = new Date().getTime();
                                                    if (err) {
                                                        instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {
                                                            if (err) {
                                                                logger.error("Unable to set instance bootstarp status. code 0", err);
                                                            } else {


                                                                logger.debug("Instance bootstrap status set to success");
                                                            }
                                                        });
                                                        logsDao.insertLog({
                                                            referenceId: logsReferenceIds,
                                                            err: true,
                                                            log: 'Bootstrap failed',
                                                            timestamp: timestampEnded
                                                        });
                                                        instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);
                                                        return;
                                                    }
                                                    if (code == 0) {

                                                        instancesDao.updateInstanceBootstrapStatus(instance.id, 'success', function(err, updateData) {
                                                            if (err) {
                                                                logger.error("Unable to set instance bootstarp status. code 0", err);
                                                            } else {


                                                                logger.debug("Instance bootstrap status set to success");
                                                            }
                                                        });


                                                        launchParams.infraManager.getNode(instance.chefNodeName, function(err, nodeData) {
                                                            if (err) {
                                                                logger.error("Failed chef.getNode", err);
                                                                return;
                                                            }
                                                            var hardwareData = {};
                                                            hardwareData.architecture = nodeData.automatic.kernel.machine;
                                                            hardwareData.platform = nodeData.automatic.platform;
                                                            hardwareData.platformVersion = nodeData.automatic.platform_version;
                                                            hardwareData.memory = {
                                                                total: 'unknown',
                                                                free: 'unknown'
                                                            };
                                                            if (nodeData.automatic.memory) {
                                                                hardwareData.memory.total = nodeData.automatic.memory.total;
                                                                hardwareData.memory.free = nodeData.automatic.memory.free;
                                                            }
                                                            hardwareData.os = instance.hardware.os;
                                                            instancesDao.setHardwareDetails(instance.id, hardwareData, function(err, updateData) {
                                                                if (err) {
                                                                    logger.error("Unable to set instance hardware details  code (setHardwareDetails)", err);
                                                                } else {
                                                                    logger.debug("Instance hardware details set successessfully");
                                                                }
                                                            });
                                                            //Checking docker status and updating
                                                            var _docker = new Docker();
                                                            _docker.checkDockerStatus(instance.id,
                                                                function(err, retCode) {
                                                                    if (err) {
                                                                        logger.error("Failed _docker.checkDockerStatus", err);
                                                                        res.send(500);
                                                                        return;
                                                                        //res.end('200');

                                                                    }
                                                                    logger.debug('Docker Check Returned:' + retCode);
                                                                    if (retCode == '0') {
                                                                        instancesDao.updateInstanceDockerStatus(instance.id, "success", '', function(data) {
                                                                            logger.debug('Instance Docker Status set to Success');
                                                                        });

                                                                    }
                                                                });

                                                        });
                                                        logsDao.insertLog({
                                                            referenceId: logsReferenceIds,
                                                            err: false,
                                                            log: 'Instance Bootstraped Successfully.',
                                                            timestamp: timestampEnded
                                                        });
                                                        instancesDao.updateActionLog(instance.id, actionLog._id, true, timestampEnded);



                                                    } else {
                                                        instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {
                                                            if (err) {
                                                                logger.error("Unable to set instance bootstarp status. code 0", err);
                                                            } else {


                                                                logger.debug("Instance bootstrap status set to success");
                                                            }
                                                        });
                                                        logsDao.insertLog({
                                                            referenceId: logsReferenceIds,
                                                            err: true,
                                                            log: 'Bootstrap failed',
                                                            timestamp: timestampEnded
                                                        });
                                                        instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);
                                                        return;

                                                    }
                                                }, function(stdOutData) {

                                                    logsDao.insertLog({
                                                        referenceId: logsReferenceIds,
                                                        err: false,
                                                        log: stdOutData.toString('ascii'),
                                                        timestamp: new Date().getTime()
                                                    });


                                                }, function(stdErrData) {

                                                    //retrying 4 times before giving up.
                                                    logsDao.insertLog({
                                                        referenceId: logsReferenceIds,
                                                        err: true,
                                                        log: stdErrData.toString('ascii'),
                                                        timestamp: new Date().getTime()
                                                    });

                                                });
                                            });
                                        });

                                    }
                                }); //end of waitforserverready
                    openstack.waitforserverready(openstackconfig.tenantId, instanceData, function(err, data) {
                        if (!err) {
                            logger.debug('Instance Ready....');
                            logger.debug(JSON.stringify(data)); // logger.debug(data);
                            logger.debug('About to bootstrap Instance');
                            //identifying pulic ip
                            var publicip = '';
                            if (data.floatingipdata) {
                                publicip = data.floatingipdata.floatingip.floating_ip_address;

                            } else {
                                //to be handled if floating ip is not received.
                                logsDao.insertLog({
                                    referenceId: logsReferenceIds,
                                    err: false,
                                    log: "Instance was not associated with an IP",
                                    timestamp: timestampStarted
                                });
                                instanceLog.endedOn = new Date().getTime();
                                instanceLog.logs = {
                                    err: false,
                                    log: "Instance was not associated with an IP",
                                    timestamp: new Date().getTime()
                                };
                                instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                    if (err) {
                                        logger.error("Failed to create or update instanceLog: ", err);
                                    }
                                });
                            }
                            instancesDao.updateInstanceState(instance.id, "running", function(err, updateCount) {
                                if (err) {
                                    logger.error("instancesDao.updateInstanceState Failed ==>", err);
                                    return;
                                }
                                logger.debug('Instance state Updated');
                            });
                            instancesDao.updateInstanceIp(instance.id, publicip, function(err, updateCount) {
                                if (err) {
                                    logger.error("instancesDao.updateInstanceIp Failed ==>", err);
                                    return;
                                }
                                logger.debug('Instance ip Updated');
                            });


                            logsDao.insertLog({
                                referenceId: logsReferenceIds,
                                err: false,
                                log: "Instance Ready..about to bootstrap",
                                timestamp: timestampStarted
                            });
                            instanceLog.status = "running";
                            instanceLog.logs = {
                                err: false,
                                log: "Instance Ready..about to bootstrap",
                                timestamp: new Date().getTime()
                            };
                            instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                if (err) {
                                    logger.error("Failed to create or update instanceLog: ", err);
                                }
                            });
                            var repoData = {};
                            repoData['projectId'] = launchParams.blueprintData.projectId;
                            if (launchParams.blueprintData.nexus.repoName) {
                                repoData['repoName'] = launchParams.blueprintData.nexus.repoName;
                            } else if (launchParams.blueprintData.docker.image) {
                                repoData['repoName'] = launchParams.blueprintData.docker.image;
                            }
                            launchParams.blueprintData.getCookBookAttributes(instance.instanceIP, repoData, function(err, jsonAttributes) {
                                var runlist = instance.runlist;
                                logger.debug("launchParams.blueprintData.extraRunlist: ", JSON.stringify(launchParams.blueprintData.extraRunlist));
                                if (launchParams.blueprintData.extraRunlist) {
                                    runlist = launchParams.blueprintData.extraRunlist.concat(instance.runlist);
                                }

                                logger.debug("runlist: ", JSON.stringify(runlist));
                                launchParams.infraManager.bootstrapInstance({
                                    instanceIp: publicip,
                                    runlist: runlist,
                                    instanceUsername: 'ubuntu',
                                    pemFilePath: tempUncryptedPemFileLoc,
                                    nodeName: instance.chef.chefNodeName,
                                    environment: launchParams.envName,
                                    instanceOS: instance.hardware.os,
                                    jsonAttributes: jsonAttributes
                                }, function(err, code) {
                                    fs.unlink(tempUncryptedPemFileLoc, function(err) {
                                        logger.debug("Deleting decryptedPemFile..");
                                        if (err) {
                                            logger.error("Error in deleting decryptedPemFile..");
                                        }

                                    });
                                    if (err) {
                                        instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {
                                            if (err) {
                                                logger.error("Unable to set instance bootstarp status. code 0", err);
                                            }
                                        });

                                        var timestampEnded = new Date().getTime();
                                        logsDao.insertLog({
                                            referenceId: logsReferenceIds,
                                            err: true,
                                            log: "Bootstrap failed",
                                            timestamp: timestampEnded
                                        });
                                        instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);
                                        instanceLog.endedOn = new Date().getTime();
                                        instanceLog.actionStatus = "failed";
                                        instanceLog.logs = {
                                            err: true,
                                            log: "Bootstrap failed",
                                            timestamp: new Date().getTime()
                                        };
                                        if(launchParams.auditTrailId !== null){
                                            var resultTaskExecution={
                                                actionStatus : "failed",
                                                status:"failed",
                                                endedOn : new Date().getTime(),
                                                actionLogId:launchParams.actionLogId
                                            }
                                            auditTrailService.updateAuditTrail(launchParams.auditType,launchParams.auditTrailId,resultTaskExecution,function(err,auditTrail){
                                                if (err) {
                                                    logger.error("Failed to create or update bots Log: ", err);
                                                }
                                            });
                                        }
                                        instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                            if (err) {
                                                logger.error("Failed to create or update instanceLog: ", err);
                                            }
                                        });
                                        return;
                                    }

                                    if (code == 0) {
                                        instancesDao.updateInstanceBootstrapStatus(instance.id, 'success', function(err, updateData) {
                                            if (err) {
                                                logger.error("Unable to set instance bootstarp status. code 0", err);
                                            }

                                        });

                                        var timestampEnded = new Date().getTime();

                                        logsDao.insertLog({
                                            referenceId: logsReferenceIds,
                                            err: false,
                                            log: "Instance Bootstraped successfully",
                                            timestamp: timestampEnded
                                        });
                                        if(launchParams.auditTrailId !== null){
                                            var resultTaskExecution={
                                                actionStatus : "success",
                                                status:"success",
                                                endedOn : new Date().getTime(),
                                                actionLogId:launchParams.actionLogId
                                            }
                                            auditTrailService.updateAuditTrail(launchParams.auditType,launchParams.auditTrailId,resultTaskExecution,function(err,auditTrail){
                                                if (err) {
                                                    logger.error("Failed to create or update bots Log: ", err);
                                                }
                                                var botOldService = require('_pr/services/botOldService');
                                                botOldService.updateSavedTimePerBots(launchParams.botId,launchParams.auditTrailId,launchParams,auditType,function(err,data){
                                                    if (err) {
                                                        logger.error("Failed to update bots saved Time: ", err);
                                                    }
                                                });
                                            });
                                        }
                                        instancesDao.updateActionLog(instance.id, actionLog._id, true, timestampEnded);
                                        instanceLog.endedOn = new Date().getTime();
                                        instanceLog.actionStatus = "success";
                                        instanceLog.logs = {
                                            err: false,
                                            log: "Instance Bootstraped successfully",
                                            timestamp: new Date().getTime()
                                        };
                                        instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                            if (err) {
                                                logger.error("Failed to create or update instanceLog: ", err);
                                            }
                                        });
                                        launchParams.infraManager.getNode(instance.chef.chefNodeName, function(err, nodeData) {
                                            if (err) {
                                                logger.error("Failed chef.getNode", err);
                                                return;
                                            }
                                            instanceLog.platform = nodeData.automatic.platform;
                                            instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                                if (err) {
                                                    logger.error("Failed to create or update instanceLog: ", err);
                                                }
                                            });
                                            var hardwareData = {};
                                            hardwareData.architecture = nodeData.automatic.kernel.machine;
                                            hardwareData.platform = nodeData.automatic.platform;
                                            hardwareData.platformVersion = nodeData.automatic.platform_version;
                                            hardwareData.memory = {
                                                total: 'unknown',
                                                free: 'unknown'
                                            };
                                            if (nodeData.automatic.memory) {
                                                hardwareData.memory.total = nodeData.automatic.memory.total;
                                                hardwareData.memory.free = nodeData.automatic.memory.free;
                                            }
                                            hardwareData.os = instance.hardware.os;
                                            instancesDao.setHardwareDetails(instance.id, hardwareData, function(err, updateData) {
                                                if (err) {
                                                    logger.error("Unable to set instance hardware details  code (setHardwareDetails)", err);
                                                } else {
                                                    logger.debug("Instance hardware details set successessfully");
                                                }
                                            });

                                        });

                                    } else {

                                        instancesDao.updateInstanceBootstrapStatus(instance.id, 'failed', function(err, updateData) {
                                            if (err) {
                                                logger.error("Unable to set instance bootstarp status code != 0", err);
                                            } else {
                                                logger.debug("Instance bootstrap status set to failed");
                                            }
                                        });
                                        var timestampEnded = new Date().getTime();
                                        logsDao.insertLog({
                                            referenceId: logsReferenceIds,
                                            err: false,
                                            log: "Bootstrap Failed",
                                            timestamp: timestampEnded
                                        });
                                        if(launchParams.auditTrailId !== null){
                                            var resultTaskExecution={
                                                actionStatus : "failed",
                                                status:"failed",
                                                endedOn : new Date().getTime(),
                                                actionLogId:launchParams.actionLogId
                                            }
                                            auditTrailService.updateAuditTrail(launchParams.auditType,launchParams.auditTrailId,resultTaskExecution,function(err,auditTrail){
                                                if (err) {
                                                    logger.error("Failed to create or update bots Log: ", err);
                                                }
                                            });
                                        }
                                        instancesDao.updateActionLog(instance.id, actionLog._id, false, timestampEnded);
                                        instanceLog.endedOn = new Date().getTime();
                                        instanceLog.actionStatus = "failed";
                                        instanceLog.logs = {
                                            err: false,
                                            log: "Bootstrap Failed",
                                            timestamp: new Date().getTime()
                                        };
                                        instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                            if (err) {
                                                logger.error("Failed to create or update instanceLog: ", err);
                                            }
                                        });

                                    }

                                }, function(stdOutData) {

                                    logsDao.insertLog({
                                        referenceId: logsReferenceIds,
                                        err: false,
                                        log: stdOutData.toString('ascii'),
                                        timestamp: new Date().getTime()
                                    });
                                    instanceLog.logs = {
                                        err: false,
                                        log: stdOutData.toString('ascii'),
                                        timestamp: new Date().getTime()
                                    };
                                    instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                        if (err) {
                                            logger.error("Failed to create or update instanceLog: ", err);
                                        }
                                    });

                                }, function(stdErrData) {

                                    //retrying 4 times before giving up.
                                    logsDao.insertLog({
                                        referenceId: logsReferenceIds,
                                        err: true,
                                        log: stdErrData.toString('ascii'),
                                        timestamp: new Date().getTime()
                                    });
                                    instanceLog.logs = {
                                        err: true,
                                        log: stdErrData.toString('ascii'),
                                        timestamp: new Date().getTime()
                                    };
                                    instanceLogModel.createOrUpdate(actionLog._id, instance.id, instanceLog, function(err, logData) {
                                        if (err) {
                                            logger.error("Failed to create or update instanceLog: ", err);
                                        }
                                    });

                                });
                            });
                        } else {
                            logger.debug('Err Creating Instance:' + err);
                            return;
                        }
                    });