trigger the installer when the variables change (#18)

* use terraform-module-icp-deploy standalone to install, update permissions

* update documentation, support install off remote registry

* add diagrams, trigger reinstallation when configuration changes

Author: jkwong888

README.md

@@ -88,15 +88,25 @@ To move forward and create the objects, use the following command:
 terraform apply
 ```
 
-This will kick off all infrastructure objects.  Once the infrastructure is created, if no bastion host is provisioned, the installation runs on the boot master (i.e. `icp-master01`) until it completes; otherwise the installation will continue synchronously using the bastion host's public IP (by setting the number of bastion nodes in `terraform.tfvars` to 1).  
+The following diagram illustrates the process:
+
+![terraform install](imgs/terraform_icp_install.png)
+
+1. Terraform creates the infrastructure objects including EC2 instances.  
+2. The scripts in the `scripts` directory will be uploaded to an S3 bucket 
+3. The EC2 instances are configured with [cloud-init](https://cloud-init.io/) to retrieve the scripts from the S3 bucket on startup.  The `bootstrap.sh` script is executed silently on every node and bootstraps each node (install docker, prepare storage, etc).  
+4. A configuration file (`terraform.tfvars`) is generated from the outputs of the infrastructure for the [terraform-module-icp-deploy](https://github.com/ibm-cloud-architecture/terraform-module-icp-deploy) module and copied to the S3 bucket.
+5. The `start_install.sh` script is run on the first ICP master host, which clones the github module, downloads the `terraform.tfvars` file from the S3 bucket, and runs `terraform apply` in a docker container that triggers the rest of the ICP installation.  
+
+If no bastion host is provisioned, the installation runs silently on the boot master (i.e. `icp-master01`) using [cloud-init](https://cloud-init.io/) until it completes; otherwise the installation will continue synchronously using the bastion host's public IP (by setting the number of bastion nodes in `terraform.tfvars` to 1).  
 
 ```
 bastion = {
  nodes = "1"
 }
 ```
 
-The installation output will be written to `/var/log/cloud-init-output.log` for RHEL 7.4 systems and `/var/log/messages` on RHEL 7.5+ systems.  Bastion hosts are not required for normal operation of the cluster.
+The installation output will be written to `/tmp/icp_logs/start_install.log` on the first master.  Bastion hosts are not required for normal operation of the cluster, but may be desired if you want to synchronously wait for the installation to complete, such as in execution from a devops pipeline.
 
 When the installation completes,  the `/opt/ibm/cluster` directory on the boot master (i.e. `icp-master01`) is backed up to S3 in a bucket named `icpbackup-<clusterid>`, which can be used in master recovery in case one of the master nodes fails.  It is recommended after every time `terraform apply` is performed, to commit the `terraform.tfstate` into a backend so that the state is stored in source control.
 

icp-deploy.tf

@@ -142,6 +142,12 @@ resource "tls_private_key" "installkey" {
 
 # kick off the installer from the bastion node, if one exists.  otherwise it will get kicked off from cloud-init
 resource "null_resource" "start_install" {
+  # trigger a reinstall if the cluster config changes
+  triggers {
+    terraform_tfvars_contents = "${aws_s3_bucket_object.terraform_tfvars.content}"
+    icp_config_yaml_contents = "${aws_s3_bucket_object.icp_config_yaml.content}"
+  }
+
   count = "${var.bastion["nodes"] != 0 ? 1 : 0}"
 
   provisioner "remote-exec" {

imgs/terraform_icp_install.png

@@ -13,36 +13,6 @@ logmsg() {
   echo $date $hostname $1 | tee -a $logfile
 }
 
-#
-# This function will populate the local Docker repository with the ICP images.
-#
-image_load() {
-  if [[ ! -z $(docker images -q ${inception_image}) ]]; then
-    # If we don't have an image locally we'll pull from docker hub registry
-    logmsg "Not required to load images. Exiting"
-    return 0
-  fi
-
-  if [[ ! -z "${image_location}" ]]; then
-    # Decide which protocol to use
-    if [[ "${image_location:0:2}" == "s3" ]]; then
-      # stream it right out of s3 into docker
-      logmsg "Copying binary package from ${image_location} ..."
-      ${awscli} s3 cp ${image_location} /tmp 
-
-      logmsg "Loading docker images from /tmp/`basename ${image_location}` ..."
-      tar zxf /tmp/`basename ${image_location}` -O | docker load | tee -a $logfile
-
-      logmsg "Copying binary package to /opt/ibm/cluster/images ..."
-      mkdir -p /opt/ibm/cluster/images
-      mv /tmp/`basename ${image_location}` /opt/ibm/cluster/images
-
-      logmsg "Completed loading docker images from ${image_location} ..."
-    fi
-  fi
-}
-
-
 logmsg "~~~~~~~~ Starting ICP installation Code ~~~~~~~~"
 
 ##### MAIN #####
@@ -66,14 +36,13 @@ if [ -z "${awscli}" ]; then
 fi
 
 if ! docker --version; then
-  logmsg "Docker is not installed."
+  logmsg "ERROR: Docker is not installed."
   exit 1
 fi
 
 # Figure out the version
 # This will populate $org $repo and $tag
 parse_icpversion ${inception_image}
-logmsg "Populating the registry."
 logmsg "registry=${registry:-not specified} org=$org repo=$repo tag=$tag"
 
 if [ ! -z "${username}" -a ! -z "${password}" ]; then
@@ -84,7 +53,30 @@ if [ ! -z "${username}" -a ! -z "${password}" ]; then
 fi
 
 # load images
-image_load
+if [[ ! -z $(docker images -q ${inception_image}) ]]; then
+  # If we don't have an image locally we'll pull from docker hub registry
+  logmsg "Not required to load images. Exiting"
+  return 0
+fi
+
+if [[ ! -z "${image_location}" ]]; then
+  # Decide which protocol to use
+  if [[ "${image_location:0:2}" == "s3" ]]; then
+    # stream it right out of s3 into docker
+    logmsg "Copying binary package from ${image_location} ..."
+    ${awscli} s3 cp ${image_location} /tmp 
+
+    logmsg "Loading docker images from /tmp/`basename ${image_location}` ..."
+    tar zxf /tmp/`basename ${image_location}` -O | docker load | tee -a $logfile
+
+    logmsg "Copying binary package to /opt/ibm/cluster/images ..."
+    mkdir -p /opt/ibm/cluster/images
+    mv /tmp/`basename ${image_location}` /opt/ibm/cluster/images
+
+    logmsg "Completed loading docker images from ${image_location} ..."
+  fi
+fi
+
 
 inception_image=${registry}${registry:+/}${org}/${repo}:${tag}