Improve deploy.sh/uninstall.sh, particularly when a role is deleted/undeleted

Author: JoshuaFox

README.md

@@ -132,11 +132,9 @@ names start `_gcp_`. The part of the function name after `_gcp_` is used for the
   * If you use **both** `-c` and `-e` or **neither**, both types of labeling occur.
   * If you change from having Cloud Scheduler labeling to not having it, or vice versa, be sure to deploy both org-level and project-level elements , not just project elements, since this involves the org-level sink.
 * Organization-level and project-level elements
-  * First, note that Iris is an organization-level application. Iris labels all projects in an org (unless you filter it down). Iris has architecture elements which are deployed to both the org and the project.
-  * By default, deployment is of both  organization-level elements (e.g., Log Sinks) and project-level elements (e.g., App Engine app). In general, you can just use this default every time you redeploy. 
-  * Alternatively, you can have a person without org-level permissions redeploy only the project-level elements (e.g. when you change configuration), after the first deployment. Do this with the `-p` switch on `deploy.sh`.
-  * Likewise, org-level elements only are deployed when you use the `-o` switch on `deploy.sh`.
-  * If you use **both** `-p` and `-o` or **neither**, both types of elements are deployed.
+  * First, note that Iris is an organization-level application. A single instance of Iris labels all projects in an org (unless you limit it by configuration). Iris has architecture elements which are deployed to both the org and the project.
+  * If you want a person with all permissions to deploy the org-level elements and a person without org-level permissions to redeploy only the project-level elements (e.g. when you change configuration), you can do this with flags on the script. Run   `deploy.sh -h`.
+
 
 # Configuration
 

main.py

@@ -66,6 +66,7 @@
 gcp_utils.set_env()
 
 logging.info("env  is %s", sort_dict((os.environ.copy())))
+logging.info("Configuration is %s", json.dumps(config_utils.get_config()).replace("\n",""))
 
 PluginHolder.init()
 

scripts/_deploy-org.sh

@@ -4,33 +4,36 @@
 # Usage
 # - Called from deploy.sh
 
-#set -x
-
-# The following line must come before set -u
+set -x
+# The following lines must come before set -u
 if [[ -z "$IRIS_CUSTOM_ROLE" ]]; then IRIS_CUSTOM_ROLE=iris3; fi
-
+if [[ -z "$SKIP_ADDING_IAM_BINDINGS" ]] ; then SKIP_ADDING_IAM_BINDINGS=""; fi
 set -u
 set -e
 
-
-if [[ -z "$IRIS_CUSTOM_ROLE" ]]; then IRIS_CUSTOM_ROLE=iris3; fi
-
 LOG_SINK=iris_log
 
 # Get organization id for this project
-ORGID=$(curl -X POST -H "Authorization: Bearer \"$(gcloud auth print-access-token)\"" \
-  -H "Content-Type: application/json; charset=utf-8" \
-  https://cloudresourcemanager.googleapis.com/v1/projects/"${PROJECT_ID}":getAncestry | grep -A 1 organization |
-  tail -n 1 | tr -d ' ' | cut -d'"' -f4)
+ORGID=$(gcloud projects get-ancestors $PROJECT_ID --format='value(TYPE,ID)' | awk '/org/ {print $2}')
 
 set +e
 # Create custom role to run iris
-if gcloud iam roles describe "$IRIS_CUSTOM_ROLE" --organization "$ORGID"  > /dev/null; then
+existing_role=$(gcloud iam roles describe --organization "$ORGID" $IRIS_CUSTOM_ROLE --format='value(deleted,etag)')
+# existing_role variable as follows:
+# 1. For soft-deleted role, existing_role is like "True BwYSsZlhISU="
+# 2. For active role, without the string True.
+# 3. For non-existing role, empty-string
+if [ -n "$existing_role" ]; then
+  if [[ "$existing_role" == *"True"* ]]; then # It's a soft-deleted role
+    gcloud iam roles undelete -q "$IRIS_CUSTOM_ROLE"  --organization "$ORGID"  >/dev/null
+  fi
+
   gcloud iam roles update -q "$IRIS_CUSTOM_ROLE" --organization "$ORGID" --file iris-custom-role.yaml >/dev/null
   role_error=$?
+
 else
-  gcloud iam roles create -q "$IRIS_CUSTOM_ROLE"  --organization "$ORGID" --file iris-custom-role.yaml  >/dev/null
-  role_error=$?
+    gcloud iam roles create -q "$IRIS_CUSTOM_ROLE"  --organization "$ORGID" --file iris-custom-role.yaml  >/dev/null
+    role_error=$?
 fi
 
 set -e

scripts/_deploy-project.sh

@@ -9,6 +9,8 @@
 
 
 #set -x
+# The following muyst come before set -u
+if [[ -z "$SKIP_ADDING_IAM_BINDINGS" ]] ; then SKIP_ADDING_IAM_BINDINGS=""; fi
 set -u
 set -e
 

uninstall_scripts/_uninstall-for-org.sh

@@ -2,12 +2,11 @@
 # This script deletes, on the org level (see `_deploy-org.sh`):
 #  * Iris Custom role   along with the policy binding  granting this role to the  built-in App Engine service account `[email protected]`
 #  * Log sinks `iris_sink`
-
-
-#set -x
+set -x
 
 # The following line must come before set -u
 if [[ -z "$IRIS_CUSTOM_ROLE" ]]; then IRIS_CUSTOM_ROLE=iris3; fi
+if [[ -z "$SKIP_ADDING_IAM_BINDINGS" ]]; then SKIP_ADDING_IAM_BINDINGS=""; fi
 
 set -u
 set -e
@@ -22,8 +21,8 @@ gcloud organizations remove-iam-policy-binding "$ORGID" --all \
   --member "serviceAccount:$PROJECT_ID@appspot.gserviceaccount.com" \
   --role "organizations/$ORGID/roles/$IRIS_CUSTOM_ROLE" >/dev/null|| true
 
-# Just leave the role; it causes too much trouble in its "soft delete" state
-#gcloud iam roles delete -q "$IRIS_CUSTOM_ROLE" --organization "$ORGID" >/dev/null || true
+gcloud iam roles delete -q "$IRIS_CUSTOM_ROLE" --organization "$ORGID" >/dev/null || true
+
 
 if  gcloud logging sinks describe --organization="$ORGID" "$LOG_SINK" >&/dev/null; then
     svcaccount=$(gcloud logging sinks describe --organization="$ORGID" "$LOG_SINK" |

util/config_utils.py

@@ -63,24 +63,23 @@ def label_all_on_cron() -> bool:
 
 @functools.lru_cache
 def get_config() -> typing.Dict:
-    test_config = "config-test.yaml"
-    prod_config = "config.yaml"
-    if os.path.isfile(test_config):
-        config_name = test_config
+    test_config_file = "config-test.yaml"
+    prod_config_file = "config.yaml"
+    if os.path.isfile(test_config_file):
+        config_file_to_use = test_config_file
 
     else:
-        config_name = prod_config
-
-    print("Using config file:", config_name, file=sys.stderr)  # logging not yet enabled
+        config_file_to_use = prod_config_file
 
     try:
-        with open(config_name) as config_file:
-            config = yaml.full_load(config_file)
+        with open(config_file_to_use) as f:
+            config = yaml.full_load(f)
     except FileNotFoundError:
         raise FileNotFoundError(
-            f"Could not find the config-*.yaml file, specifically {config_name}. You may want to create one, based perhaps on config.yaml.original"
+            f"Could not find the needed config file {config_file_to_use}."
+            f"You may want to create one, based perhaps on config.yaml.original"
         )
-    config["config_file"] = config_name
+    config["config_file"] = config_file_to_use
 
     return config
 

util/gcp/gcp_utils.py

@@ -17,9 +17,6 @@
 
 __invocation_count = Counter()
 
-global_counter = 0
-
-
 def increment_invocation_count(path: str):
     global __invocation_count
     __invocation_count[path] += 1