From c88a12e33ee1aec2d5f7be0556e7a4373d3b515d Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Fri, 19 Apr 2024 21:01:09 +0200
Subject: [PATCH 01/19] try coldsnap

---
 .github/workflows/upload-legacy-ami.yml      |  13 +-
 tf/iam_github_actions.tf                     |  11 ++
 upload-ami/default.nix                       |   4 +
 upload-ami/pyproject.toml                    |   1 +
 upload-ami/src/upload_ami/upload_ami.py      |   5 +-
 upload-ami/src/upload_ami/upload_coldsnap.py | 142 +++++++++++++++++++
 6 files changed, 167 insertions(+), 9 deletions(-)
 create mode 100644 upload-ami/src/upload_ami/upload_coldsnap.py

diff --git a/.github/workflows/upload-legacy-ami.yml b/.github/workflows/upload-legacy-ami.yml
index 8cfec26e..5dd545a6 100644
--- a/.github/workflows/upload-legacy-ami.yml
+++ b/.github/workflows/upload-legacy-ami.yml
@@ -47,26 +47,23 @@ jobs:
         run: |
           image_info='${{ steps.download_ami.outputs.image_info }}'
           images_bucket='${{ vars.IMAGES_BUCKET }}'
-          image_ids=$(nix run .#upload-ami -- \
+          image_id=$(nix run .#upload-coldsnap -- \
             --image-info "$image_info" \
-            --prefix "smoketest/" \
-            --s3-bucket "$images_bucket")
-          echo "image_ids=$image_ids" >> "$GITHUB_OUTPUT"
+            --prefix "smoketest-coldsnap/")
+          echo "image_id=$image_id" >> "$GITHUB_OUTPUT"
 
       - name: Smoke test
         id: smoke_test
         # NOTE: make sure smoke test isn't cancelled. Such that instance gets cleaned up.
         run: |
-          image_ids='${{ steps.upload_smoke_test_ami.outputs.image_ids }}'
-          image_id=$(echo "$image_ids" | jq -r '.["${{ vars.AWS_REGION }}"]')
+          image_id='${{ steps.upload_smoke_test_ami.outputs.image_id }}'
           run_id='${{ github.run_id }}'
           nix run .#smoke-test -- --image-id "$image_id"
 
       - name: Clean up smoke test
         if: ${{ cancelled() }}
         run: |
-          image_ids='${{ steps.upload_smoke_test_ami.outputs.image_ids }}'
-          image_id=$(echo "$image_ids" | jq -r '.["${{ vars.AWS_REGION }}"]')
+          image_id='${{ steps.upload_smoke_test_ami.outputs.image_id }}'
           run_id='${{ github.run_id }}'
           nix run .#smoke-test -- --image-id "$image_id" --cancel
       
diff --git a/tf/iam_github_actions.tf b/tf/iam_github_actions.tf
index a84082e6..f028a9d1 100644
--- a/tf/iam_github_actions.tf
+++ b/tf/iam_github_actions.tf
@@ -34,6 +34,17 @@ data "aws_iam_policy_document" "upload_ami" {
     ]
     resources = ["*"]
   }
+  statement {
+    effect = "Allow"
+    actions = [
+      "ebs:StartSnapshot",
+      "ebs:PutSnapshotBlock",
+      "ebs:ListChangedBlocks",
+      "ebs:ListSnapshotBlocks",
+      "ebs:CompleteSnapshot",
+    ]
+    resources = ["arn:aws:ec2:*:*:snapshot/*"]
+  }
   statement {
     effect = "Allow"
     actions = [
diff --git a/upload-ami/default.nix b/upload-ami/default.nix
index c4f16ae4..d307a5f0 100644
--- a/upload-ami/default.nix
+++ b/upload-ami/default.nix
@@ -1,6 +1,7 @@
 { buildPythonApplication
 , python3Packages
 , lib
+, coldsnap
 }:
 
 let
@@ -37,6 +38,9 @@ buildPythonApplication {
       python3Packages.black
     ];
 
+
+  makeWrapperArgs = [ "--prefix PATH : ${coldsnap}/bin" ];
+
   propagatedBuildInputs = lib.flatten (map resolvePackages pyproject.project.dependencies);
 
   checkPhase = ''
diff --git a/upload-ami/pyproject.toml b/upload-ami/pyproject.toml
index c483e56b..7cf32694 100644
--- a/upload-ami/pyproject.toml
+++ b/upload-ami/pyproject.toml
@@ -17,5 +17,6 @@ disable-image-block-public-access = "upload_ami.disable_image_block_public_acces
 enable-regions = "upload_ami.enable_regions:main"
 request-public-ami-quota-increase  = "upload_ami.request_public_ami_quota_increase:main"
 describe-images = "upload_ami.describe_images:main"
+upload-coldsnap = "upload_ami.upload_coldsnap:main"
 [tool.mypy]
 strict=true
diff --git a/upload-ami/src/upload_ami/upload_ami.py b/upload-ami/src/upload_ami/upload_ami.py
index ac53a0ba..9a85fef7 100644
--- a/upload-ami/src/upload_ami/upload_ami.py
+++ b/upload-ami/src/upload_ami/upload_ami.py
@@ -9,12 +9,15 @@
 import botocore.exceptions
 
 from mypy_boto3_ec2.client import EC2Client
-from mypy_boto3_ec2.literals import BootModeValuesType
 from mypy_boto3_ec2.type_defs import RegionTypeDef
 from mypy_boto3_s3.client import S3Client
 
 from concurrent.futures import ThreadPoolExecutor
 
+from typing import TypedDict
+
+from mypy_boto3_ec2.literals import BootModeValuesType
+
 
 class ImageInfo(TypedDict):
     file: str
diff --git a/upload-ami/src/upload_ami/upload_coldsnap.py b/upload-ami/src/upload_ami/upload_coldsnap.py
new file mode 100644
index 00000000..7156aa4c
--- /dev/null
+++ b/upload-ami/src/upload_ami/upload_coldsnap.py
@@ -0,0 +1,142 @@
+import argparse
+import json
+import logging
+from re import I
+import boto3
+import subprocess
+from typing import Literal, TypedDict
+from mypy_boto3_ec2 import EC2Client
+from mypy_boto3_ec2.literals import BootModeValuesType
+
+
+class ImageInfo(TypedDict):
+    file: str
+    label: str
+    system: str
+    boot_mode: BootModeValuesType
+    format: str
+
+
+def register_image_if_not_exists(
+    ec2: EC2Client,
+    image_name: str,
+    image_info: ImageInfo,
+    snapshot_id: str,
+    public: bool,
+) -> str:
+    """
+    Register image if it doesn't exist yet
+
+    This function is idempotent because image_name is unique
+    """
+    describe_images = ec2.describe_images(
+        Owners=["self"], Filters=[{"Name": "name", "Values": [image_name]}]
+    )
+    if len(describe_images["Images"]) != 0:
+        assert len(describe_images["Images"]) == 1
+        assert "ImageId" in describe_images["Images"][0]
+        image_id = describe_images["Images"][0]["ImageId"]
+    else:
+        architecture: Literal["x86_64", "arm64"]
+        assert "system" in image_info
+        if image_info["system"] == "x86_64-linux":
+            architecture = "x86_64"
+        elif image_info["system"] == "aarch64-linux":
+            architecture = "arm64"
+        else:
+            raise Exception("Unknown system: " + image_info["system"])
+
+        logging.info(f"Registering image {image_name} with snapshot {snapshot_id}")
+
+        # TODO(arianvp): Not all instance types support TPM 2.0 yet. We should
+        # upload two images, one with and one without TPM 2.0 support.
+
+        # if architecture == "x86_64" and image_info["boot_mode"] == "uefi":
+        #    tpmsupport['TpmSupport'] = "v2.0"
+
+        register_image = ec2.register_image(
+            Name=image_name,
+            Architecture=architecture,
+            BootMode=image_info["boot_mode"],
+            BlockDeviceMappings=[
+                {
+                    "DeviceName": "/dev/xvda",
+                    "Ebs": {
+                        "SnapshotId": snapshot_id,
+                        "VolumeType": "gp3",
+                    },
+                }
+            ],
+            RootDeviceName="/dev/xvda",
+            VirtualizationType="hvm",
+            EnaSupport=True,
+            ImdsSupport="v2.0",
+            SriovNetSupport="simple",
+        )
+        image_id = register_image["ImageId"]
+
+    ec2.get_waiter("image_available").wait(ImageIds=[image_id])
+    if public:
+        logging.info(f"Making {image_id} public")
+        ec2.modify_image_attribute(
+            ImageId=image_id,
+            Attribute="launchPermission",
+            LaunchPermission={"Add": [{"Group": "all"}]},
+        )
+    return image_id
+
+
+def upload_coldsnap(
+    *,
+    image_info: ImageInfo,
+    prefix: str,
+) -> str:
+    logging.info(f"Uploading image to coldsnap")
+
+    snapshot_id = str(
+        subprocess.check_output(
+            [
+                "coldsnap",
+                "upload",
+                "--wait",
+            ]
+        )
+    )
+
+    ec2 = boto3.client("ec2")
+    image_name = prefix + image_info["label"] + "-" + image_info["system"]
+
+    image_id = register_image_if_not_exists(
+        ec2=ec2,
+        image_name=image_name,
+        image_info=image_info,
+        snapshot_id=snapshot_id,
+        public=False,
+    )
+    return image_id
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--image-info", help="Path to image info", required=True)
+    parser.add_argument("--prefix", help="Prefix for image name", required=True)
+    parser.add_argument("--debug", action="store_true")
+
+    args = parser.parse_args()
+
+    if args.debug:
+        level = logging.DEBUG
+    else:
+        level = logging.INFO
+    logging.basicConfig(level=level)
+
+    image_info: ImageInfo
+    with open(args.image_info) as f:
+        image_info = json.load(f)
+
+    print(
+        upload_coldsnap(
+            image_info=args.image_info,
+            prefix=args.prefix,
+        )
+    )

From af73e0f6373040344e5d8b9ed104600b4e9b6dc2 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Fri, 19 Apr 2024 21:17:45 +0200
Subject: [PATCH 02/19] fix cli

---
 upload-ami/src/upload_ami/upload_coldsnap.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/upload-ami/src/upload_ami/upload_coldsnap.py b/upload-ami/src/upload_ami/upload_coldsnap.py
index 7156aa4c..6682ff15 100644
--- a/upload-ami/src/upload_ami/upload_coldsnap.py
+++ b/upload-ami/src/upload_ami/upload_coldsnap.py
@@ -99,6 +99,7 @@ def upload_coldsnap(
                 "coldsnap",
                 "upload",
                 "--wait",
+                image_info["file"],
             ]
         )
     )

From fcc5dd9070164d0504d3333612137dceaba02d87 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Fri, 19 Apr 2024 21:20:20 +0200
Subject: [PATCH 03/19] Fix another typo

---
 upload-ami/src/upload_ami/upload_coldsnap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/upload-ami/src/upload_ami/upload_coldsnap.py b/upload-ami/src/upload_ami/upload_coldsnap.py
index 6682ff15..ffb3365e 100644
--- a/upload-ami/src/upload_ami/upload_coldsnap.py
+++ b/upload-ami/src/upload_ami/upload_coldsnap.py
@@ -137,7 +137,7 @@ def main() -> None:
 
     print(
         upload_coldsnap(
-            image_info=args.image_info,
+            image_info=image_info,
             prefix=args.prefix,
         )
     )

From a33d952b4c23359756baef4f06e73b4b024a0c7a Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Fri, 19 Apr 2024 21:52:26 +0200
Subject: [PATCH 04/19] fix snapshot_id

---
 upload-ami/src/upload_ami/upload_coldsnap.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/upload-ami/src/upload_ami/upload_coldsnap.py b/upload-ami/src/upload_ami/upload_coldsnap.py
index ffb3365e..41c346f7 100644
--- a/upload-ami/src/upload_ami/upload_coldsnap.py
+++ b/upload-ami/src/upload_ami/upload_coldsnap.py
@@ -93,16 +93,14 @@ def upload_coldsnap(
 ) -> str:
     logging.info(f"Uploading image to coldsnap")
 
-    snapshot_id = str(
-        subprocess.check_output(
+    snapshot_id = subprocess.check_output(
             [
                 "coldsnap",
                 "upload",
                 "--wait",
                 image_info["file"],
             ]
-        )
-    )
+        ).decode().strip()
 
     ec2 = boto3.client("ec2")
     image_name = prefix + image_info["label"] + "-" + image_info["system"]

From 159b2611809705015634e887fa2c87e590e3ac0f Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Fri, 19 Apr 2024 22:13:15 +0200
Subject: [PATCH 05/19] Use raw image instead of vhd

---
 upload-ami/default.nix                       |  3 ++-
 upload-ami/src/upload_ami/upload_coldsnap.py | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/upload-ami/default.nix b/upload-ami/default.nix
index d307a5f0..bcd8ca87 100644
--- a/upload-ami/default.nix
+++ b/upload-ami/default.nix
@@ -2,6 +2,7 @@
 , python3Packages
 , lib
 , coldsnap
+, qemu
 }:
 
 let
@@ -39,7 +40,7 @@ buildPythonApplication {
     ];
 
 
-  makeWrapperArgs = [ "--prefix PATH : ${coldsnap}/bin" ];
+  makeWrapperArgs = [ "--prefix PATH : ${coldsnap}/bin" "--prefix PATH : ${qemu}/bin" ];
 
   propagatedBuildInputs = lib.flatten (map resolvePackages pyproject.project.dependencies);
 
diff --git a/upload-ami/src/upload_ami/upload_coldsnap.py b/upload-ami/src/upload_ami/upload_coldsnap.py
index 41c346f7..90c86ee6 100644
--- a/upload-ami/src/upload_ami/upload_coldsnap.py
+++ b/upload-ami/src/upload_ami/upload_coldsnap.py
@@ -1,7 +1,12 @@
 import argparse
+from email.mime import image
 import json
 import logging
+from pathlib import Path
+from posixpath import basename
 from re import I
+from tempfile import mktemp
+import tempfile
 import boto3
 import subprocess
 from typing import Literal, TypedDict
@@ -93,12 +98,19 @@ def upload_coldsnap(
 ) -> str:
     logging.info(f"Uploading image to coldsnap")
 
+    image_file_vhd = Path(image_info["file"])
+    image_file_raw = image_file_vhd.with_suffix(".raw")
+
+    subprocess.check_call(
+        [ "qemu-img" , "convert", "-O", "raw", image_file_vhd, image_file_raw ]
+    )
+
     snapshot_id = subprocess.check_output(
             [
                 "coldsnap",
                 "upload",
                 "--wait",
-                image_info["file"],
+                image_file_raw,
             ]
         ).decode().strip()
 

From 4c38c5c0242d2a0e10d63c31cb4b28f1c140d3f1 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Fri, 19 Apr 2024 22:23:34 +0200
Subject: [PATCH 06/19] Add qemu-utils

---
 upload-ami/default.nix                       |  7 ++++---
 upload-ami/src/upload_ami/upload_coldsnap.py | 12 ++++++++----
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/upload-ami/default.nix b/upload-ami/default.nix
index bcd8ca87..17bbb51e 100644
--- a/upload-ami/default.nix
+++ b/upload-ami/default.nix
@@ -2,7 +2,7 @@
 , python3Packages
 , lib
 , coldsnap
-, qemu
+, qemu-utils
 }:
 
 let
@@ -40,9 +40,10 @@ buildPythonApplication {
     ];
 
 
-  makeWrapperArgs = [ "--prefix PATH : ${coldsnap}/bin" "--prefix PATH : ${qemu}/bin" ];
 
-  propagatedBuildInputs = lib.flatten (map resolvePackages pyproject.project.dependencies);
+  propagatedBuildInputs =
+    lib.flatten (map resolvePackages pyproject.project.dependencies)
+    ++ [ coldsnap qemu-utils ];
 
   checkPhase = ''
     mypy src
diff --git a/upload-ami/src/upload_ami/upload_coldsnap.py b/upload-ami/src/upload_ami/upload_coldsnap.py
index 90c86ee6..da470be7 100644
--- a/upload-ami/src/upload_ami/upload_coldsnap.py
+++ b/upload-ami/src/upload_ami/upload_coldsnap.py
@@ -4,7 +4,7 @@
 import logging
 from pathlib import Path
 from posixpath import basename
-from re import I
+from re import I, sub
 from tempfile import mktemp
 import tempfile
 import boto3
@@ -102,17 +102,21 @@ def upload_coldsnap(
     image_file_raw = image_file_vhd.with_suffix(".raw")
 
     subprocess.check_call(
-        [ "qemu-img" , "convert", "-O", "raw", image_file_vhd, image_file_raw ]
+        ["qemu-img", "convert", "-O", "raw", image_file_vhd, image_file_raw]
     )
 
-    snapshot_id = subprocess.check_output(
+    snapshot_id = (
+        subprocess.check_output(
             [
                 "coldsnap",
                 "upload",
                 "--wait",
                 image_file_raw,
             ]
-        ).decode().strip()
+        )
+        .decode()
+        .strip()
+    )
 
     ec2 = boto3.client("ec2")
     image_name = prefix + image_info["label"] + "-" + image_info["system"]

From e17f65e947ea86f63f58e8c6f4e72e6f8952d514 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Fri, 19 Apr 2024 22:27:33 +0200
Subject: [PATCH 07/19] use current directory

---
 upload-ami/src/upload_ami/upload_coldsnap.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/upload-ami/src/upload_ami/upload_coldsnap.py b/upload-ami/src/upload_ami/upload_coldsnap.py
index da470be7..2b54fdb8 100644
--- a/upload-ami/src/upload_ami/upload_coldsnap.py
+++ b/upload-ami/src/upload_ami/upload_coldsnap.py
@@ -99,7 +99,8 @@ def upload_coldsnap(
     logging.info(f"Uploading image to coldsnap")
 
     image_file_vhd = Path(image_info["file"])
-    image_file_raw = image_file_vhd.with_suffix(".raw")
+    image_file_raw = Path(image_file_vhd.with_suffix(".raw").name)
+
 
     subprocess.check_call(
         ["qemu-img", "convert", "-O", "raw", image_file_vhd, image_file_raw]

From 6a7fce3855ceeee12797611bedf3308a68789fc7 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sat, 20 Apr 2024 16:50:01 +0200
Subject: [PATCH 08/19] Add some logs

---
 upload-ami/src/upload_ami/upload_coldsnap.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/upload-ami/src/upload_ami/upload_coldsnap.py b/upload-ami/src/upload_ami/upload_coldsnap.py
index 2b54fdb8..5af40f93 100644
--- a/upload-ami/src/upload_ami/upload_coldsnap.py
+++ b/upload-ami/src/upload_ami/upload_coldsnap.py
@@ -101,17 +101,17 @@ def upload_coldsnap(
     image_file_vhd = Path(image_info["file"])
     image_file_raw = Path(image_file_vhd.with_suffix(".raw").name)
 
-
+    logging.info(f"Converting {image_file_vhd} to {image_file_raw}")
     subprocess.check_call(
         ["qemu-img", "convert", "-O", "raw", image_file_vhd, image_file_raw]
     )
 
+    logging.info(f"Uploading {image_file_raw} to ebs")
     snapshot_id = (
         subprocess.check_output(
             [
                 "coldsnap",
                 "upload",
-                "--wait",
                 image_file_raw,
             ]
         )
@@ -119,9 +119,12 @@ def upload_coldsnap(
         .strip()
     )
 
+    logging.info(f"Waiting for snapshot {snapshot_id} to be available")
     ec2 = boto3.client("ec2")
-    image_name = prefix + image_info["label"] + "-" + image_info["system"]
+    ec2.get_waiter("snapshot_completed").wait(SnapshotIds=[snapshot_id])
 
+    image_name = prefix + image_info["label"] + "-" + image_info["system"]
+    logging.info(f"Registering image {image_name}")
     image_id = register_image_if_not_exists(
         ec2=ec2,
         image_name=image_name,

From ed1213fce26201156f0eca62db54b335f282b532 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sat, 20 Apr 2024 17:05:32 +0200
Subject: [PATCH 09/19] Dont coldsnap in parallel. we're hitting some quota I
 think

---
 .github/workflows/upload-legacy-ami.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/upload-legacy-ami.yml b/.github/workflows/upload-legacy-ami.yml
index 5dd545a6..71865711 100644
--- a/.github/workflows/upload-legacy-ami.yml
+++ b/.github/workflows/upload-legacy-ami.yml
@@ -21,7 +21,8 @@ jobs:
       matrix:
         system:
           - x86_64-linux
-          - aarch64-linux
+          # Lets disable parallelism for now.  I think it's making EBS-direct slow
+          # - aarch64-linux
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - uses: DeterminateSystems/nix-installer-action@cd46bde16ab981b0a7b2dce0574509104543276e # v9

From 09e86515d52cd8fa79695ffc4e0c952dfbb27d23 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sun, 21 Apr 2024 15:01:01 +0200
Subject: [PATCH 10/19] Remove legacy flow for now

---
 .github/workflows/upload-legacy-ami.yml | 117 ------------------------
 1 file changed, 117 deletions(-)
 delete mode 100644 .github/workflows/upload-legacy-ami.yml

diff --git a/.github/workflows/upload-legacy-ami.yml b/.github/workflows/upload-legacy-ami.yml
deleted file mode 100644
index 71865711..00000000
--- a/.github/workflows/upload-legacy-ami.yml
+++ /dev/null
@@ -1,117 +0,0 @@
-name: Upload Legacy Amazon Image
-permissions:
-  contents: read
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-  workflow_dispatch:
-  schedule:
-    - cron: '0 0 * * 0' 
-jobs:
-  upload-ami:
-    name: Upload Legacy Amazon Image
-    runs-on: ubuntu-latest
-    environment: images
-    permissions:
-      contents: read
-      id-token: write
-    strategy:
-      matrix:
-        system:
-          - x86_64-linux
-          # Lets disable parallelism for now.  I think it's making EBS-direct slow
-          # - aarch64-linux
-    steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - uses: DeterminateSystems/nix-installer-action@cd46bde16ab981b0a7b2dce0574509104543276e # v9
-      - uses: DeterminateSystems/magic-nix-cache-action@eeabdb06718ac63a7021c6132129679a8e22d0c7 # v3
-      # NOTE: We download the AMI from Hydra instead of building it ourselves
-      # because aarch64 is currently not supported by AWS EC2 and the legacy
-      # image builder requires nested virtualization.
-      - name: Download AMI from Hydra
-        id: download_ami
-        run: |
-          set -o pipefail
-          out=$(curl --location --silent --fail-with-body --header 'Accept: application/json' https://hydra.nixos.org/job/nixos/release-23.11/nixos.amazonImage.${{ matrix.system }}/latest-finished  | jq --raw-output '.buildoutputs.out.path')
-          nix-store --realise "$out" --add-root ./result
-          echo "image_info=$out/nix-support/image-info.json" >> "$GITHUB_OUTPUT"
-
-      - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
-        with:
-          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/upload-ami
-          aws-region: ${{ vars.AWS_REGION }}
-
-      - name: Upload Smoke test AMI
-        id: upload_smoke_test_ami
-        run: |
-          image_info='${{ steps.download_ami.outputs.image_info }}'
-          images_bucket='${{ vars.IMAGES_BUCKET }}'
-          image_id=$(nix run .#upload-coldsnap -- \
-            --image-info "$image_info" \
-            --prefix "smoketest-coldsnap/")
-          echo "image_id=$image_id" >> "$GITHUB_OUTPUT"
-
-      - name: Smoke test
-        id: smoke_test
-        # NOTE: make sure smoke test isn't cancelled. Such that instance gets cleaned up.
-        run: |
-          image_id='${{ steps.upload_smoke_test_ami.outputs.image_id }}'
-          run_id='${{ github.run_id }}'
-          nix run .#smoke-test -- --image-id "$image_id"
-
-      - name: Clean up smoke test
-        if: ${{ cancelled() }}
-        run: |
-          image_id='${{ steps.upload_smoke_test_ami.outputs.image_id }}'
-          run_id='${{ github.run_id }}'
-          nix run .#smoke-test -- --image-id "$image_id" --cancel
-      
-
-      # NOTE: We do not pass run-id as we're not  building the image ourselves
-      # and we thus need to poll hydra periodically.  Including the run-id would
-      # cause us to register  the same snapshot as an image over and over again
-      # for each run.
-      - name: Upload AMIs to all available regions
-        if: github.ref == 'refs/heads/main' 
-        run: |
-          image_info='${{ steps.download_ami.outputs.image_info }}'
-          images_bucket='${{ vars.IMAGES_BUCKET }}'
-          nix run .#upload-ami -- \
-            --image-info "$image_info" \
-            --prefix "nixos/" \
-            --s3-bucket "$images_bucket" \
-            --copy-to-regions \
-            --public
-
-  deploy-pages:
-    name: Deploy images page
-    if: github.ref == 'refs/heads/main'
-    runs-on: ubuntu-latest
-    needs: upload-ami
-    permissions:
-      contents: read
-      id-token: write
-      pages: write
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - uses: DeterminateSystems/nix-installer-action@cd46bde16ab981b0a7b2dce0574509104543276e # v9
-      - uses: DeterminateSystems/magic-nix-cache-action@eeabdb06718ac63a7021c6132129679a8e22d0c7 # v3
-      - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.1
-        with:
-          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/github-pages
-          aws-region: ${{ vars.AWS_REGION }}
-      - name: Describe images
-        run: nix run .#describe-images > ./site/images.json
-      - name: Upload pages
-        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1
-        with:
-          path: ./site
-      - name: Deploy pages
-        uses: actions/deploy-pages@decdde0ac072f6dcbe43649d82d9c635fff5b4e4 # v4.0.4
-        id: deployment
-        if: github.ref == 'refs/heads/main' 

From 73df2d180294ff0dddffc10ba0d2ae0a09549f7c Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sun, 21 Apr 2024 15:21:20 +0200
Subject: [PATCH 11/19] Try out coldsnap directly

---
 .github/workflows/ami.yml | 18 ++++++++++++++++++
 flake.nix                 | 13 +++----------
 2 files changed, 21 insertions(+), 10 deletions(-)
 create mode 100644 .github/workflows/ami.yml

diff --git a/.github/workflows/ami.yml b/.github/workflows/ami.yml
new file mode 100644
index 00000000..3424063a
--- /dev/null
+++ b/.github/workflows/ami.yml
@@ -0,0 +1,18 @@
+name: Build and Publish AMI
+on:
+  pull_request:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: DeterminateSystems/nix-installer-action@cd46bde16ab981b0a7b2dce0574509104543276e # v9
+      - uses: DeterminateSystems/magic-nix-cache-action@eeabdb06718ac63a7021c6132129679a8e22d0c7 # v3
+      - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
+        with:
+          aws-region: eu-north-1
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/upload-ami
+      - run: nix build github:${{ github.repository }}/${{ github.ref }}#legacyAmazonImage
+      - run: file="$(jq '.file' < ./result/nix-support/image-info.json)"
+      - run: nix run nixpkgs#coldsnap -- --no-progress  "$file"
+
+
diff --git a/flake.nix b/flake.nix
index 74f952a9..c9a46ec8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -19,12 +19,6 @@
         amazonImage = ./modules/amazon-image.nix;
 
         mock-imds = ./modules/mock-imds.nix;
-        version = { config, ... }: {
-          system.stateVersion = config.system.nixos.release;
-          # NOTE: This will cause an image to be built per commit.
-          # system.nixos.versionSuffix = lib.mkForce
-          #  ".${lib.substring 0 8 (nixpkgs.lastModifiedDate or nixpkgs.lastModified or "19700101")}.${nixpkgs.shortRev}.${lib.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}.${self.shortRev or "dirty"}";
-        };
       };
 
       lib.supportedSystems = [ "aarch64-linux" "x86_64-linux" "aarch64-darwin" ];
@@ -52,7 +46,6 @@
             modules = [
               self.nixosModules.ec2-instance-connect
               self.nixosModules.amazonImage
-              self.nixosModules.version
             ];
           }).config.system.build.amazonImage;
           legacyAmazonImage = (lib.nixosSystem {
@@ -61,10 +54,10 @@
             modules = [
               self.nixosModules.legacyAmazonImage
               {
-                boot.loader.grub.enable = false;
-                boot.loader.systemd-boot.enable = true;
+                ec2.efi = true;
+                # amazonImage.sizeMB = "auto";
+                amazonImage.format = "raw"; # coldsnap requires raw
               }
-              { ec2.efi = true; amazonImage.sizeMB = "auto"; }
               self.nixosModules.version
             ];
           }).config.system.build.amazonImage;

From 61be3f703719e4bc96965f17e14bf8f260092a83 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sun, 21 Apr 2024 15:22:31 +0200
Subject: [PATCH 12/19] add permissions

---
 .github/workflows/ami.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ami.yml b/.github/workflows/ami.yml
index 3424063a..ad577f98 100644
--- a/.github/workflows/ami.yml
+++ b/.github/workflows/ami.yml
@@ -4,6 +4,9 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - uses: DeterminateSystems/nix-installer-action@cd46bde16ab981b0a7b2dce0574509104543276e # v9
       - uses: DeterminateSystems/magic-nix-cache-action@eeabdb06718ac63a7021c6132129679a8e22d0c7 # v3

From 544e503abfc814f2871e1a13ead684a65e7bf3e6 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sun, 21 Apr 2024 15:23:54 +0200
Subject: [PATCH 13/19] add environment back

---
 .github/workflows/ami.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ami.yml b/.github/workflows/ami.yml
index ad577f98..635b930d 100644
--- a/.github/workflows/ami.yml
+++ b/.github/workflows/ami.yml
@@ -7,6 +7,7 @@ jobs:
     permissions:
       contents: read
       id-token: write
+    environment: images
     steps:
       - uses: DeterminateSystems/nix-installer-action@cd46bde16ab981b0a7b2dce0574509104543276e # v9
       - uses: DeterminateSystems/magic-nix-cache-action@eeabdb06718ac63a7021c6132129679a8e22d0c7 # v3

From 3fabe6f44a05363c9843b2f288ba674b189d9019 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sun, 21 Apr 2024 15:26:57 +0200
Subject: [PATCH 14/19] hmm version

---
 flake.nix | 2 --
 1 file changed, 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index c9a46ec8..1adf24d9 100644
--- a/flake.nix
+++ b/flake.nix
@@ -58,7 +58,6 @@
                 # amazonImage.sizeMB = "auto";
                 amazonImage.format = "raw"; # coldsnap requires raw
               }
-              self.nixosModules.version
             ];
           }).config.system.build.amazonImage;
 
@@ -82,7 +81,6 @@
             node.specialArgs.selfPackages = self.packages.${system};
             defaults = { name, ... }: {
               imports = [
-                self.nixosModules.version
                 self.nixosModules.amazonImage
                 self.nixosModules.mock-imds
               ];

From 45932e257f48abb97942548065cb7118ab3e356d Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sun, 21 Apr 2024 15:30:58 +0200
Subject: [PATCH 15/19] Size auto

---
 flake.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flake.nix b/flake.nix
index 1adf24d9..66d709b0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -56,6 +56,7 @@
               {
                 ec2.efi = true;
                 # amazonImage.sizeMB = "auto";
+                amazonImage.sizeMB = "auto";
                 amazonImage.format = "raw"; # coldsnap requires raw
               }
             ];

From 7e346c39a7ae8fa602a6223ce8219341fb1aea44 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sun, 21 Apr 2024 15:43:20 +0200
Subject: [PATCH 16/19] Fixes

---
 .github/workflows/ami.yml |  6 ++++--
 .github/workflows/ci.yml  | 24 ------------------------
 flake.lock                |  6 +++---
 modules/amazon-image.nix  |  4 ++--
 4 files changed, 9 insertions(+), 31 deletions(-)
 delete mode 100644 .github/workflows/ci.yml

diff --git a/.github/workflows/ami.yml b/.github/workflows/ami.yml
index 635b930d..b727995a 100644
--- a/.github/workflows/ami.yml
+++ b/.github/workflows/ami.yml
@@ -8,6 +8,8 @@ jobs:
       contents: read
       id-token: write
     environment: images
+    env:
+      FLAKE_REF: "github:${{ github.repository }}/${{ github.ref }}"
     steps:
       - uses: DeterminateSystems/nix-installer-action@cd46bde16ab981b0a7b2dce0574509104543276e # v9
       - uses: DeterminateSystems/magic-nix-cache-action@eeabdb06718ac63a7021c6132129679a8e22d0c7 # v3
@@ -15,8 +17,8 @@ jobs:
         with:
           aws-region: eu-north-1
           role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/upload-ami
-      - run: nix build github:${{ github.repository }}/${{ github.ref }}#legacyAmazonImage
+      - run: nix build ${{ env.FLAKE_REF }}#legacyAmazonImage
       - run: file="$(jq '.file' < ./result/nix-support/image-info.json)"
-      - run: nix run nixpkgs#coldsnap -- --no-progress  "$file"
+      - run: nix run --inputs-from ${{ env.FLAKE_REF }} nixpkgs#coldsnap -- upload --no-progress  "$file"
 
 
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
deleted file mode 100644
index 85e0d3dd..00000000
--- a/.github/workflows/ci.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-name: CI
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-permissions:
-  contents: read
-jobs:
-  check:
-    runs-on: ${{ matrix.runs-on.labels }}
-    strategy:
-      matrix:
-        runs-on:
-          - labels: [ubuntu-latest]
-            system: x86_64-linux
-    steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - uses: DeterminateSystems/nix-installer-action@cd46bde16ab981b0a7b2dce0574509104543276e # v9
-      - uses: DeterminateSystems/magic-nix-cache-action@eeabdb06718ac63a7021c6132129679a8e22d0c7 # v3
-      - run: nix build .#amazonImage -L --system ${{ matrix.runs-on.system }}
-      - run: nix flake check -L --system ${{ matrix.runs-on.system }}
diff --git a/flake.lock b/flake.lock
index db54019b..071c1957 100644
--- a/flake.lock
+++ b/flake.lock
@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1713532629,
-        "narHash": "sha256-8iwNoSDOCKFnDF7f8XReiztpESA0GyFieKhWAaG7jrw=",
+        "lastModified": 1713706266,
+        "narHash": "sha256-AvFLoQ5SvxYjvMMiV9k+TSE1gkS6DbGCWZUpNiw4tAA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "7f62671ffcb37436b3df7d6ae44dfdca9e5a069d",
+        "rev": "e8664fce9b0e956ebee7214276dd807eee585c22",
         "type": "github"
       },
       "original": {
diff --git a/modules/amazon-image.nix b/modules/amazon-image.nix
index 89cdc107..aec19b06 100644
--- a/modules/amazon-image.nix
+++ b/modules/amazon-image.nix
@@ -13,13 +13,13 @@ in
     pkgs.runCommand config.system.build.image.name { } ''
       mkdir -p $out
       mkdir -p $out/nix-support
-      ${pkgs.qemu-utils}/bin/qemu-img convert -f raw -O vpc ${config.system.build.image}/${config.image.repart.imageFile} $out/${config.image.repart.imageFileBasename}.vhd
       cat <<EOF > $out/nix-support/image-info.json
       {
         "boot_mode": "uefi",
+        "format": "raw",
         "label": "${config.system.nixos.label}",
         "system": "${pkgs.stdenv.hostPlatform.system}",
-        "file": "$out/${config.image.repart.imageFileBasename}.vhd"
+        "file": "${config.image.repart.imageFile}"
       }
       EOF
     '';

From 290904177f9f5dc131b605304e52f791fc26b65f Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sun, 21 Apr 2024 17:05:46 +0200
Subject: [PATCH 17/19] Fix

---
 .github/workflows/ami.yml | 5 +++--
 modules/amazon-image.nix  | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ami.yml b/.github/workflows/ami.yml
index b727995a..ce4c7841 100644
--- a/.github/workflows/ami.yml
+++ b/.github/workflows/ami.yml
@@ -18,7 +18,8 @@ jobs:
           aws-region: eu-north-1
           role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/upload-ami
       - run: nix build ${{ env.FLAKE_REF }}#legacyAmazonImage
-      - run: file="$(jq '.file' < ./result/nix-support/image-info.json)"
-      - run: nix run --inputs-from ${{ env.FLAKE_REF }} nixpkgs#coldsnap -- upload --no-progress  "$file"
+      - run: |
+          file="$(jq '.file' < ./result/nix-support/image-info.json)"
+          nix run --inputs-from ${{ env.FLAKE_REF }} nixpkgs#coldsnap -- upload --no-progress  "$file"
 
 
diff --git a/modules/amazon-image.nix b/modules/amazon-image.nix
index aec19b06..44e36e72 100644
--- a/modules/amazon-image.nix
+++ b/modules/amazon-image.nix
@@ -19,12 +19,13 @@ in
         "format": "raw",
         "label": "${config.system.nixos.label}",
         "system": "${pkgs.stdenv.hostPlatform.system}",
-        "file": "${config.image.repart.imageFile}"
+        "file": "${config.system.build.image}/${config.image.repart.imageFile}"
       }
       EOF
     '';
 
-  image.repart.name = "${config.system.nixos.distroId}-${config.system.nixos.label}-${pkgs.stdenv.hostPlatform.system}";
+  image.repart.name = config.system.nixos.distroId;
+  image.repart.version = config.system.nixos.version;
   image.repart.partitions = {
     "00-esp" = {
       contents = {

From 7822825555ceb3c9c02e342b434082a230dc48b0 Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Sun, 21 Apr 2024 17:16:06 +0200
Subject: [PATCH 18/19] jq raw

---
 .github/workflows/ami.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ami.yml b/.github/workflows/ami.yml
index ce4c7841..e3979204 100644
--- a/.github/workflows/ami.yml
+++ b/.github/workflows/ami.yml
@@ -8,8 +8,6 @@ jobs:
       contents: read
       id-token: write
     environment: images
-    env:
-      FLAKE_REF: "github:${{ github.repository }}/${{ github.ref }}"
     steps:
       - uses: DeterminateSystems/nix-installer-action@cd46bde16ab981b0a7b2dce0574509104543276e # v9
       - uses: DeterminateSystems/magic-nix-cache-action@eeabdb06718ac63a7021c6132129679a8e22d0c7 # v3
@@ -17,9 +15,10 @@ jobs:
         with:
           aws-region: eu-north-1
           role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/upload-ami
-      - run: nix build ${{ env.FLAKE_REF }}#legacyAmazonImage
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - run: |
-          file="$(jq '.file' < ./result/nix-support/image-info.json)"
-          nix run --inputs-from ${{ env.FLAKE_REF }} nixpkgs#coldsnap -- upload --no-progress  "$file"
+          nix build .#legacyAmazonImage
+          file="$(jq -r '.file' < ./result/nix-support/image-info.json)"
+          nix run --inputs-from . nixpkgs#coldsnap -- upload --no-progress  "$file"
 
 

From 69b94a29a1490bb2f6bc6943b43047c3ccca566d Mon Sep 17 00:00:00 2001
From: Arian van Putten <arian.vanputten@gmail.com>
Date: Tue, 30 Apr 2024 18:10:15 +0200
Subject: [PATCH 19/19] Update ami.yml

---
 .github/workflows/ami.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ami.yml b/.github/workflows/ami.yml
index e3979204..783c526d 100644
--- a/.github/workflows/ami.yml
+++ b/.github/workflows/ami.yml
@@ -16,9 +16,13 @@ jobs:
           aws-region: eu-north-1
           role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/upload-ami
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - run: |
-          nix build .#legacyAmazonImage
+      - name: build image
+        run: nix build .#legacyAmazonImage
+      - name: upload image with coldnsap
+        run: |
           file="$(jq -r '.file' < ./result/nix-support/image-info.json)"
+          echo "starting coldsnap"
           nix run --inputs-from . nixpkgs#coldsnap -- upload --no-progress  "$file"
+          echo "finished coldsnap"