From 4bc632b216fd484ec478f14565fa4e371dcc10c7 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 31 Mar 2025 15:51:20 +0100 Subject: [PATCH 01/10] Refactor images, add scripts for 3x --- images/docker/README.md | 5 ++ images/docker/{ubuntu => ubuntu-2x}/BUILD | 2 +- .../docker/{ubuntu => ubuntu-2x}/Dockerfile | 0 images/docker/ubuntu-2x/README.md | 24 +++++++++ .../{ubuntu => ubuntu-2x}/assemble-docker.sh | 3 ++ .../{ubuntu => ubuntu-2x}/deploy-docker.sh | 0 .../{ubuntu => ubuntu-2x}/deployment.bzl | 0 .../{ubuntu => ubuntu-2x}/version-docker.sh | 0 images/docker/ubuntu-3x/Dockerfile.amd64 | 6 +++ images/docker/ubuntu-3x/Dockerfile.arm64 | 6 +++ images/docker/ubuntu-3x/README.md | 45 +++++++++++++++++ images/docker/ubuntu-3x/VERSION | 1 + .../docker/ubuntu-3x/assemble-deploy-all.sh | 49 +++++++++++++++++++ images/docker/ubuntu-3x/assemble-docker.sh | 19 +++++++ images/docker/ubuntu-3x/deploy-docker.sh | 18 +++++++ 15 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 images/docker/README.md rename images/docker/{ubuntu => ubuntu-2x}/BUILD (92%) rename images/docker/{ubuntu => ubuntu-2x}/Dockerfile (100%) create mode 100644 images/docker/ubuntu-2x/README.md rename images/docker/{ubuntu => ubuntu-2x}/assemble-docker.sh (78%) rename images/docker/{ubuntu => ubuntu-2x}/deploy-docker.sh (100%) rename images/docker/{ubuntu => ubuntu-2x}/deployment.bzl (100%) rename images/docker/{ubuntu => ubuntu-2x}/version-docker.sh (100%) create mode 100644 images/docker/ubuntu-3x/Dockerfile.amd64 create mode 100644 images/docker/ubuntu-3x/Dockerfile.arm64 create mode 100644 images/docker/ubuntu-3x/README.md create mode 100644 images/docker/ubuntu-3x/VERSION create mode 100755 images/docker/ubuntu-3x/assemble-deploy-all.sh create mode 100755 images/docker/ubuntu-3x/assemble-docker.sh create mode 100755 images/docker/ubuntu-3x/deploy-docker.sh diff --git a/images/docker/README.md b/images/docker/README.md new file mode 100644 index 00000000..23290c1c --- /dev/null +++ b/images/docker/README.md @@ -0,0 +1,5 @@ +# TypeDB Docker Images + +TypeDB uses custom Docker images for different applications. + +Currently, there exist multiple Ubuntu images with the required dependencies of Java-based and Rust-based [TypeDB servers](https://github.com/typedb/typedb). diff --git a/images/docker/ubuntu/BUILD b/images/docker/ubuntu-2x/BUILD similarity index 92% rename from images/docker/ubuntu/BUILD rename to images/docker/ubuntu-2x/BUILD index 8894f72a..88b33016 100644 --- a/images/docker/ubuntu/BUILD +++ b/images/docker/ubuntu-2x/BUILD @@ -3,7 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. -load("//images/docker/ubuntu:deployment.bzl", deployment_docker = "deployment") +load("//images/docker/ubuntu-2x:deployment.bzl", deployment_docker = "deployment") load("//tool/checkstyle:rules.bzl", "checkstyle_test") sh_library( diff --git a/images/docker/ubuntu/Dockerfile b/images/docker/ubuntu-2x/Dockerfile similarity index 100% rename from images/docker/ubuntu/Dockerfile rename to images/docker/ubuntu-2x/Dockerfile diff --git a/images/docker/ubuntu-2x/README.md b/images/docker/ubuntu-2x/README.md new file mode 100644 index 00000000..1613f846 --- /dev/null +++ b/images/docker/ubuntu-2x/README.md @@ -0,0 +1,24 @@ +# TypeDB Ubuntu 2.x Images + +These images are prepared for TypeDB 2.x and contain all the required dependencies like JVM. + +Images are tagged based on the current Git commit SHA. + +## Usage + +### Setup + +Images are assembled and deployed from local machines with [Docker](https://www.docker.com/get-started/). + +To authenticate before starting the work, use: + +```shell +docker login +``` + +### Execution + +On Linux of a suitable architecture, use `bazel run //images/docker/ubuntu-2x:`. + +On Mac, run `assemble-docker.sh` and `deploy-docker.sh` manually, providing all the required arguments. See `BUILD` for +passed arguments examples. diff --git a/images/docker/ubuntu/assemble-docker.sh b/images/docker/ubuntu-2x/assemble-docker.sh similarity index 78% rename from images/docker/ubuntu/assemble-docker.sh rename to images/docker/ubuntu-2x/assemble-docker.sh index 1aef6563..c8a7c2af 100755 --- a/images/docker/ubuntu/assemble-docker.sh +++ b/images/docker/ubuntu-2x/assemble-docker.sh @@ -12,3 +12,6 @@ DOCKER_ORG=$3 DOCKER_REPO=$4 docker build -f $1 -t $DOCKER_ORG/$DOCKER_REPO:$DOCKER_VERSION . + + +docker buildx build --platform linux/amd64 --load -f Dockerfile.amd64 -t typedb/ubuntu:3.1.0b-amd64 . diff --git a/images/docker/ubuntu/deploy-docker.sh b/images/docker/ubuntu-2x/deploy-docker.sh similarity index 100% rename from images/docker/ubuntu/deploy-docker.sh rename to images/docker/ubuntu-2x/deploy-docker.sh diff --git a/images/docker/ubuntu/deployment.bzl b/images/docker/ubuntu-2x/deployment.bzl similarity index 100% rename from images/docker/ubuntu/deployment.bzl rename to images/docker/ubuntu-2x/deployment.bzl diff --git a/images/docker/ubuntu/version-docker.sh b/images/docker/ubuntu-2x/version-docker.sh similarity index 100% rename from images/docker/ubuntu/version-docker.sh rename to images/docker/ubuntu-2x/version-docker.sh diff --git a/images/docker/ubuntu-3x/Dockerfile.amd64 b/images/docker/ubuntu-3x/Dockerfile.amd64 new file mode 100644 index 00000000..d36e00ac --- /dev/null +++ b/images/docker/ubuntu-3x/Dockerfile.amd64 @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +FROM ubuntu@sha256:3d1556a8a18cf5307b121e0a98e93f1ddf1f3f8e092f1fddfd941254785b95d7 +RUN apt-get -y update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists diff --git a/images/docker/ubuntu-3x/Dockerfile.arm64 b/images/docker/ubuntu-3x/Dockerfile.arm64 new file mode 100644 index 00000000..71efaa23 --- /dev/null +++ b/images/docker/ubuntu-3x/Dockerfile.arm64 @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +FROM ubuntu@sha256:7c75ab2b0567edbb9d4834a2c51e462ebd709740d1f2c40bcd23c56e974fe2a8 +RUN apt-get -y update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists diff --git a/images/docker/ubuntu-3x/README.md b/images/docker/ubuntu-3x/README.md new file mode 100644 index 00000000..e20a22c5 --- /dev/null +++ b/images/docker/ubuntu-3x/README.md @@ -0,0 +1,45 @@ +# TypeDB Ubuntu 3.x Images + +These images are prepared for TypeDB 3.x and contain a default SSL configuration. + +Images are prepared for two base architectures: `arm64` and `amd64`. For easier maintenance, the images are tagged based on the incremental version specified manually in the `VERSION` file. + +## Usage + +### Setup + +Images are assembled and deployed from local machines with [Docker](https://www.docker.com/get-started/). + +To authenticate before starting the work, use: + +```shell +docker login +``` + +If platform-specific images use `buildx`, run: +```shell +docker buildx create --use --name multiarch-builder +docker buildx inspect --bootstrap + +# If the previous command fails, run this cleanup and try again: +# docker buildx rm multiarch-builder +``` + +### Execution + +It's possible to build images of any architecture on a local machine. + +Update `VERSION`. Follow one of the branches below based on your goals. + +#### Assembly and publish all architectures + +To update all images via a single line, run: +```shell +./assemble-deploy-all.sh +``` + +This script automatically assembles and publishes all platform-specific images with a multi-arch image (both `$VERSION` and `latest` tags). + +#### Separate steps + +Use `assemble-docker.sh` and `deploy-docker.sh` separately if needed. See comments and `assemble-deploy-all.sh` for usage examples. diff --git a/images/docker/ubuntu-3x/VERSION b/images/docker/ubuntu-3x/VERSION new file mode 100644 index 00000000..13d683cc --- /dev/null +++ b/images/docker/ubuntu-3x/VERSION @@ -0,0 +1 @@ +3.0.1 \ No newline at end of file diff --git a/images/docker/ubuntu-3x/assemble-deploy-all.sh b/images/docker/ubuntu-3x/assemble-deploy-all.sh new file mode 100755 index 00000000..9fbf6b2f --- /dev/null +++ b/images/docker/ubuntu-3x/assemble-deploy-all.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -ex + +# Usage: ./assemble-deploy-all.sh + +VERSION_FILE="./VERSION" + +if [[ ! -f "${VERSION_FILE}" ]]; then + echo "VERSION file not found" + exit 1 +fi + +VERSION=$(<"$VERSION_FILE") +VERSION=$(echo "$VERSION" | tr -d '[:space:]') + +if [[ -z "$VERSION" ]]; then + echo "VERSION file is invalid" + exit 1 +fi + +echo "Preparing images for version ${VERSION}" + +DOCKER_ORG=typedb +DOCKER_REPO=ubuntu +PLATFORMS=("amd64" "arm64") # Update `docker manifest` commands below if it's changed + +for ARCH in "${PLATFORMS[@]}"; do + ./assemble-docker.sh "Dockerfile.${ARCH}" "${ARCH}" "${VERSION}" "${DOCKER_ORG}" "${DOCKER_REPO}" + ./deploy-docker.sh "${ARCH}" "${VERSION}" "${DOCKER_ORG}" "${DOCKER_REPO}" +done + +TAG="${DOCKER_ORG}/${DOCKER_REPO}:${VERSION}" +echo "Creating manifests for ${TAG} and ${LATEST_TAG}" +docker manifest create "${TAG}" \ + --amend "${TAG}-amd64" \ + --amend "${TAG}-arm64" +docker manifest push "${TAG}" + +LATEST_TAG="${DOCKER_ORG}/${DOCKER_REPO}:latest" +docker manifest create "${LATEST_TAG}" \ + --amend "${TAG}-amd64" \ + --amend "${TAG}-arm64" +docker manifest push "${LATEST_TAG}" + +echo "Success" diff --git a/images/docker/ubuntu-3x/assemble-docker.sh b/images/docker/ubuntu-3x/assemble-docker.sh new file mode 100755 index 00000000..69ec86ec --- /dev/null +++ b/images/docker/ubuntu-3x/assemble-docker.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -ex + +# Usage: ./assemble-docker.sh Dockerfile.amd64 amd64 3.1.0 typedb ubuntu + +DOCKERFILE=$1 +PLATFORM=$2 +DOCKER_VERSION=$3 +DOCKER_ORG=$4 +DOCKER_REPO=$5 +TAG="${DOCKER_ORG}/${DOCKER_REPO}:${DOCKER_VERSION}-${PLATFORM}" + +echo "Assembling image for ${PLATFORM}: ${TAG}" +docker buildx build --platform "linux/${PLATFORM}" --load -f "${DOCKERFILE}" -t "${TAG}" . +echo "Successfully assembled ${TAG}" diff --git a/images/docker/ubuntu-3x/deploy-docker.sh b/images/docker/ubuntu-3x/deploy-docker.sh new file mode 100755 index 00000000..9b93e778 --- /dev/null +++ b/images/docker/ubuntu-3x/deploy-docker.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +set -ex + +# Usage: ./deploy-docker.sh amd64 3.1.0 typedb ubuntu + +PLATFORM=$1 +DOCKER_VERSION=$2 +DOCKER_ORG=$3 +DOCKER_REPO=$4 +TAG="${DOCKER_ORG}/${DOCKER_REPO}:${DOCKER_VERSION}-${PLATFORM}" + +echo "Deploying image for ${PLATFORM}: ${TAG}" +docker push "${TAG}" +echo "Successfully pushed" From 27c1cc2c6c0c38b3012d3ec691813cc979607459 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 31 Mar 2025 15:54:48 +0100 Subject: [PATCH 02/10] Refine latest --- images/docker/ubuntu-3x/README.md | 4 +++- images/docker/ubuntu-3x/assemble-deploy-all.sh | 8 +------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/images/docker/ubuntu-3x/README.md b/images/docker/ubuntu-3x/README.md index e20a22c5..8451a8ea 100644 --- a/images/docker/ubuntu-3x/README.md +++ b/images/docker/ubuntu-3x/README.md @@ -38,7 +38,9 @@ To update all images via a single line, run: ./assemble-deploy-all.sh ``` -This script automatically assembles and publishes all platform-specific images with a multi-arch image (both `$VERSION` and `latest` tags). +This script automatically assembles and publishes all platform-specific images with a multi-arch image. + +Note that there is no `latest` tag preparation as multiple Ubuntu image versions can be supported in parallel for different purposes. #### Separate steps diff --git a/images/docker/ubuntu-3x/assemble-deploy-all.sh b/images/docker/ubuntu-3x/assemble-deploy-all.sh index 9fbf6b2f..2f8e7223 100755 --- a/images/docker/ubuntu-3x/assemble-deploy-all.sh +++ b/images/docker/ubuntu-3x/assemble-deploy-all.sh @@ -34,16 +34,10 @@ for ARCH in "${PLATFORMS[@]}"; do done TAG="${DOCKER_ORG}/${DOCKER_REPO}:${VERSION}" -echo "Creating manifests for ${TAG} and ${LATEST_TAG}" +echo "Creating ${TAG} multi-arch manifest" docker manifest create "${TAG}" \ --amend "${TAG}-amd64" \ --amend "${TAG}-arm64" docker manifest push "${TAG}" -LATEST_TAG="${DOCKER_ORG}/${DOCKER_REPO}:latest" -docker manifest create "${LATEST_TAG}" \ - --amend "${TAG}-amd64" \ - --amend "${TAG}-arm64" -docker manifest push "${LATEST_TAG}" - echo "Success" From e350a1218316473bebbeaec4030d47980fa7ca52 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 31 Mar 2025 17:02:40 +0100 Subject: [PATCH 03/10] Update readmes and fix checkstyle --- images/docker/BUILD | 12 ++++++++++++ images/docker/README.md | 10 ++++++++-- images/docker/ubuntu-2x/README.md | 4 ++-- images/docker/ubuntu-3x/BUILD | 12 ++++++++++++ images/docker/ubuntu-3x/README.md | 21 +++++++++++++++------ 5 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 images/docker/BUILD create mode 100644 images/docker/ubuntu-3x/BUILD diff --git a/images/docker/BUILD b/images/docker/BUILD new file mode 100644 index 00000000..53133e83 --- /dev/null +++ b/images/docker/BUILD @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +load("//tool/checkstyle:rules.bzl", "checkstyle_test") + +checkstyle_test( + name = "checkstyle", + include = glob(["*"]), + license_type = "mpl-header", + size = "small", +) diff --git a/images/docker/README.md b/images/docker/README.md index 23290c1c..4622c866 100644 --- a/images/docker/README.md +++ b/images/docker/README.md @@ -1,5 +1,11 @@ # TypeDB Docker Images -TypeDB uses custom Docker images for different applications. +TypeDB uses custom Docker images for different applications. -Currently, there exist multiple Ubuntu images with the required dependencies of Java-based and Rust-based [TypeDB servers](https://github.com/typedb/typedb). +These images can be created using image layers through Bazel rules (e.g., `docker_container_run_and_commit`), but we’ve +encountered OS-specific issues — presumably due to missing hidden parameters in the generated images. Additionally, we +don’t expect the base images to change frequently. Therefore, it’s more reliable to prepare them once and store them in +a separate repository. + +Currently, there are multiple Ubuntu-based images with the required dependencies for both Java-based and +Rust-based [TypeDB servers](https://github.com/typedb/typedb). diff --git a/images/docker/ubuntu-2x/README.md b/images/docker/ubuntu-2x/README.md index 1613f846..8819306c 100644 --- a/images/docker/ubuntu-2x/README.md +++ b/images/docker/ubuntu-2x/README.md @@ -18,7 +18,7 @@ docker login ### Execution -On Linux of a suitable architecture, use `bazel run //images/docker/ubuntu-2x:`. +Using a suitable OS and architecture, use `bazel run //images/docker/ubuntu-2x:`. -On Mac, run `assemble-docker.sh` and `deploy-docker.sh` manually, providing all the required arguments. See `BUILD` for +If Bazel rules do not work on your machine, run `assemble-docker.sh` and `deploy-docker.sh` manually, providing all the required arguments. See `BUILD` for passed arguments examples. diff --git a/images/docker/ubuntu-3x/BUILD b/images/docker/ubuntu-3x/BUILD new file mode 100644 index 00000000..53133e83 --- /dev/null +++ b/images/docker/ubuntu-3x/BUILD @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +load("//tool/checkstyle:rules.bzl", "checkstyle_test") + +checkstyle_test( + name = "checkstyle", + include = glob(["*"]), + license_type = "mpl-header", + size = "small", +) diff --git a/images/docker/ubuntu-3x/README.md b/images/docker/ubuntu-3x/README.md index 8451a8ea..a447a62c 100644 --- a/images/docker/ubuntu-3x/README.md +++ b/images/docker/ubuntu-3x/README.md @@ -2,10 +2,16 @@ These images are prepared for TypeDB 3.x and contain a default SSL configuration. -Images are prepared for two base architectures: `arm64` and `amd64`. For easier maintenance, the images are tagged based on the incremental version specified manually in the `VERSION` file. +Images are prepared for two base architectures: `arm64` and `amd64`. For easier maintenance, the images are tagged based +on the incremental version specified manually in the `VERSION` file. ## Usage +For simplicity and due to the current usage of these scripts, there is no Bazel integration provided. + +It's possible to build images of any architecture on a local machine (e.g., it's successfully tested on a arm-based +Mac). + ### Setup Images are assembled and deployed from local machines with [Docker](https://www.docker.com/get-started/). @@ -17,6 +23,7 @@ docker login ``` If platform-specific images use `buildx`, run: + ```shell docker buildx create --use --name multiarch-builder docker buildx inspect --bootstrap @@ -27,21 +34,23 @@ docker buildx inspect --bootstrap ### Execution -It's possible to build images of any architecture on a local machine. - -Update `VERSION`. Follow one of the branches below based on your goals. +Update `VERSION`. For better maintenance, set the TypeDB server's version expected to use this image first. +Follow one of the branches below based on your goals. #### Assembly and publish all architectures To update all images via a single line, run: + ```shell ./assemble-deploy-all.sh ``` This script automatically assembles and publishes all platform-specific images with a multi-arch image. -Note that there is no `latest` tag preparation as multiple Ubuntu image versions can be supported in parallel for different purposes. +Note that there is no `latest` tag preparation as multiple Ubuntu image versions can be supported in parallel for +different purposes. #### Separate steps -Use `assemble-docker.sh` and `deploy-docker.sh` separately if needed. See comments and `assemble-deploy-all.sh` for usage examples. +Use `assemble-docker.sh` and `deploy-docker.sh` separately if needed. See comments and `assemble-deploy-all.sh` for +usage examples. From 63676126e5d2b2b04f67e7d038e0b184e25bc31f Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 31 Mar 2025 17:11:41 +0100 Subject: [PATCH 04/10] Exclude md files from checkstyle --- images/docker/BUILD | 1 + images/docker/ubuntu-2x/BUILD | 1 + images/docker/ubuntu-3x/BUILD | 1 + images/docker/ubuntu-3x/deploy-docker.sh | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/images/docker/BUILD b/images/docker/BUILD index 53133e83..3e8118fa 100644 --- a/images/docker/BUILD +++ b/images/docker/BUILD @@ -7,6 +7,7 @@ load("//tool/checkstyle:rules.bzl", "checkstyle_test") checkstyle_test( name = "checkstyle", include = glob(["*"]), + exclude = glob(["*.md"]), license_type = "mpl-header", size = "small", ) diff --git a/images/docker/ubuntu-2x/BUILD b/images/docker/ubuntu-2x/BUILD index 88b33016..32a381b3 100644 --- a/images/docker/ubuntu-2x/BUILD +++ b/images/docker/ubuntu-2x/BUILD @@ -29,6 +29,7 @@ sh_binary( checkstyle_test( name = "checkstyle", include = glob(["*"]), + exclude = glob(["*.md"]), license_type = "mpl-header", size = "small", ) diff --git a/images/docker/ubuntu-3x/BUILD b/images/docker/ubuntu-3x/BUILD index 53133e83..3e8118fa 100644 --- a/images/docker/ubuntu-3x/BUILD +++ b/images/docker/ubuntu-3x/BUILD @@ -7,6 +7,7 @@ load("//tool/checkstyle:rules.bzl", "checkstyle_test") checkstyle_test( name = "checkstyle", include = glob(["*"]), + exclude = glob(["*.md"]), license_type = "mpl-header", size = "small", ) diff --git a/images/docker/ubuntu-3x/deploy-docker.sh b/images/docker/ubuntu-3x/deploy-docker.sh index 9b93e778..8668c124 100755 --- a/images/docker/ubuntu-3x/deploy-docker.sh +++ b/images/docker/ubuntu-3x/deploy-docker.sh @@ -15,4 +15,4 @@ TAG="${DOCKER_ORG}/${DOCKER_REPO}:${DOCKER_VERSION}-${PLATFORM}" echo "Deploying image for ${PLATFORM}: ${TAG}" docker push "${TAG}" -echo "Successfully pushed" +echo "Successfully pushed ${TAG}" From 9830f763a4e503de303d7d79fa7229a47aa1089c Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 31 Mar 2025 17:23:22 +0100 Subject: [PATCH 05/10] Exclude VERSION from checkstyle --- images/docker/ubuntu-3x/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/docker/ubuntu-3x/BUILD b/images/docker/ubuntu-3x/BUILD index 3e8118fa..e4304dd8 100644 --- a/images/docker/ubuntu-3x/BUILD +++ b/images/docker/ubuntu-3x/BUILD @@ -7,7 +7,7 @@ load("//tool/checkstyle:rules.bzl", "checkstyle_test") checkstyle_test( name = "checkstyle", include = glob(["*"]), - exclude = glob(["*.md"]), + exclude = glob(["*.md", "VERSION"]), license_type = "mpl-header", size = "small", ) From 145aef4b96dd8296b15b793286c9e7e1580aca72 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Mon, 31 Mar 2025 17:34:48 +0100 Subject: [PATCH 06/10] Remove excessive line --- images/docker/ubuntu-2x/assemble-docker.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/images/docker/ubuntu-2x/assemble-docker.sh b/images/docker/ubuntu-2x/assemble-docker.sh index c8a7c2af..1aef6563 100755 --- a/images/docker/ubuntu-2x/assemble-docker.sh +++ b/images/docker/ubuntu-2x/assemble-docker.sh @@ -12,6 +12,3 @@ DOCKER_ORG=$3 DOCKER_REPO=$4 docker build -f $1 -t $DOCKER_ORG/$DOCKER_REPO:$DOCKER_VERSION . - - -docker buildx build --platform linux/amd64 --load -f Dockerfile.amd64 -t typedb/ubuntu:3.1.0b-amd64 . From fce718340492ed7d088bc991bde766394871c247 Mon Sep 17 00:00:00 2001 From: Krishnan Govindraj Date: Mon, 28 Apr 2025 22:47:32 +0530 Subject: [PATCH 07/10] Rework back to bazel targets invoking the scripts --- images/docker/BUILD | 1 + .../docker/{ubuntu-3x => }/assemble-docker.sh | 13 +++-- .../docker/{ubuntu-3x => }/deploy-docker.sh | 11 ++-- images/docker/ubuntu-2x/BUILD | 50 +++++++++++++----- .../{Dockerfile => Dockerfile.amd64} | 2 +- .../{deploy-docker.sh => Dockerfile.arm64} | 11 +--- images/docker/ubuntu-2x/VERSION | 1 + images/docker/ubuntu-2x/assemble-docker.sh | 14 ----- images/docker/ubuntu-2x/version-docker.sh | 7 --- images/docker/ubuntu-3x/BUILD | 51 ++++++++++++++++++- images/docker/ubuntu-3x/VERSION | 2 +- .../docker/ubuntu-3x/assemble-deploy-all.sh | 7 ++- 12 files changed, 110 insertions(+), 60 deletions(-) rename images/docker/{ubuntu-3x => }/assemble-docker.sh (55%) rename images/docker/{ubuntu-3x => }/deploy-docker.sh (66%) rename images/docker/ubuntu-2x/{Dockerfile => Dockerfile.amd64} (78%) rename images/docker/ubuntu-2x/{deploy-docker.sh => Dockerfile.arm64} (52%) mode change 100755 => 100644 create mode 100644 images/docker/ubuntu-2x/VERSION delete mode 100755 images/docker/ubuntu-2x/assemble-docker.sh delete mode 100755 images/docker/ubuntu-2x/version-docker.sh diff --git a/images/docker/BUILD b/images/docker/BUILD index 3e8118fa..8da73419 100644 --- a/images/docker/BUILD +++ b/images/docker/BUILD @@ -3,6 +3,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. load("//tool/checkstyle:rules.bzl", "checkstyle_test") +exports_files(["assemble-docker.sh", "deploy-docker.sh"]) checkstyle_test( name = "checkstyle", diff --git a/images/docker/ubuntu-3x/assemble-docker.sh b/images/docker/assemble-docker.sh similarity index 55% rename from images/docker/ubuntu-3x/assemble-docker.sh rename to images/docker/assemble-docker.sh index 69ec86ec..cdfdba1f 100755 --- a/images/docker/ubuntu-3x/assemble-docker.sh +++ b/images/docker/assemble-docker.sh @@ -7,13 +7,12 @@ set -ex # Usage: ./assemble-docker.sh Dockerfile.amd64 amd64 3.1.0 typedb ubuntu -DOCKERFILE=$1 +DOCKERFILE=$(realpath $1) PLATFORM=$2 -DOCKER_VERSION=$3 +TAG=$(cat $3) DOCKER_ORG=$4 DOCKER_REPO=$5 -TAG="${DOCKER_ORG}/${DOCKER_REPO}:${DOCKER_VERSION}-${PLATFORM}" - -echo "Assembling image for ${PLATFORM}: ${TAG}" -docker buildx build --platform "linux/${PLATFORM}" --load -f "${DOCKERFILE}" -t "${TAG}" . -echo "Successfully assembled ${TAG}" +FULL_TAG="${DOCKER_ORG}/${DOCKER_REPO}:${TAG}" +echo "Assembling image for ${PLATFORM}: ${FULL_TAG}" +docker buildx build --platform "linux/${PLATFORM}" --load -f $DOCKERFILE -t "${FULL_TAG}" . +echo "Successfully assembled ${FULL_TAG}" diff --git a/images/docker/ubuntu-3x/deploy-docker.sh b/images/docker/deploy-docker.sh similarity index 66% rename from images/docker/ubuntu-3x/deploy-docker.sh rename to images/docker/deploy-docker.sh index 8668c124..b62bd65e 100755 --- a/images/docker/ubuntu-3x/deploy-docker.sh +++ b/images/docker/deploy-docker.sh @@ -7,12 +7,11 @@ set -ex # Usage: ./deploy-docker.sh amd64 3.1.0 typedb ubuntu -PLATFORM=$1 -DOCKER_VERSION=$2 -DOCKER_ORG=$3 -DOCKER_REPO=$4 -TAG="${DOCKER_ORG}/${DOCKER_REPO}:${DOCKER_VERSION}-${PLATFORM}" +TAG=$(cat $1) +DOCKER_ORG=$2 +DOCKER_REPO=$3 +TAG="${DOCKER_ORG}/${DOCKER_REPO}:${TAG}" -echo "Deploying image for ${PLATFORM}: ${TAG}" +echo "Deploying image: ${TAG}" docker push "${TAG}" echo "Successfully pushed ${TAG}" diff --git a/images/docker/ubuntu-2x/BUILD b/images/docker/ubuntu-2x/BUILD index 32a381b3..2d0da9d0 100644 --- a/images/docker/ubuntu-2x/BUILD +++ b/images/docker/ubuntu-2x/BUILD @@ -6,24 +6,50 @@ load("//images/docker/ubuntu-2x:deployment.bzl", deployment_docker = "deployment") load("//tool/checkstyle:rules.bzl", "checkstyle_test") -sh_library( - name = "version-docker", - srcs = ["version-docker.sh"], +# Gen-rules to generate - as tag +genrule( + name = "tag-amd64", + srcs = [":VERSION"], + outs = ["tag-amd64.txt"], + cmd = "cat $(location :VERSION) | tr -d '\n' > $(location tag-amd64.txt) && echo \"-amd64\" >> $(location tag-amd64.txt)", +) +genrule( + name = "tag-arm64", + srcs = [":VERSION"], + outs = ["tag-arm64.txt"], + cmd = "cat $(location :VERSION) | tr -d '\n' > $(location tag-arm64.txt) && echo \"-arm64\" >> $(location tag-arm64.txt)", +) + +sh_binary( + name = "assemble-docker-amd64", + srcs = ["//images/docker:assemble-docker.sh"], + data = ["Dockerfile.amd64", "VERSION", "tag-amd64"], + deps = [":tag-amd64"], + args = ["$(location Dockerfile.amd64)", "amd64", "$(location :tag-amd64)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] +) + +sh_binary( + name = "assemble-docker-arm64", + srcs = ["//images/docker:assemble-docker.sh"], + data = ["Dockerfile.arm64", "VERSION", "tag-arm64"], + deps = [":tag-arm64"], + args = ["$(location Dockerfile.arm64)", "arm64", "$(location :tag-arm64)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] ) sh_binary( - name = "assemble-docker", - srcs = ["assemble-docker.sh"], - data = ["Dockerfile"], - deps = [":version-docker"], - args = ["$(location Dockerfile)", "$(location :version-docker)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] + name = "deploy-docker-amd64", + deps = [], + srcs = ["//images/docker:deploy-docker.sh"], + data = ["VERSION", "tag-amd64"], + args = ["$(location :tag-amd64)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] ) sh_binary( - name = "deploy-docker", - deps = [":version-docker"], - srcs = ["deploy-docker.sh"], - args = ["$(location :version-docker)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] + name = "deploy-docker-arm64", + deps = [], + srcs = ["//images/docker:deploy-docker.sh"], + data = ["VERSION", "tag-arm64"], + args = ["$(location :tag-arm64)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] ) checkstyle_test( diff --git a/images/docker/ubuntu-2x/Dockerfile b/images/docker/ubuntu-2x/Dockerfile.amd64 similarity index 78% rename from images/docker/ubuntu-2x/Dockerfile rename to images/docker/ubuntu-2x/Dockerfile.amd64 index e0b9f395..04d1b457 100644 --- a/images/docker/ubuntu-2x/Dockerfile +++ b/images/docker/ubuntu-2x/Dockerfile.amd64 @@ -3,5 +3,5 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. -FROM ubuntu:focal-20210723 +FROM ubuntu@sha256:1e48201ccc2ab83afc435394b3bf70af0fa0055215c1e26a5da9b50a1ae367c9 RUN apt-get -y update && apt-get install -y openjdk-11-jre-headless && rm -rf /var/lib/apt/lists diff --git a/images/docker/ubuntu-2x/deploy-docker.sh b/images/docker/ubuntu-2x/Dockerfile.arm64 old mode 100755 new mode 100644 similarity index 52% rename from images/docker/ubuntu-2x/deploy-docker.sh rename to images/docker/ubuntu-2x/Dockerfile.arm64 index 91645b90..f0a093a7 --- a/images/docker/ubuntu-2x/deploy-docker.sh +++ b/images/docker/ubuntu-2x/Dockerfile.arm64 @@ -1,14 +1,7 @@ -#!/usr/bin/env bash # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -set -ex - -DOCKER_VERSION_LIB=$1 -DOCKER_VERSION=`$DOCKER_VERSION_LIB` -DOCKER_ORG=$2 -DOCKER_REPO=$3 - -docker push $DOCKER_ORG/$DOCKER_REPO:$DOCKER_VERSION +FROM ubuntu@sha256:c0dd38485ed8b6e149d2fc935d1c61a3b750cda3b3eace0ad6df4abe33fa2b90 +RUN apt-get -y update && apt-get install -y openjdk-11-jre-headless && rm -rf /var/lib/apt/lists diff --git a/images/docker/ubuntu-2x/VERSION b/images/docker/ubuntu-2x/VERSION new file mode 100644 index 00000000..f0135682 --- /dev/null +++ b/images/docker/ubuntu-2x/VERSION @@ -0,0 +1 @@ +2.29.0 diff --git a/images/docker/ubuntu-2x/assemble-docker.sh b/images/docker/ubuntu-2x/assemble-docker.sh deleted file mode 100755 index 1aef6563..00000000 --- a/images/docker/ubuntu-2x/assemble-docker.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - - -set -ex - -DOCKER_VERSION_LIB=$2 -DOCKER_VERSION=`$DOCKER_VERSION_LIB` -DOCKER_ORG=$3 -DOCKER_REPO=$4 - -docker build -f $1 -t $DOCKER_ORG/$DOCKER_REPO:$DOCKER_VERSION . diff --git a/images/docker/ubuntu-2x/version-docker.sh b/images/docker/ubuntu-2x/version-docker.sh deleted file mode 100755 index e5df4498..00000000 --- a/images/docker/ubuntu-2x/version-docker.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - - -echo "$(git -C $BUILD_WORKSPACE_DIRECTORY rev-parse HEAD)" diff --git a/images/docker/ubuntu-3x/BUILD b/images/docker/ubuntu-3x/BUILD index e4304dd8..f7e27707 100644 --- a/images/docker/ubuntu-3x/BUILD +++ b/images/docker/ubuntu-3x/BUILD @@ -1,9 +1,58 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. - load("//tool/checkstyle:rules.bzl", "checkstyle_test") +# Same repository as 2.x images. +load("//images/docker/ubuntu-2x:deployment.bzl", deployment_docker = "deployment") + +# Gen-rules to generate - as tag +genrule( + name = "tag-amd64", + srcs = [":VERSION"], + outs = ["tag-amd64.txt"], + cmd = "cat $(location :VERSION) | tr -d '\n' > $(location tag-amd64.txt) && echo \"-amd64\" >> $(location tag-amd64.txt)", +) +genrule( + name = "tag-arm64", + srcs = [":VERSION"], + outs = ["tag-arm64.txt"], + cmd = "cat $(location :VERSION) | tr -d '\n' > $(location tag-arm64.txt) && echo \"-arm64\" >> $(location tag-arm64.txt)", +) + + +sh_binary( + name = "assemble-docker-amd64", + srcs = ["//images/docker:assemble-docker.sh"], + data = ["Dockerfile.amd64", "VERSION", "tag-amd64"], + deps = [], + args = ["$(location Dockerfile.amd64)", "amd64", "$(location :tag-amd64)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] +) + +sh_binary( + name = "assemble-docker-arm64", + srcs = ["//images/docker:assemble-docker.sh"], + data = ["Dockerfile.arm64", "VERSION", "tag-arm64"], + deps = [], + args = ["$(location Dockerfile.arm64)", "arm64", "$(location :tag-arm64)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] +) + +sh_binary( + name = "deploy-docker-amd64", + deps = [], + srcs = ["//images/docker:deploy-docker.sh"], + data = ["VERSION", "tag-amd64"], + args = ["$(location :tag-amd64)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] +) + +sh_binary( + name = "deploy-docker-arm64", + deps = [], + srcs = ["//images/docker:deploy-docker.sh"], + data = ["VERSION", "tag-arm64"], + args = ["$(location :tag-arm64)", deployment_docker["docker.organisation"], deployment_docker["docker.repository"]] +) + checkstyle_test( name = "checkstyle", include = glob(["*"]), diff --git a/images/docker/ubuntu-3x/VERSION b/images/docker/ubuntu-3x/VERSION index 13d683cc..944880fa 100644 --- a/images/docker/ubuntu-3x/VERSION +++ b/images/docker/ubuntu-3x/VERSION @@ -1 +1 @@ -3.0.1 \ No newline at end of file +3.2.0 diff --git a/images/docker/ubuntu-3x/assemble-deploy-all.sh b/images/docker/ubuntu-3x/assemble-deploy-all.sh index 2f8e7223..4120151d 100755 --- a/images/docker/ubuntu-3x/assemble-deploy-all.sh +++ b/images/docker/ubuntu-3x/assemble-deploy-all.sh @@ -29,8 +29,11 @@ DOCKER_REPO=ubuntu PLATFORMS=("amd64" "arm64") # Update `docker manifest` commands below if it's changed for ARCH in "${PLATFORMS[@]}"; do - ./assemble-docker.sh "Dockerfile.${ARCH}" "${ARCH}" "${VERSION}" "${DOCKER_ORG}" "${DOCKER_REPO}" - ./deploy-docker.sh "${ARCH}" "${VERSION}" "${DOCKER_ORG}" "${DOCKER_REPO}" + # Should we just invoke the bazel targets here instead? + ASSEMBLE_TARGET="//images/docker/ubuntu-3x:assemble-docker-${ARCH}" + DEPLOY_TARGET="//images/docker/ubuntu-3x:deploy-docker-${ARCH}" + bazel run $ASSEMBLE_TARGET + bazel run $DEPLOY_TARGET done TAG="${DOCKER_ORG}/${DOCKER_REPO}:${VERSION}" From 374d8015f203c9079ccc2627c949d492eac2ef28 Mon Sep 17 00:00:00 2001 From: Krishnan Govindraj Date: Tue, 29 Apr 2025 22:13:20 +0530 Subject: [PATCH 08/10] Remove comment that I did --- images/docker/ubuntu-3x/assemble-deploy-all.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/images/docker/ubuntu-3x/assemble-deploy-all.sh b/images/docker/ubuntu-3x/assemble-deploy-all.sh index 4120151d..d92d5f15 100755 --- a/images/docker/ubuntu-3x/assemble-deploy-all.sh +++ b/images/docker/ubuntu-3x/assemble-deploy-all.sh @@ -29,7 +29,6 @@ DOCKER_REPO=ubuntu PLATFORMS=("amd64" "arm64") # Update `docker manifest` commands below if it's changed for ARCH in "${PLATFORMS[@]}"; do - # Should we just invoke the bazel targets here instead? ASSEMBLE_TARGET="//images/docker/ubuntu-3x:assemble-docker-${ARCH}" DEPLOY_TARGET="//images/docker/ubuntu-3x:deploy-docker-${ARCH}" bazel run $ASSEMBLE_TARGET From 478b7203b549d375b6cea0def5b96bed6867d6d2 Mon Sep 17 00:00:00 2001 From: Krishnan Govindraj Date: Wed, 30 Apr 2025 17:33:19 +0530 Subject: [PATCH 09/10] Fix checkstyle --- images/docker/ubuntu-2x/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/docker/ubuntu-2x/BUILD b/images/docker/ubuntu-2x/BUILD index 2d0da9d0..9fcc9117 100644 --- a/images/docker/ubuntu-2x/BUILD +++ b/images/docker/ubuntu-2x/BUILD @@ -55,7 +55,7 @@ sh_binary( checkstyle_test( name = "checkstyle", include = glob(["*"]), - exclude = glob(["*.md"]), + exclude = glob(["*.md", "VERSION"]), license_type = "mpl-header", size = "small", ) From 8a016b9e0348bcd05c4d71d71eb20437bf03e715 Mon Sep 17 00:00:00 2001 From: Georgii Novoselov Date: Wed, 30 Apr 2025 17:02:45 +0100 Subject: [PATCH 10/10] Update readmes and style --- images/docker/ubuntu-2x/BUILD | 1 + images/docker/ubuntu-2x/README.md | 2 +- images/docker/ubuntu-3x/BUILD | 2 +- images/docker/ubuntu-3x/README.md | 35 ++++--------------------------- 4 files changed, 7 insertions(+), 33 deletions(-) diff --git a/images/docker/ubuntu-2x/BUILD b/images/docker/ubuntu-2x/BUILD index 9fcc9117..3d28acff 100644 --- a/images/docker/ubuntu-2x/BUILD +++ b/images/docker/ubuntu-2x/BUILD @@ -13,6 +13,7 @@ genrule( outs = ["tag-amd64.txt"], cmd = "cat $(location :VERSION) | tr -d '\n' > $(location tag-amd64.txt) && echo \"-amd64\" >> $(location tag-amd64.txt)", ) + genrule( name = "tag-arm64", srcs = [":VERSION"], diff --git a/images/docker/ubuntu-2x/README.md b/images/docker/ubuntu-2x/README.md index 8819306c..60a3c891 100644 --- a/images/docker/ubuntu-2x/README.md +++ b/images/docker/ubuntu-2x/README.md @@ -2,7 +2,7 @@ These images are prepared for TypeDB 2.x and contain all the required dependencies like JVM. -Images are tagged based on the current Git commit SHA. +Images are prepared for two base architectures: `arm64` and `amd64`. Images are tagged based on the current `VERSION` and Git commit SHA. ## Usage diff --git a/images/docker/ubuntu-3x/BUILD b/images/docker/ubuntu-3x/BUILD index f7e27707..68681a1f 100644 --- a/images/docker/ubuntu-3x/BUILD +++ b/images/docker/ubuntu-3x/BUILD @@ -13,6 +13,7 @@ genrule( outs = ["tag-amd64.txt"], cmd = "cat $(location :VERSION) | tr -d '\n' > $(location tag-amd64.txt) && echo \"-amd64\" >> $(location tag-amd64.txt)", ) + genrule( name = "tag-arm64", srcs = [":VERSION"], @@ -20,7 +21,6 @@ genrule( cmd = "cat $(location :VERSION) | tr -d '\n' > $(location tag-arm64.txt) && echo \"-arm64\" >> $(location tag-arm64.txt)", ) - sh_binary( name = "assemble-docker-amd64", srcs = ["//images/docker:assemble-docker.sh"], diff --git a/images/docker/ubuntu-3x/README.md b/images/docker/ubuntu-3x/README.md index a447a62c..d9d6c21e 100644 --- a/images/docker/ubuntu-3x/README.md +++ b/images/docker/ubuntu-3x/README.md @@ -2,16 +2,10 @@ These images are prepared for TypeDB 3.x and contain a default SSL configuration. -Images are prepared for two base architectures: `arm64` and `amd64`. For easier maintenance, the images are tagged based -on the incremental version specified manually in the `VERSION` file. +Images are prepared for two base architectures: `arm64` and `amd64`. Images are tagged based on the current `VERSION` and Git commit SHA. ## Usage -For simplicity and due to the current usage of these scripts, there is no Bazel integration provided. - -It's possible to build images of any architecture on a local machine (e.g., it's successfully tested on a arm-based -Mac). - ### Setup Images are assembled and deployed from local machines with [Docker](https://www.docker.com/get-started/). @@ -22,33 +16,12 @@ To authenticate before starting the work, use: docker login ``` -If platform-specific images use `buildx`, run: - -```shell -docker buildx create --use --name multiarch-builder -docker buildx inspect --bootstrap - -# If the previous command fails, run this cleanup and try again: -# docker buildx rm multiarch-builder -``` - ### Execution -Update `VERSION`. For better maintenance, set the TypeDB server's version expected to use this image first. -Follow one of the branches below based on your goals. - -#### Assembly and publish all architectures - -To update all images via a single line, run: - -```shell -./assemble-deploy-all.sh -``` - -This script automatically assembles and publishes all platform-specific images with a multi-arch image. +Using a suitable OS and architecture, use `bazel run //images/docker/ubuntu-3x:`. -Note that there is no `latest` tag preparation as multiple Ubuntu image versions can be supported in parallel for -different purposes. +If Bazel rules do not work on your machine, run `assemble-docker.sh` and `deploy-docker.sh` manually, providing all the required arguments, or `assemble-deploy-all.sh` for a fast update for all versions. See `BUILD` for +passed arguments examples. #### Separate steps