automation and filedescriptorset usage

.gitignore

@@ -1,2 +1,3 @@
 /target
 Cargo.lock
+protos.fds

Cargo.toml

@@ -1,13 +1,6 @@
-[package]
-name = "k8s-pb"
-version = "0.1.0"
-license = "Apache-2.0"
-edition = "2018"
-
-[dependencies]
-bytes = "1.0.1"
-prost = "0.8.0"
-prost-types = "0.8"
-
-[build-dependencies]
-prost-build = "0.8.0"
+[workspace]
+default-members = ["k8s-pb"]
+members = [
+  "k8s-pb-codegen",
+  "k8s-pb"
+]

README.md

@@ -1,80 +1,41 @@
 Experimenting with Kubernetes protobufs.
 
-## Protobufs
+## Build Dependencies
 
-### Download
+- [fd](https://github.com/sharkdp/fd)
+- [jq](https://stedolan.github.io/jq/)
+- [just](https://github.com/casey/just)
+- [sd](https://github.com/chmln/sd)
 
-Get protos by extracting them from Kubernetes releases:
+## Protobufs
+We get protos by extracting them from pinned Kubernetes releases:
 
 - https://github.com/kubernetes/api/releases
 - https://github.com/kubernetes/apimachinery/releases
 - https://github.com/kubernetes/apiextensions-apiserver/releases
 - https://github.com/kubernetes/kube-aggregator/releases
 - https://github.com/kubernetes/metrics/releases
 
-```bash
-# In `protos/`
-VERSION=1.22.0
-for x in api apimachinery apiextensions-apiserver kube-aggregator metrics; do
-  mkdir ./$x;
-  curl -sSL https://github.com/kubernetes/$x/archive/refs/tags/kubernetes-$VERSION.tar.gz | tar xzf - -C ./$x/ --strip-components=1;
-  fd -e proto -x sh -c "mkdir -p k8s.io/'{//}'; mv '{}' k8s.io/'{}'" ';' . ./$x;
-  rm -rf ./$x;
-done
-```
+We then do minor transforms on top of that to prepare for building.
+Results of this step is committed already. But to run, invoke `just protos`
 
-### Patch
+## Openapi
+To complement the protos with generic information, we also download the swagger schema, patch it, and transform it as described below.
 
-Removing `k8s.io.`:
+Results of this step is committed already. But to run, invoke `just swagger`.
 
-```bash
-fd -e proto -x sd 'k8s\.io\.(.+);' '$1;' {}
-fd -e proto -x sd 'import "k8s\.io/(.+)";' 'import "$1";' {}
-mv protos/k8s.io/* protos/
-rmdir protos/k8s.io/
-```
 
-### Generate
+## Building
+To build the [out](./out) directory from [build.rs](./build.rs) we will use the outputs from the `swagger`, `protobuf`, and `protobuf-fds` targets.
 
-Collect all paths to generate:
-
-```bash
-# In project root.
-fd -e proto -x echo '"{}",' | sort
-```
-Copy the output to `build.rs`, then:
-
-```bash
-cargo build
-```
+Results of this step is committed already. But to run, invoke `just codegen`
 
 ### Hack
 
-Generate a [`FileDescriptorSet`] containing all of the input files:
+Generate a [`FileDescriptorSet`] containing all of the input files wih `just codegen-fds`
 
-```bash
-protoc \
-    --include_imports \
-    --include_source_info \
-    --descriptor_set_out=k8s.pb \
-    --proto_path=./protos \
-    ./protos/**/*.proto
-```
 
-Working with `FileDescriptorSet`:
-```rust
-use prost_types::{FileDescriptorProto, FileDescriptorSet};
-let buf = fs::read(fds_path).unwrap();
-let fds = FileDescriptorSet::decode(&*buf).unwrap();
-let files = fds.files;
-```
-
-See [`prost_build`](https://github.com/tokio-rs/prost/blob/32bc87cd0b7301f6af1a338e9afd7717d0f42ca9/prost-build/src/lib.rs#L765-L825).
-
-[`FileDescriptorSet`]: https://github.com/tokio-rs/prost/blob/32bc87cd0b7301f6af1a338e9afd7717d0f42ca9/prost-types/src/protobuf.rs#L1-L7
-
-
-## OpenAPI
+## OpenAPI Strategy
 
 We need to use `swagger.json` to fill in some information.
 
@@ -97,28 +58,6 @@ We should be able to find the following:
     - May also have paths for all namespaces for some verbs (e.g., `list` all pods)
   - Subresource if path contains `/{name}/` (`/` after `{name}`)
 
-### Download
-
-In `openapi/`
-
-```bash
-VERSION=1.22.0
-curl -sSL -o swagger.json \
-    https://raw.githubusercontent.com/kubernetes/kubernetes/v$VERSION/api/openapi-spec/swagger.json
-```
-
-### Bug Fix
-
-Fix path operation annotated with a `x-kubernetes-group-version-kind` that references a type that doesn't exist in the schema. (See [`k8s-openapi`](https://github.com/Arnavion/k8s-openapi/blob/445e89ec444ebb1c68e61361e64eec4c4a3f4785/k8s-openapi-codegen/src/fixups/upstream_bugs.rs#L9)).
-
-```bash
-gron swagger.json \
-| perl -pe 's/(?<=kind = ")(Pod|Node|Service)(?:Attach|Exec|PortForward|Proxy)Options(?=")/$1/' \
-| gron -u \
-> swagger-patched.json 
-mv swagger-patched.json swagger.json
-```
-
 ### Transforming
 
 Transform `swagger.json` to something easier to explore.

justfile

@@ -0,0 +1,77 @@
+VERSION := "1.22.0"
+
+default:
+  @just --list
+
+# Download protos schemas from upstream
+protos-dl:
+    #!/usr/bin/env bash
+    set -exuo pipefail
+    rm -rf protos && mkdir protos && cd protos
+    for x in api apimachinery apiextensions-apiserver kube-aggregator metrics; do
+        mkdir ./$x -p
+        curl -sSL https://github.com/kubernetes/$x/archive/refs/tags/kubernetes-{{VERSION}}.tar.gz | tar xzf - -C ./$x/ --strip-components=1
+        fd -e proto -x sh -c "mkdir -p k8s.io/'{//}'; mv '{}' k8s.io/'{}'" ';' . ./$x
+        rm -rf ./$x
+    done
+
+# Patch protos schemas to fix import paths
+protos-patch:
+    #!/usr/bin/env bash
+    set -exuo pipefail
+    fd -e proto -x sd 'k8s\.io\.(.+);' '$1;' {}
+    fd -e proto -x sd 'import "k8s\.io/(.+)";' 'import "$1";' {}
+    mv protos/k8s.io/* protos/
+    rmdir protos/k8s.io/
+
+# Generate protos path list for prost
+protos-list:
+    fd -e proto | sort > protos.list
+
+# Download and generate all protos dependent files
+protos: protos-dl protos-patch protos-list
+
+# Download swagger
+swagger-dl:
+    #!/usr/bin/env bash
+    set -exuo pipefail
+    curl -sSL -o k8s-pb-codegen/openapi/swagger.json \
+        https://raw.githubusercontent.com/kubernetes/kubernetes/v{{VERSION}}/api/openapi-spec/swagger.json
+
+# Patch swagger schema for upstream bugs
+swagger-patch:
+    #!/usr/bin/env bash
+    set -exuo pipefail
+    cd k8s-pb-codegen/openapi
+    jq -f patches/patch-nonexistent-gvk.jq < swagger.json > swagger-patched.json
+    mv swagger-patched.json swagger.json
+
+# Transform swagger schema into api-resources.json
+swagger-transform:
+    #!/usr/bin/env bash
+    set -exuo pipefail
+    cd k8s-pb-codegen/openapi
+    jq -f list-resources.jq < swagger.json > api-resources.json
+
+# Download and generate all swagger dependent files
+swagger: swagger-dl swagger-patch swagger-transform
+
+# Build a FileDescriptorSet for custom code generation
+codegen-fds:
+    #!/usr/bin/env bash
+    set -exuo pipefail
+    shopt -s globstar
+    cd k8s-pb-codegen
+    protoc \
+        --include_imports \
+        --include_source_info \
+        --descriptor_set_out=protos.fds \
+        --proto_path=./protos \
+        ./protos/**/*.proto
+
+# Generate the library code from completed swagger and protos
+codegen: codegen-fds
+    #!/usr/bin/env bash
+    set -exuo pipefail
+    rm -rf out/ && mkdir out
+    cd k8s-pb-codegen && cargo run

k8s-pb-codegen/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "k8s-pb-codegen"
+version = "0.1.0"
+edition = "2018"
+license = "Apache-2.0"
+
+[[bin]]
+name = "pbcodegen"
+path = "pbcodegen.rs"
+
+[lib]
+name = "pbcodegen"
+path = "src/lib.rs"
+
+[dependencies]
+bytes = "1.0.1"
+prost = "0.8.0"
+prost-build = "0.8.0"
+prost-types = "0.8.0"
+serde_json = "1.0.67"
+serde = { version = "1.0.130", features = ["derive"] }
+log = "0.4.14"
+anyhow = "1.0.44"
+env_logger = "0.9.0"