Add logging support to the git-sync container

Author: siegfriedweber

Cargo.lock

@@ -484,6 +484,10 @@ pub async fn reconcile_airflow(
                 &resolved_product_image,
                 &env_vars_from_rolegroup_config(rolegroup_config),
                 &airflow.volume_mounts(),
+                LOG_VOLUME_NAME,
+                &merged_airflow_config
+                    .logging
+                    .for_container(&Container::GitSync),
             )
             .context(InvalidGitSyncSpecSnafu)?;
 
@@ -628,6 +632,10 @@ async fn build_executor_template(
         resolved_product_image,
         &env_vars_from(&common_config.env_overrides),
         &airflow.volume_mounts(),
+        LOG_VOLUME_NAME,
+        &merged_executor_config
+            .logging
+            .for_container(&Container::GitSync),
     )
     .context(InvalidGitSyncSpecSnafu)?;
 

Cargo.nix

@@ -711,6 +711,7 @@ pub enum Container {
     Airflow,
     Vector,
     Base,
+    GitSync,
 }
 
 #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)]

crate-hashes.json

@@ -70,6 +70,9 @@ spec:
     vectorAggregatorConfigMapName: airflow-vector-aggregator-discovery
     loadExamples: true
     credentialsSecret: test-airflow-credentials
+    dagsGitSync:
+      - repo: https://github.com/stackabletech/example-dags
+        gitFolder: dags
   webservers:
     config:
       resources:
@@ -93,6 +96,14 @@ spec:
                 loggers:
                   ROOT:
                     level: INFO
+              git-sync:
+                console:
+                  level: INFO
+                file:
+                  level: INFO
+                loggers:
+                  ROOT:
+                    level: INFO
               vector:
                 console:
                   level: INFO
@@ -128,6 +139,8 @@ spec:
         cpu:
           min: 50m
           max: 250m
+        memory:
+          limit: 3Gi
     roleGroups:
       automatic-log-config:
         replicas: 1
@@ -143,6 +156,14 @@ spec:
                 loggers:
                   ROOT:
                     level: DEBUG
+              git-sync:
+                console:
+                  level: INFO
+                file:
+                  level: INFO
+                loggers:
+                  ROOT:
+                    level: INFO
               vector:
                 console:
                   level: INFO
@@ -181,6 +202,14 @@ spec:
             loggers:
               ROOT:
                 level: DEBUG
+          git-sync:
+            console:
+              level: INFO
+            file:
+              level: INFO
+            loggers:
+              ROOT:
+                level: INFO
           vector:
             console:
               level: INFO
@@ -198,6 +227,8 @@ spec:
         cpu:
           min: 100m
           max: 500m
+        memory:
+          limit: 1Gi
     roleGroups:
       automatic-log-config:
         replicas: 1
@@ -213,6 +244,14 @@ spec:
                 loggers:
                   ROOT:
                     level: INFO
+              git-sync:
+                console:
+                  level: INFO
+                file:
+                  level: INFO
+                loggers:
+                  ROOT:
+                    level: INFO
               vector:
                 console:
                   level: INFO

rust/operator-binary/src/airflow_controller.rs

@@ -30,6 +30,12 @@ customConfig:
       condition: >-
         .pod == "airflow-webserver-automatic-log-config-0" &&
         .container == "airflow"
+    filteredAutomaticLogConfigWebserverGitSync:
+      type: filter
+      inputs: [validEvents]
+      condition: >-
+        .pod == "airflow-webserver-automatic-log-config-0" &&
+        .container == "git-sync-0"
     filteredAutomaticLogConfigWebserverVector:
       type: filter
       inputs: [validEvents]
@@ -54,6 +60,12 @@ customConfig:
       condition: >-
         .pod == "airflow-scheduler-automatic-log-config-0" &&
         .container == "airflow"
+    filteredAutomaticLogConfigSchedulerGitSync:
+      type: filter
+      inputs: [validEvents]
+      condition: >-
+        .pod == "airflow-scheduler-automatic-log-config-0" &&
+        .container == "git-sync-0"
     filteredAutomaticLogConfigSchedulerVector:
       type: filter
       inputs: [validEvents]
@@ -79,6 +91,18 @@ customConfig:
       condition: >-
         .pod == "airflow-worker-automatic-log-config-0" &&
         .container == "airflow"
+    filteredAutomaticLogConfigWorkerGitSync:
+      type: filter
+      inputs: [validEvents]
+      condition: >-
+        .pod == "airflow-worker-automatic-log-config-0" &&
+        .container == "git-sync-0"
+    filteredAutomaticLogConfigWorkerGitSyncInit:
+      type: filter
+      inputs: [validEvents]
+      condition: >-
+        .pod == "airflow-worker-automatic-log-config-0" &&
+        .container == "git-sync-0-init"
     filteredAutomaticLogConfigWorkerVector:
       type: filter
       inputs: [validEvents]
@@ -104,6 +128,12 @@ customConfig:
       condition: >-
         starts_with(string!(.pod), "example-trigger-target-dag-bash-task-") &&
         .container == "base"
+    filteredExampleTriggerTargetDagBashGitSyncInit:
+      type: filter
+      inputs: [validEvents]
+      condition: >-
+        starts_with(string!(.pod), "example-trigger-target-dag-bash-task-") &&
+        .container == "git-sync-0-init"
     filteredExampleTriggerTargetDagBashTaskVector:
       type: filter
       inputs: [validEvents]
@@ -116,6 +146,12 @@ customConfig:
       condition: >-
         starts_with(string!(.pod), "example-trigger-target-dag-run-this-") &&
         .container == "base"
+    filteredExampleTriggerTargetDagRunThisGitSyncInit:
+      type: filter
+      inputs: [validEvents]
+      condition: >-
+        starts_with(string!(.pod), "example-trigger-target-dag-run-this-") &&
+        .container == "git-sync-0-init"
     filteredExampleTriggerTargetDagRunThisVector:
       type: filter
       inputs: [validEvents]