Updated images and manifests to 6.5.4

Author: carlosedp

README.md

@@ -18,43 +18,32 @@ kctl() {
 }
 # alias kctl='kubectl --namespace logging'
 
-# Deploy Elasticsearch service
-kctl apply -f es-discovery-svc.yaml
-kctl apply -f es-svc.yaml
+# Deploy ElasticSearch configmap
 kctl apply -f es-configmap.yaml
 
-# Deploy Elasticsearch master node and wait until it's up
-kctl apply -f es-master.yaml
-until kctl rollout status deployment es-master  > /dev/null 2>&1; do sleep 1; printf "."; done
-
-# Deploy Elasticsearch client node and wait until it's up
-kctl apply -f es-client.yaml
-until kctl rollout status deployment es-client  > /dev/null 2>&1; do sleep 1; printf "."; done
-
-# Deploy Elasticsearch data node and wait until it's up
-kctl apply -f es-data-statefulset.yaml
-until kctl rollout status statefulset es-data  > /dev/null 2>&1; do sleep 1; printf "."; done
-
-# As an alternative, replace es-discovery-svc.yaml, es-svc.yaml and the master, client and data manifests
-# with the ones below to have a three-node cluster with all roles in all nodes. 
-#kctl apply -f es-full-svc.yaml
-#kctl apply -f es-full.yaml
-
+# As an alternative, replace  master, client and data manifests applied above
+# with the ones below to have a three-node cluster with all roles in all nodes.
+kctl apply -f es-full-svc.yaml
+kctl apply -f es-full-statefulset.yaml
+until kctl rollout status statefulset es-full  > /dev/null 2>&1; do sleep 1; printf "."; done
 
 # Deploy Curator
-kctl apply -f es-curator-config.yaml
-kctl apply -f es-curator_v1beta1.yaml
+kctl apply -f es-curator-configmap.yaml
+kctl apply -f es-curator-cronjob.yaml
+
+# Deploy Cerebro
+kctl apply -f cerebro.yaml
+kctl apply -f cerebro-external-ingress.yaml
 
 # Deploy Kibana
 kctl apply -f kibana-configmap.yaml
-kctl apply -f kibana-external-ingress.yaml
-kctl apply -f kibana-service-account.yaml
 kctl apply -f kibana-svc.yaml
-kctl apply -f kibana.yaml
+kctl apply -f kibana-deployment.yaml
+kctl apply -f kibana-external-ingress.yaml
 
 # Deploy Fluentd
-kctl apply -f fluentd-es-configmap.yaml
-kctl apply -f fluentd-es-ds.yaml
+kctl apply -f fluentd-configmap.yaml
+kctl apply -f fluentd-daemonset.yaml
 
 echo "done!"
 

deploy

@@ -26,7 +26,7 @@ data:
           direction: older
           timestring: '%Y.%m.%d'
           unit: days
-          unit_count: 15
+          unit_count: 10
           field:
           stats_result:
           epoch:

es-curator-configmap.yaml

@@ -6,6 +6,7 @@ spec:
   schedule: 0 8 * * *
   jobTemplate:
     spec:
+      ttlSecondsAfterFinished: 180
       template:
         spec:
           containers:

es-curator-cronjob.yaml

@@ -33,10 +33,10 @@ spec:
         image: carlosedp/docker-elasticsearch-kubernetes:6.5.4
         resources:
           limits:
-            cpu: 1
+            cpu: 2
           requests:
             cpu: 1
-            memory: "768Mi"
+            memory: 1Gi
         ports:
         - containerPort: 9300
           name: transport
@@ -54,7 +54,7 @@ spec:
         - name: CLUSTER_NAME
           value: myesdb
         - name: ES_JAVA_OPTS
-          value: -Xms512m -Xmx512m
+          value: -Xms1G -Xmx1G
         - name: NETWORK_HOST
           value: _site_,_lo_
         - name: NUMBER_OF_MASTERS
@@ -68,13 +68,8 @@ spec:
         livenessProbe:
           tcpSocket:
             port: transport
-          initialDelaySeconds: 180
+          initialDelaySeconds: 120
           periodSeconds: 10
-        readinessProbe:
-          httpGet:
-            path: /_cluster/health
-            port: http
-          initialDelaySeconds: 180
           timeoutSeconds: 60
         volumeMounts:
         - name: config

all-roles-nodes/es-full-svc.yaml renamed to es-full-svc.yaml

@@ -446,6 +446,9 @@ data:
       request_timeout 45s
       reload_connections false
       logstash_format true
+      template_name template-logstash
+      template_file /etc/fluent/config.d/template-logstash.json
+      template_overwrite true
       id_key _hash        # specify same key name which is specified in hash_id_key
       remove_keys _hash   # Elasticsearch doesn't like keys that start with _
       <buffer>
@@ -463,3 +466,11 @@ data:
       </buffer>
     </match>
 
+  template-logstash.json: |-
+    {
+      "index_patterns": ["logstash*"],
+      "settings": {
+        "number_of_shards": 3,
+        "number_of_replicas": 0
+      }
+    }

all-roles-nodes/es-full.yaml renamed to es-full.yaml

@@ -52,8 +52,6 @@ metadata:
   namespace: logging
   labels:
     k8s-app: fluentd-es
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
@@ -62,14 +60,7 @@ spec:
     metadata:
       labels:
         k8s-app: fluentd-es
-        kubernetes.io/cluster-service: "true"
-      # This annotation ensures that fluentd does not get evicted if the node
-      # supports critical pod annotation based priority scheme.
-      # Note that this does not guarantee admission on the nodes (#40573).
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
-      priorityClassName: system-node-critical
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es

fluentd-configmap.yaml

@@ -3,7 +3,7 @@ status = error
 appender.console.type = Console
 appender.console.name = console
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
 
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console

fluentd-daemonset.yaml

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+if [ -z "${KUBECONFIG}" ]; then
+    export KUBECONFIG=~/.kube/config
+fi
+
+# CAUTION - setting NAMESPACE will deploy most components to the given namespace
+# however some are hardcoded to 'monitoring'. Only use if you have reviewed all manifests.
+
+if [ -z "${NAMESPACE}" ]; then
+    NAMESPACE=logging
+fi
+
+kubectl create namespace "$NAMESPACE"
+
+kctl() {
+    kubectl --namespace "$NAMESPACE" "$@"
+}
+# alias kctl='kubectl --namespace logging'
+
+# Deploy ElasticSearch configmap
+kctl apply -f es-configmap.yaml
+
+# Deploy Elasticsearch master node and wait until it's up
+kctl apply -f es-master-svc.yaml
+kctl apply -f es-master-statefulset.yaml
+until kctl rollout status statefulset es-master  > /dev/null 2>&1; do sleep 1; printf "."; done
+
+# Deploy Elasticsearch client node and wait until it's up
+kctl apply -f es-ingest-svc.yaml
+kctl apply -f es-ingest-statefulset.yaml
+until kctl rollout status deployment es-client  > /dev/null 2>&1; do sleep 1; printf "."; done
+
+# Deploy Elasticsearch data node and wait until it's up
+kctl apply -f es-data-svc.yaml
+kctl apply -f es-data-statefulset.yaml
+until kctl rollout status statefulset es-data  > /dev/null 2>&1; do sleep 1; printf "."; done
+
+# Deploy Curator
+kctl apply -f es-curator-configmap.yaml
+kctl apply -f es-curator-cronjob.yaml
+
+# Deploy Cerebro
+kctl apply -f cerebro.yaml
+kctl apply -f cerebro-external-ingress.yaml
+
+# Deploy Kibana
+kctl apply -f kibana-configmap.yaml
+kctl apply -f kibana-svc.yaml
+kctl apply -f kibana-deployment.yaml
+kctl apply -f kibana-external-ingress.yaml
+
+# Deploy Fluentd
+kctl apply -f fluentd-configmap.yaml
+kctl apply -f fluentd-daemonset.yaml
+
+echo "done!"
+

images/docker-elasticsearch/config/log4j2.properties

@@ -56,10 +56,12 @@ spec:
           value: "false"
         - name: NODE_INGEST
           value: "false"
+        - name: NODE_DATA
+          value: "true"
         - name: HTTP_ENABLE
           value: "true"
         - name: ES_JAVA_OPTS
-          value: -Xms1g -Xmx1g
+          value: -Xms512m -Xmx512m
         - name: PROCESSORS
           valueFrom:
             resourceFieldRef:
@@ -69,19 +71,13 @@ spec:
         livenessProbe:
           tcpSocket:
             port: transport
-          initialDelaySeconds: 180
-          periodSeconds: 10
-        readinessProbe:
-          httpGet:
-            path: /_cluster/health
-            port: http
-          initialDelaySeconds: 180
-          timeoutSeconds: 60
+          initialDelaySeconds: 360
+          periodSeconds: 30
         volumeMounts:
         - name: config
           mountPath: /elasticsearch/config/elasticsearch.yml
           subPath: elasticsearch.yml
-        - name: storage-data
+        - name: elasticsearch-data
           mountPath: /data
       affinity:
         podAntiAffinity:
@@ -105,7 +101,7 @@ spec:
           name: es-configmap
   volumeClaimTemplates:
   - metadata:
-      name: storage-data
+      name: elasticsearch-data
     spec:
       accessModes: [ "ReadWriteOnce" ]
       resources:

poddisruptionbudget/es-data-pdb.yaml renamed to pdb/es-data-pdb.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: elasticsearch-data
+  name: elasticsearch-data-discovery
   labels:
     component: elasticsearch
     role: data
@@ -10,6 +10,6 @@ spec:
     component: elasticsearch
     role: data
   ports:
-  - port: 9200
-    name: http
+  - port: 9300
+    name: transport
   clusterIP: None

poddisruptionbudget/es-master-pdb.yaml renamed to pdb/es-master-pdb.yaml

@@ -1,22 +1,16 @@
-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
-  name: elasticsearch-ingest
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 5Gi
----
-apiVersion: apps/v1beta1
-kind: Deployment
+apiVersion: apps/v1
+kind: StatefulSet
 metadata:
   name: es-ingest
   labels:
     component: elasticsearch
     role: ingest
 spec:
+  selector:
+    matchLabels:
+      component: elasticsearch
+      role: ingest
+  serviceName: elasticsearch-ingest
   replicas: 1
   template:
     metadata:
@@ -49,21 +43,23 @@ spec:
           value: myesdb
         - name: NODE_MASTER
           value: "false"
+        - name: NODE_INGEST
+          value: "true"
         - name: NODE_DATA
           value: "false"
         - name: HTTP_ENABLE
           value: "true"
         - name: ES_JAVA_OPTS
-          value: -Xms1g -Xmx1g
+          value: -Xms512m -Xmx512m
         - name: NETWORK_HOST
           value: _site_,_lo_
         - name: PROCESSORS
           valueFrom:
             resourceFieldRef:
               resource: limits.cpu
-        #- name: "ES_PLUGINS_INSTALL"
-        #  value: "x-pack"
         resources:
+          requests:
+            cpu: 1
           limits:
             cpu: 2
         ports:
@@ -72,27 +68,26 @@ spec:
         - containerPort: 9300
           name: transport
         livenessProbe:
-          tcpSocket:
-            port: transport
-          initialDelaySeconds: 180
-          periodSeconds: 10
-        readinessProbe:
-          httpGet:
+          httpGet:                                                                                                                                                          
             path: /_cluster/health
-            port: http
-          initialDelaySeconds: 180
+            port: 9200
+          initialDelaySeconds: 120
           timeoutSeconds: 60
         volumeMounts:
-        - name: storage
+        - name: elasticsearch-ingest
           mountPath: /data
         - name: config
           mountPath: /elasticsearch/config/elasticsearch.yml
           subPath: elasticsearch.yml
       volumes:
-      - name: storage
-        persistentVolumeClaim:
-          claimName: elasticsearch-ingest
       - name : config
         configMap:
           name: es-configmap
-
+  volumeClaimTemplates:
+  - metadata:
+      name: elasticsearch-ingest
+    spec:
+      accessModes: [ ReadWriteOnce ]
+      resources:
+        requests:
+          storage: 5Gi