Master

.dockerignore

@@ -0,0 +1,2 @@
+.git
+docker

.gitignore

@@ -0,0 +1,5 @@
+.idea/
+vendor/
+composer.lock
+__pycache__
+.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Comsave
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,81 @@
-# morty-counts
+# PrometheusPushGatewayBundle
 
-![](https://media.giphy.com/media/e6tJpLvjY8jXa/giphy.gif)
+## Symfony Prometheus + PushGateway integration
+
+Send metrics to Prometheus. +High Availability Setup 
+
+## Requirements
+
+1. `Redis` (the service & the PHP extension): to act as a buffer before push and avoid latency in the code, and as a store for the current counter values. 
+
+2. `bin/console comsave:prometheus:push` cronjob: to push data periodically to Prometheus Pushgateway
+
+## Configuration
+
+In your `services.yml` add:
+
+```yaml
+comsave_prometheus_pushgateway:
+    prometheus:
+        host: 'prometheus:9090'
+        username: 'admin' # optional
+        password: 'duuude' # optional
+        instance: 'moms_basement:6666' # your server host/name/etc
+    pushgateway:
+        host: 'pushgateway:9191'
+        username: 'admin2' # optional
+        password: 'duuude2' # optional
+        redis: 'redis:6379'
+    metrics: 
+      api: # metric namespace
+        - name: 'orders'
+          type: 'counter'
+          help: 'counts the number of orders'
+          prefetch_group_label: 'user_id' # optional & only available for the counter; prefetch current value from prometheus by grouping sum() query by this label and populate the counter if it's missing in redis cache 
+          labels:
+            - 'order_id'
+            - 'user_id'
+```
+
+Add the bundle to your Symfony kernel.
+```php
+new Comsave\PrometheusPushGatewayBundle\ComsavePrometheusPushGatewayBundle(),
+```
+
+## How does it work?
+
+### General Prometheus architecture overview
+
+![](https://camo.githubusercontent.com/78b3b29d22cea8eee673e34fd204818ea532c171/68747470733a2f2f63646e2e6a7364656c6976722e6e65742f67682f70726f6d6574686575732f70726f6d65746865757340633334323537643036396336333036383564613335626365663038343633326666643564363230392f646f63756d656e746174696f6e2f696d616765732f6172636869746563747572652e737667)
+
+### Single Node Prometheus + Pushgateway
+
+Single node is pretty straightforward. 
+
+1. Use `PushGatewayClient` to create a metric. Metric is stored in `Redis`.
+2. Use `PushGatewayClient` can be pushed manually or with a command. After push metrics stored in Redis are transported to the actual `PushGateway` service.
+3. `Prometheus` periodically pulls in new metrics from `PushGateway`.
+
+![](./images/basic_prometheus_cluster_setup.png)
+
+### Multi-Node Prometheus + Pushgateway Cluster
+
+Multi-node set up works with the basics described above, with a couple exceptions:
+
+1. There's an `Haproxy` (or other load balancer) that decides which `PushGateway` will receive the `push`.
+2. Each `Prometheus` pulls from every `PushGateway` in every node. That way each `Prometheus` has the latest metrics.
+3. Each `Prometheus` pulls (federates) from other `Prometheus` nodes (all but itself) though less often. This ensures data integrity (sort of replication).
+
+![](./images/advanced_prometheus_cluster_setup.png)
+
+## Development
+
+Start single node `docker-compose up -d`
+
+Or multi node     `docker-compose up -f docker-compose.multi-node.yml -d`
+
+Tests `docker exec $(docker ps | grep _php | awk '{print $1}') vendor/bin/phpunit tests`
+
+## License
+
+MIT

composer.json

@@ -0,0 +1,37 @@
+{
+  "name": "comsave/prometheus-pushgateway-bundle",
+  "description": "Symfony integration to push metrics to Prometheus + PushGateway",
+  "license": "MIT",
+  "require": {
+    "php": ">=7.3",
+    "comsave/dependency-injection-config-to-parameters": "^0.1.0",
+    "comsave/prometheus_client_php": "^1.0.5",
+    "jms/serializer": "^3.0|^2.0",
+    "symfony/config": "^3.4|^4.4",
+    "symfony/console": "^3.4|^4.4",
+    "symfony/http-kernel": "^3.4|^4.4"
+  },
+  "require-dev": {
+    "phpunit/phpunit": "^8.0",
+    "spatie/phpunit-snapshot-assertions": "^3.0"
+  },
+  "autoload": {
+    "psr-4": {
+      "Comsave\\PrometheusPushGatewayBundle\\": "src/PrometheusPushGatewayBundle"
+    }
+  },
+  "autoload-dev": {
+    "psr-4": {
+      "Comsave\\PrometheusPushGatewayBundle\\Tests\\": "tests"
+    }
+  },
+  "authors": [
+    {
+      "name": "Vaidas Bagdonas",
+      "email": "[email protected]"
+    }
+  ],
+  "config": {
+    "sort-packages": true
+  }
+}

docker-compose.multi-node.yml

@@ -0,0 +1,101 @@
+version: "3"
+
+services:
+  php:
+    build:
+      dockerfile: ./docker/php-fpm/Dockerfile
+      context: .
+    depends_on:
+      - redis
+    volumes:
+      - ./:/app
+
+  redis:
+    image: redis:alpine
+
+  haproxy:
+    image: haproxy:alpine
+    depends_on:
+      - prometheus
+      - prometheus2
+      - prometheus3
+      - pushgateway
+      - pushgateway2
+      - pushgateway3
+    volumes:
+      - ./docker/haproxy:/usr/local/etc/haproxy:ro
+
+  prometheus:
+    image: prom/prometheus
+    command:
+      - '--web.listen-address=:9091'
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+    depends_on:
+      - pushgateway
+    ports:
+      - 9091:9091
+    volumes:
+      - ./docker/prometheus/prometheus1.yml/:/etc/prometheus/prometheus.yml
+
+  pushgateway:
+    image: prom/pushgateway
+    command:
+      - '--web.listen-address=:9191'
+    ports:
+      - 9191:9191
+
+  prometheus2:
+    image: prom/prometheus
+    command:
+      - '--web.listen-address=:9092'
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+    depends_on:
+      - pushgateway2
+    ports:
+      - 9092:9092
+    volumes:
+      - ./docker/prometheus/prometheus2.yml/:/etc/prometheus/prometheus.yml
+
+  pushgateway2:
+    image: prom/pushgateway
+    command:
+      - '--web.listen-address=:9192'
+    ports:
+      - 9192:9192
+
+  prometheus3:
+    image: prom/prometheus
+    command:
+      - '--web.listen-address=:9093'
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+    depends_on:
+      - pushgateway3
+    ports:
+      - 9093:9093
+    volumes:
+      - ./docker/prometheus/prometheus3.yml/:/etc/prometheus/prometheus.yml
+
+  pushgateway3:
+    image: prom/pushgateway
+    command:
+      - '--web.listen-address=:9193'
+    ports:
+      - 9193:9193
+
+  grafana:
+    image: grafana/grafana
+    depends_on:
+      - haproxy
+    ports:
+      - 3000:3000
+    volumes:
+      - ./docker/grafana/datasources1.yml:/etc/grafana/provisioning/datasources/datasources.yml
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false

docker-compose.yml

@@ -0,0 +1,49 @@
+version: "3"
+
+services:
+  php:
+    build:
+      dockerfile: ./docker/php-fpm/Dockerfile
+      context: .
+    depends_on:
+      - redis
+    volumes:
+      - ./:/app
+
+  redis:
+    image: redis:alpine
+
+  prometheus:
+    image: prom/prometheus
+    command:
+      - '--web.listen-address=:9090'
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+    depends_on:
+      - pushgateway
+    ports:
+      - 9090:9090
+    volumes:
+      - ./docker/prometheus/prometheus.yml/:/etc/prometheus/prometheus.yml
+
+  pushgateway:
+    image: prom/pushgateway
+    command:
+      - '--web.listen-address=:9191'
+      - '--push.disable-consistency-check'
+      - '--persistence.interval=5m'
+    ports:
+      - 9191:9191
+
+  grafana:
+    image: grafana/grafana
+    depends_on:
+      - prometheus
+    ports:
+      - 3000:3000
+    volumes:
+      - ./docker/grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false

docker/grafana/datasources.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true

docker/grafana/datasources1.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://haproxy:9090
+    isDefault: true

docker/haproxy/haproxy.cfg

@@ -0,0 +1,31 @@
+global
+# global settings here
+
+defaults
+# defaults here
+
+frontend pushgateway_fe
+    bind :9191
+    mode tcp
+    default_backend pushgateway_be
+
+backend pushgateway_be
+    mode tcp
+    balance roundrobin
+    default-server inter 1s
+    server pushgateway  pushgateway:9191  check id 1
+    server pushgateway2 pushgateway2:9192 check id 2
+    server pushgateway3 pushgateway3:9193 check id 3
+
+frontend prometheus_fe
+    bind :9090
+    mode tcp
+    default_backend prometheus_be
+
+backend prometheus_be
+    mode tcp
+    balance roundrobin
+    default-server inter 1s
+    server prometheus  prometheus:9091  check id 1
+    server prometheus2 prometheus2:9092 check id 2
+    server prometheus3 prometheus3:9093 check id 3

docker/php-fpm/Dockerfile

@@ -0,0 +1,18 @@
+FROM php:7.3-fpm
+
+RUN apt-get update && apt-get install git -y
+
+RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && \
+    php -r "if (hash_file('sha384', 'composer-setup.php') === 'e0012edf3e80b6978849f5eff0d4b4e4c79ff1609dd1e613307e16318854d24ae64f26d17af3ef0bf7cfb710ca74755a') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;" && \
+    php composer-setup.php && \
+    php -r "unlink('composer-setup.php');" && \
+    mv composer.phar /usr/local/bin/composer && \
+    chmod +x /usr/local/bin/composer
+
+RUN pecl install redis && docker-php-ext-enable redis
+RUN apt-get install zip unzip -y
+
+WORKDIR /app
+COPY composer.* ./
+RUN composer install
+COPY ./* ./

docker/prometheus/prometheus.yml

@@ -0,0 +1,9 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'pushgateway'
+    honor_labels: true
+    scrape_interval: 500ms
+    static_configs:
+      - targets: ['pushgateway:9191']