Skip to content

PMM-13794 fix pmm agent on pmm server #4032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: v3
Choose a base branch
from
+417 −369
Open

PMM-13794 fix pmm agent on pmm server #4032

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
04a5770
Reapply "PMM-13794: Add HAService integration to server initializatio…
BupycHuk May 19, 2025
36f2249
PMM-13794 Enhance HA services with concurrency handling
BupycHuk May 19, 2025
771de22
PMM-13794 Enhance HA services with concurrency handling
BupycHuk May 19, 2025
b5a3cbf
Update managed/services/ha/leaderservice.go
BupycHuk May 19, 2025
0f3b6d9
PMM-13794: Split HA setup into a separate compose file
BupycHuk May 19, 2025
26988e0
PMM-13794 Update docker-compose.ha.yml for improved configurations
BupycHuk May 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build/ansible/roles/initialization/tasks/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
host: 127.0.0.1
port: 5432
timeout: 150
when: lookup('env','PMM_POSTGRES_ADDR') == '127.0.0.1:5432' and lookup('env','PMM_DISABLE_BUILTIN_POSTGRES') != '1'
when: lookup('env','GF_DATABASE_URL') == '' and lookup('env','GF_DATABASE_HOST') == '' and lookup('env','PMM_DISABLE_BUILTIN_POSTGRES') == ''
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if the user sets PMM_DISABLE_BUILTIN_POSTGRES to 0?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it will be very weird decission 😊, but this solution is easier to support then multiple variations of True value

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, there is a better way, as usual. For example, we could solve it with the following syntax:

    when: 
      - lookup('env','GF_DATABASE_URL') == '' 
      - lookup('env','GF_DATABASE_HOST') == '' 
      - not (lookup('env','PMM_DISABLE_BUILTIN_POSTGRES') | bool)


- name: Create Grafana DB
block:
Expand Down
365 changes: 365 additions & 0 deletions docker-compose.ha.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,365 @@
---
services:
# PMM with external DBs
ch:
image: ${CH_IMAGE:-clickhouse/clickhouse-server:23.8.2.7-alpine}
platform: linux/amd64
hostname: ${CH_HOSTNAME:-ch}
ports:
- ${CH_PORT:-9000}:9000
environment:
- CLICKHOUSE_PASSWORD=clickhouse
networks:
ha:
ipv4_address: 172.20.0.7
volumes:
- chdata:/var/lib/clickhouse # Volume for ClickHouse data


victoriametrics:
hostname: ${VM_HOSTNAME:-victoriametrics}
image: victoriametrics/victoria-metrics:v1.93.4
ports:
- 8428:8428
- 8089:8089
- 8089:8089/udp
- 2003:2003
- 2003:2003/udp
- 4242:4242
volumes:
- vmdata:/storage
command:
- "--storageDataPath=/storage"
- "--graphiteListenAddr=:2003"
- "--opentsdbListenAddr=:4242"
- "--httpListenAddr=:8428"
- "--influxListenAddr=:8089"
networks:
ha:
ipv4_address: 172.20.0.4

# PMM with external Postgres DB
pg:
# build:
# context: ./managed/testdata/pg
# args:
# POSTGRES_IMAGE: ${POSTGRES_IMAGE:-postgres:14}
# dockerfile: Dockerfile
image: postgres:14
container_name: pg
environment:
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-pmm-password}
ports:
- ${POSTGRES_PORT:-5432}:5432
command: |
postgres
-c shared_preload_libraries=pg_stat_statements
-c pg_stat_statements.max=10000
-c pg_stat_statements.track=all
-c pg_stat_statements.save=off
-c fsync=off
-c hba_file=/conf/pg_hba.conf
-c log_statement=all
# -c ssl=on
# -c ssl_ca_file=/certs/root.crt
# -c ssl_cert_file=/certs/server.crt
# -c ssl_key_file=/certs/server.key
networks:
ha:
ipv4_address: 172.20.0.3
volumes:
- pgdata:/var/lib/postgresql/data # Volume for PostgreSQL data
- ./managed/testdata/pg/conf/:/conf/
- ./managed/testdata/pg/queries/:/docker-entrypoint-initdb.d/
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s
timeout: 5s
retries: 5
start_period: 10s


haproxy:
image: haproxy:latest
container_name: haproxy
hostname: haproxy
networks:
ha:
ipv4_address: 172.20.0.10
volumes:
- ./managed/testdata/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
- ./managed/testdata/haproxy/localhost.pem:/usr/local/etc/ssl/private/localhost.pem:ro
ports:
- 7080:80
- 7443:443
depends_on:
- pmm-server-active
- pmm-server-passive
- pmm-server-passive-2

pmm-server-active:
depends_on:
ch:
condition: service_started
pg:
condition: service_healthy
victoriametrics:
condition: service_started
image: ${PMM_CONTAINER:-perconalab/pmm-server:3-dev-container}
platform: linux/amd64
container_name: pmm-server-active
hostname: pmm-server-active
networks:
ha:
ipv4_address: 172.20.0.5
environment:
- PMM_RELEASE_PATH=/root/go/src/github.com/percona/pmm/bin
- REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN:-}
- AWS_ACCESS_KEY=${AWS_ACCESS_KEY:-}
- AWS_SECRET_KEY=${AWS_SECRET_KEY:-}
- PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000
- PMM_CLICKHOUSE_DATABASE=pmm
- PMM_DISABLE_BUILTIN_CLICKHOUSE=1
- PMM_POSTGRES_ADDR=pg:5432
- PMM_POSTGRES_USERNAME=pmm-managed
- PMM_POSTGRES_DBPASSWORD=pmm-managed
- PMM_DISABLE_BUILTIN_POSTGRES=1
- GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana
- GF_DATABASE_NAME=grafana
- GF_DATABASE_USER=grafana
- GF_DATABASE_PASSWORD=grafana
- GF_DATABASE_HOST=pg
- GF_DATABASE_PORT=5432
- GO_VERSION=1.22
- PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/}
- PMM_TEST_HA_ENABLE=1
- PMM_TEST_HA_BOOTSTRAP=1
- PMM_TEST_HA_NODE_ID=pmm-server-active
- PMM_TEST_HA_ADVERTISE_ADDRESS=172.20.0.5
- PMM_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2
- PMM_TEST_HA_GOSSIP_PORT=9096
- PMM_TEST_HA_GRAFANA_GOSSIP_PORT=9094

extra_hosts:
- host.docker.internal:host-gateway
# - portal.localhost:${PORTAL_HOST:-host-gateway}
# - check.localhost:${PORTAL_CHECK_HOST:-host-gateway}
# - pmm.localhost:${PORTAL_PMM_HOST:-host-gateway}
# - check-dev.percona.com:${PORTAL_PMM_HOST:-host-gateway}

# for delve
cap_add:
- SYS_PTRACE
security_opt:
- seccomp:unconfined

# see https://github.com/golang/go/wiki/LinuxKernelSignalVectorBug#what-to-do
ulimits:
memlock: 67108864

# ports:
# - ${PMM_PORT_HTTP:-8081}:8080
# - ${PMM_PORT_HTTPS:-8441}:8443
# For headless delve
# - ${PMM_PORT_DELVE:-2345}:2345
volumes:
- ./:/root/go/src/github.com/percona/pmm
# - "../grafana/public:/usr/share/grafana/public"
- ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro # change Makefile in devcontainer
# caching
- go-modules:/root/go/pkg/mod
- root-cache:/root/.cache
- ./managed/testdata/pg/certs/:/certs/
healthcheck:
test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8080/v1/server/readyz"]
interval: 10s
timeout: 5s
start_period: 30s
retries: 10

pmm-server-passive:
depends_on:
ch:
condition: service_started
pg:
condition: service_healthy
victoriametrics:
condition: service_started
pmm-server-active:
condition: service_healthy
image: ${PMM_CONTAINER:-perconalab/pmm-server:3-dev-container}
platform: linux/amd64
container_name: pmm-server-passive
hostname: pmm-server-passive
networks:
ha:
ipv4_address: 172.20.0.6
environment:
- PMM_RELEASE_PATH=/root/go/src/github.com/percona/pmm/bin
- REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN:-}
- AWS_ACCESS_KEY=${AWS_ACCESS_KEY:-}
- AWS_SECRET_KEY=${AWS_SECRET_KEY:-}
# - PMM_DEV_PERCONA_PLATFORM_ADDRESS=https://check.localhost
# - PMM_DEV_PERCONA_PLATFORM_INSECURE=1
# - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY=<public key>
# - PMM_DEV_TELEMETRY_INTERVAL=10s
# - PMM_DEV_TELEMETRY_RETRY_BACKOFF=10s
# - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1
- PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000
- PMM_CLICKHOUSE_DATABASE=pmm
- PMM_DISABLE_BUILTIN_CLICKHOUSE=1
- PMM_POSTGRES_ADDR=pg:5432
- PMM_POSTGRES_USERNAME=pmm-managed
- PMM_POSTGRES_DBPASSWORD=pmm-managed
- PMM_DISABLE_BUILTIN_POSTGRES=1
# - PMM_POSTGRES_SSL_MODE=require
# - PMM_POSTGRES_SSL_CA_PATH=/certs/root.crt
# - PMM_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key
# - PMM_POSTGRES_SSL_CERT_PATH=/certs/pmm-managed.crt
- GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana
- GF_DATABASE_NAME=grafana
- GF_DATABASE_USER=grafana
- GF_DATABASE_PASSWORD=grafana
- GF_DATABASE_HOST=pg
- GF_DATABASE_PORT=5432
# - GF_DATABASE_SSL_MODE=require
# - PMM_DEBUG=1
- GO_VERSION=1.20
- PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/}
- PMM_TEST_HA_ENABLE=1
- PMM_TEST_HA_NODE_ID=pmm-server-passive
- PMM_TEST_HA_ADVERTISE_ADDRESS=172.20.0.6
- PMM_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2
- PMM_TEST_HA_GOSSIP_PORT=9096
- PMM_TEST_HA_GRAFANA_GOSSIP_PORT=9094

extra_hosts:
- host.docker.internal:host-gateway
# - portal.localhost:${PORTAL_HOST:-host-gateway}
# - check.localhost:${PORTAL_CHECK_HOST:-host-gateway}
# - pmm.localhost:${PORTAL_PMM_HOST:-host-gateway}
# - check-dev.percona.com:${PORTAL_PMM_HOST:-host-gateway}

# for delve
cap_add:
- SYS_PTRACE
security_opt:
- seccomp:unconfined

# see https://github.com/golang/go/wiki/LinuxKernelSignalVectorBug#what-to-do
ulimits:
memlock: 67108864

# ports:
# - ${PMM_PORT_HTTP:-8082}:8080
# - ${PMM_PORT_HTTPS:-8432}:8443
# For headless delve
# - ${PMM_PORT_DELVE:-12345}:2345
volumes:
- ./:/root/go/src/github.com/percona/pmm
# - "../grafana/public:/usr/share/grafana/public"
- ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro # change Makefile in devcontainer
# caching
- go-modules:/root/go/pkg/mod
- root-cache:/root/.cache
- ./managed/testdata/pg/certs/:/certs/

pmm-server-passive-2:
depends_on:
ch:
condition: service_started
pg:
condition: service_healthy
victoriametrics:
condition: service_started
pmm-server-active:
condition: service_healthy
image: ${PMM_CONTAINER:-perconalab/pmm-server:3-dev-container}
platform: linux/amd64
container_name: pmm-server-passive-2
hostname: pmm-server-passive-2
networks:
ha:
ipv4_address: 172.20.0.11
environment:
- PMM_RELEASE_PATH=/root/go/src/github.com/percona/pmm/bin
- REVIEWDOG_GITHUB_API_TOKEN=${REVIEWDOG_GITHUB_API_TOKEN:-}
- AWS_ACCESS_KEY=${AWS_ACCESS_KEY:-}
- AWS_SECRET_KEY=${AWS_SECRET_KEY:-}
# - PMM_DEV_PERCONA_PLATFORM_ADDRESS=https://check.localhost
# - PMM_DEV_PERCONA_PLATFORM_INSECURE=1
# - PMM_DEV_PERCONA_PLATFORM_PUBLIC_KEY=<public key>
# - PMM_DEV_TELEMETRY_INTERVAL=10s
# - PMM_DEV_TELEMETRY_RETRY_BACKOFF=10s
# - PMM_DEV_TELEMETRY_DISABLE_START_DELAY=1
- PMM_CLICKHOUSE_ADDR=${CH_HOSTNAME:-ch}:9000
- PMM_CLICKHOUSE_DATABASE=pmm
- PMM_DISABLE_BUILTIN_CLICKHOUSE=1
- PMM_POSTGRES_ADDR=pg:5432
- PMM_POSTGRES_USERNAME=pmm-managed
- PMM_POSTGRES_DBPASSWORD=pmm-managed
- PMM_DISABLE_BUILTIN_POSTGRES=1
# - PMM_POSTGRES_SSL_MODE=require
# - PMM_POSTGRES_SSL_CA_PATH=/certs/root.crt
# - PMM_POSTGRES_SSL_KEY_PATH=/certs/pmm-managed.key
# - PMM_POSTGRES_SSL_CERT_PATH=/certs/pmm-managed.crt
- GF_DATABASE_URL=postgres://grafana:grafana@pg:5432/grafana
- GF_DATABASE_NAME=grafana
- GF_DATABASE_USER=grafana
- GF_DATABASE_PASSWORD=grafana
- GF_DATABASE_HOST=pg
- GF_DATABASE_PORT=5432
# - GF_DATABASE_SSL_MODE=require
# - PMM_DEBUG=1
- GO_VERSION=1.20
- PMM_VM_URL=${PMM_VM_URL:-http://victoriametrics:8428/}
- PMM_TEST_HA_ENABLE=1
- PMM_TEST_HA_NODE_ID=pmm-server-passive-2
- PMM_TEST_HA_ADVERTISE_ADDRESS=172.20.0.11
- PMM_TEST_HA_PEERS=pmm-server-active,pmm-server-passive,pmm-server-passive-2
- PMM_TEST_HA_GOSSIP_PORT=9096
- PMM_TEST_HA_GRAFANA_GOSSIP_PORT=9094

extra_hosts:
- host.docker.internal:host-gateway
# - portal.localhost:${PORTAL_HOST:-host-gateway}
# - check.localhost:${PORTAL_CHECK_HOST:-host-gateway}
# - pmm.localhost:${PORTAL_PMM_HOST:-host-gateway}
# - check-dev.percona.com:${PORTAL_PMM_HOST:-host-gateway}

# for delve
cap_add:
- SYS_PTRACE
security_opt:
- seccomp:unconfined

# see https://github.com/golang/go/wiki/LinuxKernelSignalVectorBug#what-to-do
ulimits:
memlock: 67108864

# ports:
# - ${PMM_PORT_HTTP:-8083}:8080
# - ${PMM_PORT_HTTPS:-8433}:8443
# For headless delve
# - ${PMM_PORT_DELVE:-12345}:2345
volumes:
- ./:/root/go/src/github.com/percona/pmm
# - "../grafana/public:/usr/share/grafana/public"
- ./Makefile.devcontainer:/root/go/src/github.com/percona/pmm/Makefile:ro # change Makefile in devcontainer
# caching
- go-modules:/root/go/pkg/mod
- root-cache:/root/.cache
- ./managed/testdata/pg/certs/:/certs/

volumes:
chdata: # Volume for ClickHouse data
vmdata: # Volume for VictoriaMetrics data
pgdata: # Volume for PostgreSQL data
go-modules:
root-cache:

networks:
ha:
ipam:
config:
- subnet: 172.20.0.0/24
Loading
Loading