diff --git a/prod-upgrade/prom/grafana.yaml b/prod-upgrade/prom/grafana.yaml new file mode 100644 index 0000000..46d81fd --- /dev/null +++ b/prod-upgrade/prom/grafana.yaml @@ -0,0 +1,82 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: grafana + name: grafana +spec: + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + securityContext: + fsGroup: 472 + supplementalGroups: + - 0 + containers: + - name: grafana + image: grafana/grafana:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + name: http-grafana + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /robots.txt + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: 3000 + timeoutSeconds: 1 + resources: + requests: + cpu: 250m + memory: 750Mi + volumeMounts: + - mountPath: /var/lib/grafana + name: grafana-pv + volumes: + - name: grafana-pv + persistentVolumeClaim: + claimName: grafana-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana +spec: + ports: + - port: 3000 + protocol: TCP + targetPort: http-grafana + selector: + app: grafana + sessionAffinity: None + type: ClusterIP \ No newline at end of file diff --git a/prod-upgrade/prom/mysql-exporter.yaml b/prod-upgrade/prom/mysqld-exporter.yaml similarity index 100% rename from prod-upgrade/prom/mysql-exporter.yaml rename to prod-upgrade/prom/mysqld-exporter.yaml diff --git a/prod-upgrade/prom/prometheus.yaml b/prod-upgrade/prom/prometheus.yaml new file mode 100644 index 0000000..58b465f --- /dev/null +++ b/prod-upgrade/prom/prometheus.yaml @@ -0,0 +1,120 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prom-config +data: + prom-config: |- + global: + scrape_interval: 15s + + scrape_configs: + - job_name: mysql # To get metrics about the mysql exporter’s targets + params: + auth_module: [client] + static_configs: + - targets: + # All mysql hostnames or unix sockets to monitor. + - mysql-0.base.svc:3306 + - mysql-1.base.svc:3306 + - mysql-2.base.svc:3306 + - mysql-3.base.svc:3306 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: mysqld-exporter.base.svc:9104 + rule_files: + - /etc/prometheus/rules/rule-0.yaml + alerting: + alert_relabel_configs: + alertmanagers: + prom-rule-0: |- + groups: + - name: mysqld_rules + rules: + + # Record slave lag seconds for pre-computed timeseries that takes + # `mysql_slave_status_sql_delay` into account + - record: instance:mysql_slave_lag_seconds + expr: mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay + + # Record slave lag via heartbeat method + - record: instance:mysql_heartbeat_lag_seconds + expr: mysql_heartbeat_now_timestamp_seconds - mysql_heartbeat_stored_timestamp_seconds + + - record: job:mysql_transactions:rate5m + expr: sum without (command) (rate(mysql_global_status_commands_total{command=~"(commit|rollback)"}[5m])) + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: prometheus +spec: + replicas: 1 + serviceName: mysql + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + containers: + - name: prometheus + image: prom/prometheus:v2.51.2 + args: + - --config.file=/etc/prometheus/prometheus.yaml + - --storage.tsdb.path=/etc/prometheus/data + - --storage.tsdb.retention.time=15d + volumeMounts: + - name: prometheus + mountPath: /etc/prometheus/data + - name: prom-config + mountPath: /etc/prometheus/prometheus.yaml + subPath: prometheus.yaml + readOnly: true + - name: prom-config + mountPath: /etc/prometheus/rules/rule-0.yaml + subPath: rule-0.yaml + readOnly: true + volumes: + - name: prometheus + hostPath: + path: /data/prometheus/data + - name: prom-config + configMap: + name: prom-config + items: + - key: prom-config + path: prometheus.yaml + - key: prom-rule-0 + path: rule-0.yaml + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "kubernetes.io/hostname" + operator: In + values: + - bfs-k8snode-10-2-2-7.hetzner.base.beaconfireinc.com + +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + labels: + app: prometheus +spec: + ports: + - protocol: TCP + port: 9090 + targetPort: 9090 + selector: + app: prometheus + type: ClusterIP \ No newline at end of file