From fc4dc65b0fc0978e8c0caddc7d04e86281e5d72a Mon Sep 17 00:00:00 2001 From: ycz008 Date: Wed, 6 Dec 2023 17:26:05 +0800 Subject: [PATCH] add tcs log elk --- tcs-upgrade/elastic/filebeat.yaml | 203 ++++++++++++++++++++++++++++++ tcs-upgrade/elastic/kibana.yaml | 66 ++++++++++ tcs-upgrade/elastic/logstash.yaml | 97 ++++++++++++++ 3 files changed, 366 insertions(+) create mode 100644 tcs-upgrade/elastic/filebeat.yaml create mode 100644 tcs-upgrade/elastic/kibana.yaml create mode 100644 tcs-upgrade/elastic/logstash.yaml diff --git a/tcs-upgrade/elastic/filebeat.yaml b/tcs-upgrade/elastic/filebeat.yaml new file mode 100644 index 0000000..3fe4a86 --- /dev/null +++ b/tcs-upgrade/elastic/filebeat.yaml @@ -0,0 +1,203 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat +data: + filebeat.yml: |- + filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + + # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: + filebeat.autodiscover: + providers: + - type: kubernetes + node: ${NODE_NAME} + hints.enabled: true + # hints.default_config: + # type: container + # paths: + # - /var/log/containers/*${data.kubernetes.container.id}.log + + processors: + - add_cloud_metadata: ~ + + filebeat.inputs: + - type: log + paths: + - /data/app_logs/*/*/logstash/*.log + json.keys_under_root: true + tags: ['json'] + + #All data to indexed to Elasticsearch + output.logstash: + hosts: ["logstash.tcs.svc:5044"] + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + selector: + matchLabels: + k8s-app: filebeat + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:7.16.3 + args: [ + "-c", "/etc/filebeat.yml", + "-e", + ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: data + mountPath: /usr/share/filebeat/data + - name: app-logs + mountPath: /data/app_logs + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0640 + name: filebeat-config + - name: app-logs + hostPath: + path: /data/app_logs + - name: data + hostPath: + # When filebeat runs as non-root user, this directory needs to be writable by group (g+w). + path: /var/lib/filebeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + - nodes + verbs: + - get + - watch + - list +- apiGroups: ["apps"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat + # should be the namespace where filebeat is running + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat \ No newline at end of file diff --git a/tcs-upgrade/elastic/kibana.yaml b/tcs-upgrade/elastic/kibana.yaml new file mode 100644 index 0000000..6894e6f --- /dev/null +++ b/tcs-upgrade/elastic/kibana.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kibana-config +data: + kibana.yml: |- + server.name: kibana + server.host: 0.0.0.0 + elasticsearch.hosts: [ "http://elastic:9200" ] + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + labels: + app: kibana +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:7.16.3 + ports: + - containerPort: 5601 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1 + memory: 1Gi + volumeMounts: + - name: config + mountPath: /usr/share/kibana/config/kibana.yml + subPath: kibana.yml + readOnly: true + volumes: + - name: config + configMap: + name: kibana-config + items: + - key: kibana.yml + path: kibana.yml + +--- +apiVersion: v1 +kind: Service +metadata: + name: kibana + labels: + app: kibana +spec: + ports: + - port: 5601 + targetPort: 5601 + selector: + app: kibana + type: ClusterIP \ No newline at end of file diff --git a/tcs-upgrade/elastic/logstash.yaml b/tcs-upgrade/elastic/logstash.yaml new file mode 100644 index 0000000..9189cd5 --- /dev/null +++ b/tcs-upgrade/elastic/logstash.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: logstash-config +data: + logstash.conf: |- + input { + beats { + port => 5044 + } + } + + filter { + if [tags][json] { + json { + source => "message" + } + } + mutate { + split => { "[log][file][path]" => "/" } + add_field => { "env" => "%{[log][file][path][3]}" } + } + } + + output { + stdout { codec => rubydebug } + elasticsearch { + hosts => ["http://elastic:9200"] + index => "%{[env]}-logback-beaconfire-%{+YYYY.MM.dd}" + } + } + logstash.yml: |- + http.host: "0.0.0.0" + xpack.monitoring.elasticsearch.hosts: [ "http://elastic:9200" ] + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: logstash + labels: + app: logstash +spec: + replicas: 1 + selector: + matchLabels: + app: logstash + template: + metadata: + labels: + app: logstash + spec: + containers: + - name: logstash + image: docker.elastic.co/logstash/logstash:7.16.3 + ports: + - containerPort: 5044 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1 + memory: 1Gi + volumeMounts: + - name: config + mountPath: /usr/share/logstash/pipeline/logstash.conf + subPath: logstash.conf + readOnly: true + - name: config + mountPath: /usr/share/logstash/config/logstash.yml + subPath: logstash.yml + readOnly: true + volumes: + - name: config + configMap: + name: logstash-config + items: + - key: logstash.conf + path: logstash.conf + - key: logstash.yml + path: logstash.yml + +--- +apiVersion: v1 +kind: Service +metadata: + name: logstash + labels: + app: logstash +spec: + ports: + - port: 5044 + targetPort: 5044 + selector: + app: logstash + type: ClusterIP \ No newline at end of file