From 394986a915f00f72aeb773e7f7db84fb73a09dee Mon Sep 17 00:00:00 2001 From: gitlawr Date: Wed, 28 Aug 2019 14:42:42 +0800 Subject: [PATCH] Adapt monitoring to restricted PSP clusters Problem: Fail to enable monitoring on restricted PSP clusters. Solution: 1. Update grafana to run as grafana user. 2. Update nginx sidecars to run as nginx user, move files to the user home directory(/var/cache/nginx) and listen on non privileged 8080 port. With 1 & 2, project monitoring works in restricted clusters out of the box. 3. Add necessary PSP/role/rb resource to node-exporter, which is used by cluster monitoring. --- .../charts/exporter-node/templates/psp.yaml | 44 +++++++++++++++++++ .../charts/exporter-node/templates/rbac.yaml | 29 ++++++++++++ .../v0.0.4/charts/exporter-node/values.yaml | 1 + .../charts/grafana/templates/deployment.yaml | 7 ++- .../grafana/templates/nginx-configmap.yaml | 7 ++- .../prometheus/templates/nginx-configmap.yaml | 13 +++--- .../prometheus/templates/prometheus.yaml | 8 ++-- 7 files changed, 94 insertions(+), 15 deletions(-) create mode 100644 charts/rancher-monitoring/v0.0.4/charts/exporter-node/templates/psp.yaml diff --git a/charts/rancher-monitoring/v0.0.4/charts/exporter-node/templates/psp.yaml b/charts/rancher-monitoring/v0.0.4/charts/exporter-node/templates/psp.yaml new file mode 100644 index 0000000..1c5f1b3 --- /dev/null +++ b/charts/rancher-monitoring/v0.0.4/charts/exporter-node/templates/psp.yaml @@ -0,0 +1,44 @@ +{{- if .Values.enabledPSP }} +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "app.fullname" . }} +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI + - persistentVolumeClaim + - hostPath + allowedHostPaths: + - pathPrefix: / + readOnly: true +{{- if .Values.enabledHostNetwork }} + hostNetwork: true + hostPorts: + - min: {{ .Values.ports.metrics.port }} + max: {{ .Values.ports.metrics.port }} +{{- end }} +{{- if .Values.enabledHostPID }} + hostPID: true +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/v0.0.4/charts/exporter-node/templates/rbac.yaml b/charts/rancher-monitoring/v0.0.4/charts/exporter-node/templates/rbac.yaml index 48f32c4..d93c037 100644 --- a/charts/rancher-monitoring/v0.0.4/charts/exporter-node/templates/rbac.yaml +++ b/charts/rancher-monitoring/v0.0.4/charts/exporter-node/templates/rbac.yaml @@ -53,4 +53,33 @@ subjects: - kind: ServiceAccount name: {{ template "app.fullname" . }} namespace: {{ .Release.Namespace }} +{{- if .Values.enabledPSP }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "app.fullname" . }}-psp-role +rules: +- apiGroups: + - extensions + resourceNames: + - {{ template "app.fullname" . }} + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "app.fullname" . }}-psp-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "app.fullname" . }}-psp-role +subjects: +- kind: ServiceAccount + name: {{ template "app.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- end }} {{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/v0.0.4/charts/exporter-node/values.yaml b/charts/rancher-monitoring/v0.0.4/charts/exporter-node/values.yaml index 970d0e8..79c3127 100755 --- a/charts/rancher-monitoring/v0.0.4/charts/exporter-node/values.yaml +++ b/charts/rancher-monitoring/v0.0.4/charts/exporter-node/values.yaml @@ -1,4 +1,5 @@ enabledRBAC: true +enabledPSP: true ## # Default collector settings diff --git a/charts/rancher-monitoring/v0.0.4/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/v0.0.4/charts/grafana/templates/deployment.yaml index 688ff05..94ec884 100755 --- a/charts/rancher-monitoring/v0.0.4/charts/grafana/templates/deployment.yaml +++ b/charts/rancher-monitoring/v0.0.4/charts/grafana/templates/deployment.yaml @@ -101,7 +101,7 @@ spec: - /nginx/nginx.conf ports: - name: http - containerPort: 80 + containerPort: 8080 protocol: TCP volumeMounts: - mountPath: /nginx/ @@ -109,6 +109,9 @@ spec: {{- if and .Values.resources .Values.resources.proxy }} resources: {{ toYaml .Values.resources.proxy | indent 10 }} + securityContext: + runAsUser: 100 + runAsGroup: 101 {{- end }} nodeSelector: {{- include "linux-node-selector" . | nindent 8 }} @@ -129,6 +132,8 @@ spec: {{ toYaml .Values.tolerations | indent 8 }} {{- end }} securityContext: + runAsUser: 472 + runAsGroup: 472 fsGroup: 472 volumes: - name: grafana-static-contents diff --git a/charts/rancher-monitoring/v0.0.4/charts/grafana/templates/nginx-configmap.yaml b/charts/rancher-monitoring/v0.0.4/charts/grafana/templates/nginx-configmap.yaml index ea76932..aa3228f 100644 --- a/charts/rancher-monitoring/v0.0.4/charts/grafana/templates/nginx-configmap.yaml +++ b/charts/rancher-monitoring/v0.0.4/charts/grafana/templates/nginx-configmap.yaml @@ -43,10 +43,9 @@ data: exit 1 nginx.conf: |- - user nginx; worker_processes auto; error_log /dev/stdout warn; - pid /var/run/nginx.pid; + pid /var/cache/nginx/nginx.pid; events { worker_connections 1024; @@ -60,10 +59,10 @@ data: proxy_read_timeout 180; proxy_send_timeout 5; proxy_buffering off; - proxy_cache_path /tmp/nginx levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; server { - listen 80; + listen 8080; access_log off; gzip on; diff --git a/charts/rancher-monitoring/v0.0.4/charts/prometheus/templates/nginx-configmap.yaml b/charts/rancher-monitoring/v0.0.4/charts/prometheus/templates/nginx-configmap.yaml index 0b3a672..842cee5 100644 --- a/charts/rancher-monitoring/v0.0.4/charts/prometheus/templates/nginx-configmap.yaml +++ b/charts/rancher-monitoring/v0.0.4/charts/prometheus/templates/nginx-configmap.yaml @@ -13,22 +13,23 @@ data: set -e srcpath="/nginx/nginx-conf.tmpl" - dstpath="/var/run/nginx.conf" + dstpath="/var/cache/nginx/nginx.conf" if ! [ -f $srcpath ]; then exit 1 fi + mkdir -p /var/cache/nginx/cache + token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) sed "s/REPLACE_PARAM_AUTHORIZATION/Bearer ${token}/g" $srcpath | sed "s/REPLACE_PARAM_IP/${POD_IP}/g" > $dstpath - exec nginx -g "daemon off;" -c /var/run/nginx.conf + exec nginx -g "daemon off;" -c /var/cache/nginx/nginx.conf nginx-conf.tmpl: |- - user nginx; worker_processes auto; error_log /dev/stdout warn; - pid /var/run/nginx.pid; + pid /var/cache/nginx/nginx.pid; events { worker_connections 1024; @@ -42,10 +43,10 @@ data: proxy_read_timeout 180; proxy_send_timeout 5; proxy_buffering off; - proxy_cache_path /tmp/nginx levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; server { - listen 80; + listen 8080; access_log off; gzip on; diff --git a/charts/rancher-monitoring/v0.0.4/charts/prometheus/templates/prometheus.yaml b/charts/rancher-monitoring/v0.0.4/charts/prometheus/templates/prometheus.yaml index ff43c19..c5e4bf2 100755 --- a/charts/rancher-monitoring/v0.0.4/charts/prometheus/templates/prometheus.yaml +++ b/charts/rancher-monitoring/v0.0.4/charts/prometheus/templates/prometheus.yaml @@ -17,7 +17,7 @@ spec: command: - /bin/sh - -c - - cp /nginx/run-sh.tmpl /var/run/nginx-start.sh; chmod +x /var/run/nginx-start.sh; /var/run/nginx-start.sh + - cp /nginx/run-sh.tmpl /var/cache/nginx/nginx-start.sh; chmod +x /var/cache/nginx/nginx-start.sh; /var/cache/nginx/nginx-start.sh env: - name: POD_IP valueFrom: @@ -25,7 +25,7 @@ spec: fieldPath: status.podIP image: {{ template "system_default_registry" . }}{{ .Values.image.proxy.repository }}:{{ .Values.image.proxy.tag }} ports: - - containerPort: 80 + - containerPort: 8080 name: http protocol: TCP {{- if and .Values.resources .Values.resources.proxy }} @@ -33,8 +33,8 @@ spec: {{ toYaml .Values.resources.proxy | indent 6 }} {{- end }} securityContext: - runAsNonRoot: false - runAsUser: 0 + runAsUser: 100 + runAsGroup: 101 volumeMounts: - mountPath: /nginx name: configmap-{{ template "app.nginx.fullname" . }} -- GitLab