一百八十二
再這樣下去我真的要變成 250 了,這怎么能忍,立馬打開 Google 研究了一把 Grafana 備份的各種騷操作,發(fā)現(xiàn)大部分備份方案都是通過 shell 腳本調(diào)用 Grafana 的 API 來導(dǎo)出各種配置。備份腳本大部分都集中在這個 gist 中:
https://gist.github.com/crisidev/bd52bdcc7f029be2f295
我挑選出幾個比較好用的,大家也可以自行挑選其他的。
導(dǎo)出腳本
#!/bin/bash
Usage:
export_grafana_dashboards.sh https://admin:REDACTED@grafana.dedevsecops.com
create_slug () {
echo "1"∣iconv?tascii//TRANSLIT∣sed?rs/[a?zA?Z0?9]+/?/g∣sed?rs/?+∥?+1" | iconv -t ascii//TRANSLIT | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+1"∣iconv?tascii//TRANSLIT∣sed?rs/[a?zA?Z0?9]+/?/g∣sed?rs/?+∥?+//g | tr A-Z a-z
}
full_url=1username=1 username=1username=(echo "fullurl"∣cut?d/?f3∣cut?d:?f1)baseurl={full_url}" | cut -d/ -f 3 | cut -d: -f 1) base_url=fullu?rl"∣cut?d/?f3∣cut?d:?f1)baseu?rl=(echo "fullurl"∣cut?d@?f2)folder={full_url}" | cut -d@ -f 2) folder=fullu?rl"∣cut?d@?f2)folder=(create_slug “username?{username}-username?{base_url}”)
mkdir “${folder}”
for db_uid in (curl?s"(curl -s "(curl?s"{full_url}/api/search" | jq -r .[].uid); do
db_json=(curl?s"(curl -s "(curl?s"{full_url}/api/dashboards/uid/dbuid")dbslug={db_uid}") db_slug=dbu?id")dbs?lug=(echo "dbjson"∣jq?r.meta.slug)dbtitle={db_json}" | jq -r .meta.slug) db_title=dbj?son"∣jq?r.meta.slug)dbt?itle=(echo “dbjson"∣jq?r.dashboard.title)filename="{db_json}" | jq -r .dashboard.title) filename="dbj?son"∣jq?r.dashboard.title)filename="{folder}/KaTeX parse error: Can't use function '\"' in math mode at position 35: …cho "Exporting \?"?{db_title}” to “KaTeX parse error: Can't use function '\"' in math mode at position 11: {filename}\?"?..." echo "{db_json}” | jq -r . > “${filename}”
done
echo “Done”
這個腳本比較簡單,直接導(dǎo)出了所有 Dashboard 的 json 配置,也沒有標(biāo)記目錄信息,如果你用它導(dǎo)出的配置來恢復(fù) Grafana,所有的 Dashboard 都會導(dǎo)入到 Grafana 的 General 目錄下,不太友好。
導(dǎo)入腳本
grafana-dashboard-importer.sh
#!/bin/bash
add the “-x” option to the shebang line if you want a more verbose output
OPTSPEC=":hp:t:k:"
show_help() {
cat << EOF
Usage: $0 [-p PATH] [-t TARGET_HOST] [-k API_KEY]
Script to import dashboards into Grafana
-p Required. Root path containing JSON exports of the dashboards you want imported.
-t Required. The full URL of the target host
-k Required. The API key to use on the target host
EOF
}
Check script invocation options
while getopts “OPTSPEC"optchar;docase"OPTSPEC" optchar; do case "OPTSPEC"optchar;docase"optchar” in
h)
show_help
exit
;;
p)
DASH_DIR=“OPTARG";;t)HOST="OPTARG";; t) HOST="OPTARG";;t)HOST="OPTARG”;;
k)
KEY=“KaTeX parse error: Undefined control sequence: \? at position 19: …ARG";; \???) ech…OPTARG” >&2
exit 1
;;
😃
echo “Option -$OPTARG requires an argument.” >&2
exit 1
;;
esac
done
if [ -z “DASHDIR"]∣∣[?z"DASH_DIR" ] || [ -z "DASHD?IR"]∣∣[?z"HOST” ] || [ -z “$KEY” ]; then
show_help
exit 1
fi
set some colors for status OK, FAIL and titles
SETCOLOR_SUCCESS=“echo -en \033[0;32m”
SETCOLOR_FAILURE=“echo -en \033[1;31m”
SETCOLOR_NORMAL=“echo -en \033[0;39m”
SETCOLOR_TITLE_PURPLE=“echo -en \033[0;35m” # purple
usage log “string to log” “color option”
function log_success() {
if [ $# -lt 1 ]; then
${SETCOLOR_FAILURE}
echo “Not enough arguments for log function! Expecting 1 argument got $#”
exit 1
fi
timestamp=$(date “+%Y-%m-%d %H:%M:%S %Z”)
${SETCOLOR_SUCCESS}
printf “[%s] KaTeX parse error: Undefined control sequence: \n at position 2: 1\?n?" "timestamp”
${SETCOLOR_NORMAL}
}
function log_failure() {
if [ $# -lt 1 ]; then
${SETCOLOR_FAILURE}
echo “Not enough arguments for log function! Expecting 1 argument got $#”
exit 1
fi
timestamp=$(date “+%Y-%m-%d %H:%M:%S %Z”)
${SETCOLOR_FAILURE}
printf “[%s] KaTeX parse error: Undefined control sequence: \n at position 2: 1\?n?" "timestamp”
${SETCOLOR_NORMAL}
}
function log_title() {
if [ $# -lt 1 ]; then
${SETCOLOR_FAILURE}
log_failure “Not enough arguments for log function! Expecting 1 argument got $#”
exit 1
fi
${SETCOLOR_TITLE_PURPLE}
printf “|-------------------------------------------------------------------------|\n”
printf “|%s|\n” “$1”;
printf “|-------------------------------------------------------------------------|\n”
${SETCOLOR_NORMAL}
}
if [ -d "DASHDIR"];thenDASHLIST=DASH_DIR" ]; then DASH_LIST=DASHD?IR"];thenDASHL?IST=(find “KaTeX parse error: Undefined control sequence: \* at position 29: …ndepth 1 -name \?*?.json) if […DASH_LIST” ]; then
log_title “----------------- $DASH_DIR contains no JSON files! -----------------”
log_failure "Directory DASHDIRdoesnotappeartocontainanyJSONfilesforimport.Checkyourpathandtryagain."exit1elseFILESTOTAL=DASH_DIR does not appear to contain any JSON files for import. Check your path and try again." exit 1 else FILESTOTAL=DASHD?IRdoesnotappeartocontainanyJSONfilesforimport.Checkyourpathandtryagain."exit1elseFILESTOTAL=(echo “$DASH_LIST” | wc -l)
log_title “----------------- Starting import of $FILESTOTAL dashboards -----------------”
fi
else
log_title “----------------- $DASH_DIR directory not found! -----------------”
log_failure “Directory $DASH_DIR does not exist. Check your path and try again.”
exit 1
fi
NUMSUCCESS=0
NUMFAILURE=0
COUNTER=0
for DASH_FILE in DASHLIST;doCOUNTER=DASH_LIST; do COUNTER=DASHL?IST;doCOUNTER=((COUNTER + 1))
echo “Import COUNTER/COUNTER/COUNTER/FILESTOTAL: DASHFILE..."RESULT=DASH_FILE..." RESULT=DASHF?ILE..."RESULT=(cat “$DASH_FILE” | jq ‘. * {overwrite: true, dashboard: {id: null}}’ | curl -s -X POST -H “Content-Type: application/json” -H “Authorization: Bearer KEY""KEY" "KEY""HOST”/api/dashboards/db -d @-)
if [[ “RESULT"==?"success"?]];thenlogsuccess"RESULT" == *"success"* ]]; then log_success "RESULT"==?"success"?]];thenlogs?uccess"RESULT”
NUMSUCCESS=((NUMSUCCESS+1))elselogfailure"((NUMSUCCESS + 1)) else log_failure "((NUMSUCCESS+1))elselogf?ailure"RESULT”
NUMFAILURE=$((NUMFAILURE + 1))
fi
done
log_title “Import complete. $NUMSUCCESS dashboards were successfully imported. $NUMFAILURE dashboard imports failed.”;
log_title “------------------------------ FINISHED ---------------------------------”;
導(dǎo)入腳本需要目標(biāo)機器上的 Grafana 已經(jīng)啟動,而且需要提供管理員 API Key。登錄 Grafana Web 界面,打開 API Keys:
新建一個 API Key,角色選擇 Admin,過期時間自己調(diào)整:
導(dǎo)入方式:
$ ./grafana-dashboard-importer.sh -t http://<grafana_svc_ip>:<grafana_svc_port> -k <api_key> -p
其中 -p 參數(shù)指定的是之前導(dǎo)出的 json 所在的目錄。
目前的方案痛點在于只能備份 Dashboard,不能備份其他的配置(例如,數(shù)據(jù)源、用戶、秘鑰等),而且沒有將 Dashboard 和目錄對應(yīng)起來,即不支持備份 Folder。下面介紹一個比較完美的備份恢復(fù)方案,支持所有配置的備份恢復(fù),簡直不要太香。
更高級的方案已經(jīng)有人寫好了,項目地址是:
https://github.com/ysde/grafana-backup-tool
該備份工具支持以下幾種配置:
目錄
Dashboard
數(shù)據(jù)源
Grafana 告警頻道(Alert Channel)
組織(Organization)
用戶(User)
使用方法很簡單,跑個容器就好了嘛,不過作者提供的 Dockerfile 我不是很滿意,自己修改了點內(nèi)容:
FROM alpine:latest
LABEL maintainer=“grafana-backup-tool Docker Maintainers https://fuckcloudnative.io”
ENV ARCHIVE_FILE “”
RUN echo “@edge http://dl-cdn.alpinelinux.org/alpine/edge/community” >> /etc/apk/repositories;
apk --no-cache add python3 py3-pip py3-cffi py3-cryptography ca-certificates bash git;
git clone https://github.com/ysde/grafana-backup-tool /opt/grafana-backup-tool;
cd /opt/grafana-backup-tool;
pip3 --no-cache-dir install .;
chown -R 1337:1337 /opt/grafana-backup-tool
WORKDIR /opt/grafana-backup-tool
USER 1337
只有 Dockerfile 不行,還得通過 CI/CD 自動構(gòu)建并推送到 docker.io。不要問我用什么,當(dāng)然是白嫖 GitHub Action,workflow 內(nèi)容如下:
#=================================================
https://github.com/yangchuansheng/docker-image
Description: Build and push grafana-backup-tool Docker image
Lisence: MIT
Author: Ryan
Blog: https://fuckcloudnative.io
#=================================================
name: Build and push grafana-backup-tool Docker image
Controls when the action will run. Triggers the workflow on push or pull request
events but only for the master branch
on:
push:
branches: [ master ]
paths:
- ‘grafana-backup-tool/Dockerfile’
- ‘.github/workflows/grafana-backup-tool.yml’
pull_request:
branches: [ master ]
paths:
- ‘grafana-backup-tool/Dockerfile’
#watch:
#types: started
A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
This workflow contains a single job called “build”
build:
# The type of runner that the job will run on
runs-on: ubuntu-latest
這里我不打算解釋 workflow 的內(nèi)容,有點基礎(chǔ)的應(yīng)該都能看懂,實在不行,以后我會單獨寫文章解釋(又可以繼續(xù)水文了~)。這個 workflow 實現(xiàn)的功能就是自動構(gòu)建各個 CPU 架構(gòu)的鏡像,并推送到 docker.io 和 ghcr.io,特么的真香!
就問爽不爽?
你可以直接關(guān)注我的倉庫:
https://github.com/yangchuansheng/docker-image
構(gòu)建好鏡像后,就可以直接運行容器來進行備份和恢復(fù)操作了。如果你想在集群內(nèi)操作,可以通過 Deployment 或 Job 來實現(xiàn);如果你想在本地或 k8s 集群外操作,可以選擇 docker run,我不反對,你也可以選擇 docker-compose,這都沒問題。但我要告訴你一個更騷的辦法,可以騷到讓你無法自拔。
首先需要在本地或集群外安裝 Podman,如果操作系統(tǒng)是 Win10,可以考慮通過 WSL 來安裝;如果操作系統(tǒng)是 Linux,那就不用說了;如果操作系統(tǒng)是 MacOS,請參考我的上篇文章:在 macOS 中使用 Podman。
裝好了 Podman 之后,就可以進行騷操作了,請睜大眼睛。
先編寫一個 Deployment 配置清單(什么?Deployment?是的,你沒聽錯):
grafana-backup-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: grafana-backup
labels:
app: grafana-backup
spec:
replicas: 1
selector:
matchLabels:
app: grafana-backup
template:
metadata:
labels:
app: grafana-backup
spec:
containers:
- name: grafana-backup
image: yangchuansheng/grafana-backup-tool:latest
imagePullPolicy: IfNotPresent
command: ["/bin/bash"]
tty: true
stdin: true
env:
- name: GRAFANA_TOKEN
value: “eyJr0NkFBeWV1QVpMNjNYWXA3UXNOM2JWMWdZOTB2ZFoiLCJuIjoiYWRtaW4iLCJpZCI6MX0=”
- name: GRAFANA_URL
value: “http://<grafana_ip>:<grafana_port>”
- name: GRAFANA_ADMIN_ACCOUNT
value: “admin”
- name: GRAFANA_ADMIN_PASSWORD
value: “admin”
- name: VERIFY_SSL
value: “False”
volumeMounts:
- mountPath: /opt/grafana-backup-tool
name: data
volumes:
- name: data
hostPath:
path: /mnt/manifest/grafana/backup
這里面的環(huán)境變量根據(jù)自己的實際情況修改,一定不要照抄我的!
不要一臉懵逼,我先來解釋一下為什么要準(zhǔn)備這個 Deployment 配置清單,因為 Podman 可以直接通過這個配置清單運行容器,命令如下:
$ podman play kube grafana-backup-deployment.yaml
我第一次見到這個操作的時候也不禁連連我艸,這也可以?確實可以,不過呢,Podman 只是將其翻譯一下,跑個容器而已,并不是真正運行 Deployment,因為它沒有控制器啊,但是,還是真香!
想象一下,你可以將 k8s 集群中的配置清單拿到本地或測試機器直接跑,再也不用 k8s 集群準(zhǔn)備一份 yaml,docker-compose 再準(zhǔn)備一份 yaml 了,一份 yaml 走天下,服不服?
docker-compose 混到今天這個地步,也是蠻可憐的。
細(xì)心的讀者應(yīng)該能發(fā)現(xiàn)上面的配置清單有點奇怪,Dockerfile 也有點奇怪。Dockerfile 中沒有寫 CMD 或 ENTRYPOINT,Deployment 中直接將啟動命令設(shè)置為 bash,這是因為在我之前測試的過程中發(fā)現(xiàn)該鏡像啟動的容器有點問題,它會陷入一個循環(huán),備份完了之后又會繼續(xù)備份,不斷重復(fù),導(dǎo)致備份目錄下生成了一坨壓縮包。目前還沒找到比較好的解決辦法,只能將容器的啟動命令設(shè)置為 bash,等容器運行后再進入容器進行備份操作:
$ podman pod ls
POD ID NAME STATUS CREATED # OF CONTAINERS INFRA ID
728aec216d66 grafana-backup-pod-0 Running 3 minutes ago 2 92aa0824fe7d
$ podman ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
b523fa8e4819 yangchuansheng/grafana-backup-tool:latest /bin/bash 3 minutes ago Up 3 minutes ago grafana-backup-pod-0-grafana-backup
92aa0824fe7d k8s.gcr.io/pause:3.2 3 minutes ago Up 3 minutes ago 728aec216d66-infra
$ podman exec -it grafana-backup-pod-0-grafana-backup bash
bash-5.0$ grafana-backup save
…
…
########################################
backup folders at: OUTPUT/folders/202012111556
backup datasources at: OUTPUT/datasources/202012111556
backup dashboards at: OUTPUT/dashboards/202012111556
backup alert_channels at: OUTPUT/alert_channels/202012111556
backup organizations at: OUTPUT/organizations/202012111556
backup users at: OUTPUT/users/202012111556
created archive at: OUTPUT/202012111556.tar.gz
默認(rèn)情況下會備份所有的組件,你也可以指定備份的組件:
$ grafana-backup save --components=<folders,dashboards,datasources,alert-channels,organizations,users>
比如,我只想備份 Dashboards 和 Folders:
$ grafana-backup save --components=folders,dashboards
當(dāng)然,你也可以全部備份,恢復(fù)的時候再選擇自己想恢復(fù)的組件:
$ grafana-backup restore --components=folders,dashboards
至此,再也不用怕 Dashboard 被改掉或刪除啦。
最后提醒一下,Prometheus Operator 項目中的 Grafana 通過 Provisioning 的方式預(yù)導(dǎo)入了一些默認(rèn)的 Dashboards,這本來沒有什么問題,但 grafana-backup-tool 工具無法忽略跳過已經(jīng)存在的配置,如果恢復(fù)的過程中遇到已經(jīng)存在的配置,會直接報錯退出。本來這也很好解決,一般情況下到 Grafana Web 界面中刪除所有的 Dashboard 就好了,但通過 Provisioning 導(dǎo)入的 Dashboard 是無法刪除的,這就很尷尬了。
在作者修復(fù)這個 bug 之前,要想解決這個問題,有兩個辦法:
第一個辦法是在恢復(fù)之前將 Grafana Deployment 中關(guān)于 Provisioning 的配置全部刪除,就是這些配置:
volumeMounts:- mountPath: /etc/grafana/provisioning/datasourcesname: grafana-datasourcesreadOnly: false- mountPath: /etc/grafana/provisioning/dashboardsname: grafana-dashboardsreadOnly: false- mountPath: /grafana-dashboard-definitions/0/apiservername: grafana-dashboard-apiserverreadOnly: false- mountPath: /grafana-dashboard-definitions/0/cluster-totalname: grafana-dashboard-cluster-totalreadOnly: false- mountPath: /grafana-dashboard-definitions/0/controller-managername: grafana-dashboard-controller-managerreadOnly: false- mountPath: /grafana-dashboard-definitions/0/k8s-resources-clustername: grafana-dashboard-k8s-resources-clusterreadOnly: false- mountPath: /grafana-dashboard-definitions/0/k8s-resources-namespacename: grafana-dashboard-k8s-resources-namespacereadOnly: false- mountPath: /grafana-dashboard-definitions/0/k8s-resources-nodename: grafana-dashboard-k8s-resources-nodereadOnly: false- mountPath: /grafana-dashboard-definitions/0/k8s-resources-podname: grafana-dashboard-k8s-resources-podreadOnly: false- mountPath: /grafana-dashboard-definitions/0/k8s-resources-workloadname: grafana-dashboard-k8s-resources-workloadreadOnly: false- mountPath: /grafana-dashboard-definitions/0/k8s-resources-workloads-namespacename: grafana-dashboard-k8s-resources-workloads-namespacereadOnly: false- mountPath: /grafana-dashboard-definitions/0/kubeletname: grafana-dashboard-kubeletreadOnly: false- mountPath: /grafana-dashboard-definitions/0/namespace-by-podname: grafana-dashboard-namespace-by-podreadOnly: false- mountPath: /grafana-dashboard-definitions/0/namespace-by-workloadname: grafana-dashboard-namespace-by-workloadreadOnly: false- mountPath: /grafana-dashboard-definitions/0/node-cluster-rsrc-usename: grafana-dashboard-node-cluster-rsrc-usereadOnly: false- mountPath: /grafana-dashboard-definitions/0/node-rsrc-usename: grafana-dashboard-node-rsrc-usereadOnly: false- mountPath: /grafana-dashboard-definitions/0/nodesname: grafana-dashboard-nodesreadOnly: false- mountPath: /grafana-dashboard-definitions/0/persistentvolumesusagename: grafana-dashboard-persistentvolumesusagereadOnly: false- mountPath: /grafana-dashboard-definitions/0/pod-totalname: grafana-dashboard-pod-totalreadOnly: false- mountPath: /grafana-dashboard-definitions/0/prometheus-remote-writename: grafana-dashboard-prometheus-remote-writereadOnly: false- mountPath: /grafana-dashboard-definitions/0/prometheusname: grafana-dashboard-prometheusreadOnly: false- mountPath: /grafana-dashboard-definitions/0/proxyname: grafana-dashboard-proxyreadOnly: false- mountPath: /grafana-dashboard-definitions/0/schedulername: grafana-dashboard-schedulerreadOnly: false- mountPath: /grafana-dashboard-definitions/0/statefulsetname: grafana-dashboard-statefulsetreadOnly: false- mountPath: /grafana-dashboard-definitions/0/workload-totalname: grafana-dashboard-workload-totalreadOnly: false…
…
volumes:
- name: grafana-datasources
secret:
secretName: grafana-datasources
- configMap:
name: grafana-dashboards
name: grafana-dashboards
- configMap:
name: grafana-dashboard-apiserver
name: grafana-dashboard-apiserver
- configMap:
name: grafana-dashboard-cluster-total
name: grafana-dashboard-cluster-total
- configMap:
name: grafana-dashboard-controller-manager
name: grafana-dashboard-controller-manager
- configMap:
name: grafana-dashboard-k8s-resources-cluster
name: grafana-dashboard-k8s-resources-cluster
- configMap:
name: grafana-dashboard-k8s-resources-namespace
name: grafana-dashboard-k8s-resources-namespace
- configMap:
name: grafana-dashboard-k8s-resources-node
name: grafana-dashboard-k8s-resources-node
- configMap:
name: grafana-dashboard-k8s-resources-pod
name: grafana-dashboard-k8s-resources-pod
- configMap:
name: grafana-dashboard-k8s-resources-workload
name: grafana-dashboard-k8s-resources-workload
- configMap:
name: grafana-dashboard-k8s-resources-workloads-namespace
name: grafana-dashboard-k8s-resources-workloads-namespace
- configMap:
name: grafana-dashboard-kubelet
name: grafana-dashboard-kubelet
- configMap:
name: grafana-dashboard-namespace-by-pod
name: grafana-dashboard-namespace-by-pod
- configMap:
name: grafana-dashboard-namespace-by-workload
name: grafana-dashboard-namespace-by-workload
- configMap:
name: grafana-dashboard-node-cluster-rsrc-use
name: grafana-dashboard-node-cluster-rsrc-use
- configMap:
name: grafana-dashboard-node-rsrc-use
name: grafana-dashboard-node-rsrc-use
- configMap:
name: grafana-dashboard-nodes
name: grafana-dashboard-nodes
- configMap:
name: grafana-dashboard-persistentvolumesusage
name: grafana-dashboard-persistentvolumesusage
- configMap:
name: grafana-dashboard-pod-total
name: grafana-dashboard-pod-total
- configMap:
name: grafana-dashboard-prometheus-remote-write
name: grafana-dashboard-prometheus-remote-write
- configMap:
name: grafana-dashboard-prometheus
name: grafana-dashboard-prometheus
- configMap:
name: grafana-dashboard-proxy
name: grafana-dashboard-proxy
- configMap:
name: grafana-dashboard-scheduler
name: grafana-dashboard-scheduler
- configMap:
name: grafana-dashboard-statefulset
name: grafana-dashboard-statefulset
- configMap:
name: grafana-dashboard-workload-total
name: grafana-dashboard-workload-total
第二個辦法就是刪除 Prometheus Operator 自帶的 Grafana,自己通過 Helm 或者 manifest 部署不使用 Provisioning 的 Grafana。
如果你既不想刪除 Provisioning 的配置,也不想自己部署 Grafana,那只能使用上文提到的低級方案了。
Kubernetes 1.18.2 1.17.5 1.16.9 1.15.12離線安裝包發(fā)布地址http://store.lameleg.com ,歡迎體驗。 使用了最新的sealos v3.3.6版本。 作了主機名解析配置優(yōu)化,lvscare 掛載/lib/module解決開機啟動ipvs加載問題, 修復(fù)lvscare社區(qū)netlink與3.10內(nèi)核不兼容問題,sealos生成百年證書等特性。更多特性 https://github.com/fanux/sealos 。歡迎掃描下方的二維碼加入釘釘群 ,釘釘群已經(jīng)集成sealos的機器人實時可以看到sealos的動態(tài)。
總結(jié)
- 上一篇: GeForce Experience界面
- 下一篇: 【免费分享】让思路更清晰,思维导图教程及