灾备方案

数据备份

数据源	备份方案	备份周期
PVC	暂停velero或者手动脚本	未定
mysql	mysql&&xtrabackup	每天全备+小时增备
etcd	定时脚本	每小时
bootstrap	nfs定时备份	每天

mysql备份

逻辑备份

#!/bin/bash
# 数据库用户名
db_user=root
# 数据库密码
db_password=masterpwd
# 备份存放路径
backup_dir=/mnt/nfs/mysql_bakfile
# mysql地址
mysql_host=10.6.31.20
# 备份命名所使用的日期格式
date=$(date +%Y%m%d)
# 对备份进行压缩：
mysqldump -u$db_user -p$db_password -h$mysql_host -P30016 --all-databases --single-transaction --master-data=2 | gzip >$backup_dir/mysql_jd_$date.sql.gz
# 清理七天前备份数据
find $backup_dir -name "*.sql.gz" -type f -mtime +7 -exec rm -rf {} \; > /dev/null 2>&1
 
 
crontab
3 0 * * * /bin/bash /opt/mysql_jd_bak.sh  > /dev/null 2>&1

物理备份

docker load < xtrabuckup-k8s.tar
mkdir /mysqlbak
docker run -tid --name xtrabackup --privileged=true -v /mnt/mysql/data:/bitnami/mysql/data -v /mysqlbak:/mysqlbak xtrabuckup-k8s
 
# 定时任务
nohup docker exec xtrabackup /bin/bash -c "innobackupex --defaults-file=/etc/my.cnf --user=bak --password=Linktime2023 -H10.88.56.135 /mysqlbak/" &
 
# mysql用户创建（cli）
CREATE USER backup@'%' IDENTIFIED BY 'Linktime2023';
grant process,lock tables,reload,replication client,super on *.* to backup@'%';
flush privileges;

容器化xtrabackup

导入xtrabuckup镜像

docker load < xtrabuckup.tar

运行xtrabuckup容器

# 在备份数据库节点运行xtrabuckup容器（切记需要将准备备份的数据库数据目录挂载到容器的/bitnami/mysql/data目录下）
docker run -tid --name xtrabackup -v /data/mysql:/bitnami/mysql/data xtrabuckup:2.4.8
# 这里以/data/mysql目录为备份库数据目录为例

全量备份

[root@2e70e90f864b /]innobackupex --defaults-file=/etc/my.cnf --user=root --password=root -H192.168.100.90 /data/basedir/
[root@2e70e90f864b /]# ls /data/basedir/
2021-12-31_02-47-48 2021-12-31_03-29-31（当前日期为标识的备份目录）

全量恢复

全量恢复前一定要关闭备份数据库服务并且删除数据文件及事务日志文件，否则会报错。

[root@host10~] # innobackupex --defaults-file=/etc/my.cnf --apply-log /data/basedir/2021-12-31_02-47-48
# 使用此参数使用相关数据性文件保持一致性状态
 
# 下面是恢复数据文件到数据库
[root@host10~] # innobackupex --defaults-file=/etc/my.cnf --copy-back /data/basedir/2021-12-31_02-47-48
数据恢复完成后把数据文件改回原来的权限并重启mysql

定时全量备份

编写系统定时脚本，周期性全量备份mysql数据库，然后上传备份数据到oss对象存储中。

#!/bin/bash  
docker run -tid --name xtrabackup -v /data/mysql:/var/lib/mysql -v /mysqlbak:/mysqlbak xtrabuckup:2.4.8  
 
docker exec -ti xtrabackup /bin/bash -c 'innobackupex --defaults-file=/etc/my.cnf -H192.168.100.90 -uroot -proot /mysqlbak/basedir/'  
 
docker rm -f xtrabackup  
 
if [ $? -eq 0 ]; then
  echo "scp database backup upload OSS"
  cd /mnt/
  ossutil64 cp -r /mysqlbak/basedir/ oss://linktime-it/BACKUP/ --update
fi

Grafana面板备份

首先获取Grafana的api_key：menu–configuration–api keys – add API key

导出脚本grafana-dashboard-exporter.sh

#/bin/bash
#
# add the "-x" option to the shebang line if you want a more verbose output
#
# set some colors for status OK, FAIL and titles
SETCOLOR_SUCCESS="echo -en \\033[0;32m"
SETCOLOR_FAILURE="echo -en \\033[1;31m"
SETCOLOR_NORMAL="echo -en \\033[0;39m"
SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple
 
# usage log "string to log" "color option"
function log_success() {
   if [ $# -lt 1 ]; then
       ${SETCOLOR_FAILURE}
       echo "Not enough arguments for log function! Expecting 1 argument got $#"
       exit 1
   fi
 
   timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z")
 
   ${SETCOLOR_SUCCESS}
   printf "[${timestamp}] $1\n"
   ${SETCOLOR_NORMAL}
}
 
function log_failure() {
   if [ $# -lt 1 ]; then
       ${SETCOLOR_FAILURE}
       echo "Not enough arguments for log function! Expecting 1 argument got $#"
       exit 1
   fi
 
   timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z")
 
   ${SETCOLOR_FAILURE}
   printf "[${timestamp}] $1\n"
   ${SETCOLOR_NORMAL}
}
 
function log_title() {
   if [ $# -lt 1 ]; then
       ${SETCOLOR_FAILURE}
       log_failure "Not enough arguments for log function! Expecting 1 argument got $#"
       exit 1
   fi
 
   ${SETCOLOR_TITLE_PURPLE}
   printf "|-------------------------------------------------------------------------|\n"
   printf "|$1|\n";
   printf "|-------------------------------------------------------------------------|\n"
   ${SETCOLOR_NORMAL}
}
 
function init() {
   # Check if hostname and key are provided
   if [ $1 -lt 2 ]; then
       ${SETCOLOR_FAILURE}
       log_failure "Not enough command line arguments! Expecting two: \$HOSTNAME and \$KEY. Recieved only $1."
       exit 1
   fi
 
   DASH_DIR=$(echo $HOST | awk -F[/:] '{print $4}')
 
   if [ ! -d "${DASH_DIR}" ]; then
     mkdir "${DASH_DIR}"
   else
    log_title "----------------- A $DASH_DIR directory already exists! -----------------"
   fi
}
 
 
HOST=$1
KEY=$2
init $# $HOST $KEY
 
counter=0
 
for dashboard_uid in $(curl -sS -H "Authorization: Bearer $KEY" $HOST/api/search\?query\=\& | jq -r '.[] | select( .type | contains("dash-db")) | .uid'); do
 
   counter=$((counter + 1))
   url=`echo $HOST/api/dashboards/uid/$dashboard_uid | tr -d '\r'`
   dashboard_json=$(curl -sS -H "Authorization: Bearer $KEY" $url)
   dashboard_title=$(echo $dashboard_json | jq -r '.dashboard | .title' | sed -r 's/[ \/]+/_/g' )
   dashboard_version=$(echo $dashboard_json | jq -r '.dashboard | .version')
   folder_title=$(echo $dashboard_json | jq -r '.meta | .folderTitle')
 
   mkdir -p "$DASH_DIR/$folder_title"
   echo $dashboard_json > "$DASH_DIR/$folder_title/${dashboard_title}_v${dashboard_version}.json"
 
   log_success "Dashboard has been saved\t\t title=\"${dashboard_title}\", uid=\"${dashboard_uid}\", path=\"${DASH_DIR}/$folder_title/${dashboard_title}_v${dashboard_version}.json\"."
done
 
log_title "${counter} dashboards were saved";
 
log_title "------------------------------ FINISHED ---------------------------------";

使用示例

bash grafana-dashboard-exporter.sh http://localhost:3000 eyJrIjoiZkhiQUVDczNFT0QyUVE4YkJwb1RBbHasfaefaefdDVXT1dkVlkiLCJuIjoiemFpaHVpX2FXkiLCJpZCI6MX0=

导入脚本grafana-dashboard-importer.sh

#!/bin/bash
#
# add the "-x" option to the shebang line if you want a more verbose output
#
#
OPTSPEC=":hp:t:k:"
 
show_help() {
cat << EOF
Usage: $0 [-p PATH] [-t TARGET_HOST] [-k API_KEY]
Script to import dashboards into Grafana
    -p      Required. Root path containing JSON exports of the dashboards you want imported.
    -t      Required. The full URL of the target host
    -k      Required. The API key to use on the target host
 
    -h      Display this help and exit.
EOF
}
 
###### Check script invocation options ######
while getopts "$OPTSPEC" optchar; do
    case "$optchar" in
        h)
            show_help
            exit
            ;;
        p)
            DASH_DIR="$OPTARG";;
        t)
            HOST="$OPTARG";;
        k)
            KEY="$OPTARG";;
        \?)
          echo "Invalid option: -$OPTARG" >&2
          exit 1
          ;;
        :)
          echo "Option -$OPTARG requires an argument." >&2
          exit 1
          ;;
    esac
done
 
if [ -z "$DASH_DIR" ] || [ -z "$HOST" ] || [ -z "$KEY" ]; then
    show_help
    exit 1
fi
 
# set some colors for status OK, FAIL and titles
SETCOLOR_SUCCESS="echo -en \\033[0;32m"
SETCOLOR_FAILURE="echo -en \\033[1;31m"
SETCOLOR_NORMAL="echo -en \\033[0;39m"
SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple
 
# usage log "string to log" "color option"
function log_success() {
   if [ $# -lt 1 ]; then
       ${SETCOLOR_FAILURE}
       echo "Not enough arguments for log function! Expecting 1 argument got $#"
       exit 1
   fi
 
   timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z")
 
   ${SETCOLOR_SUCCESS}
   printf "[%s] $1\n" "$timestamp"
   ${SETCOLOR_NORMAL}
}
 
function log_failure() {
   if [ $# -lt 1 ]; then
       ${SETCOLOR_FAILURE}
       echo "Not enough arguments for log function! Expecting 1 argument got $#"
       exit 1
   fi
 
   timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z")
 
   ${SETCOLOR_FAILURE}
   printf "[%s] $1\n" "$timestamp"
   ${SETCOLOR_NORMAL}
}
 
function log_title() {
   if [ $# -lt 1 ]; then
       ${SETCOLOR_FAILURE}
       log_failure "Not enough arguments for log function! Expecting 1 argument got $#"
       exit 1
   fi
 
   ${SETCOLOR_TITLE_PURPLE}
   printf "|-------------------------------------------------------------------------|\n"
   printf "|%s|\n" "$1";
   printf "|-------------------------------------------------------------------------|\n"
   ${SETCOLOR_NORMAL}
}
 
if [ -d "$DASH_DIR" ]; then
    DASH_LIST=$(find "$DASH_DIR" -mindepth 1 -name \*.json)
    if [ -z "$DASH_LIST" ]; then
        log_title "----------------- $DASH_DIR contains no JSON files! -----------------"
        log_failure "Directory $DASH_DIR does not appear to contain any JSON files for import. Check your path and try again."
        exit 1
    else
        FILESTOTAL=$(echo "$DASH_LIST" | wc -l)
        log_title "----------------- Starting import of $FILESTOTAL dashboards -----------------"
    fi
else
    log_title "----------------- $DASH_DIR directory not found! -----------------"
    log_failure "Directory $DASH_DIR does not exist. Check your path and try again."
    exit 1
fi
 
NUMSUCCESS=0
NUMFAILURE=0
COUNTER=0
 
for DASH_FILE in $DASH_LIST; do
    COUNTER=$((COUNTER + 1))
    echo "Import $COUNTER/$FILESTOTAL: $DASH_FILE..."
    RESULT=$(cat "$DASH_FILE" | jq '. * {overwrite: true, dashboard: {id: null}}' | curl -s -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $KEY" "$HOST"/api/dashboards/db -d @-)
    if [[ "$RESULT" == *"success"* ]]; then
        log_success "$RESULT"
        NUMSUCCESS=$((NUMSUCCESS + 1))
    else
        log_failure "$RESULT"
        NUMFAILURE=$((NUMFAILURE + 1))
    fi
done
 
log_title "Import complete. $NUMSUCCESS dashboards were successfully imported. $NUMFAILURE dashboard imports failed.";
log_title "------------------------------ FINISHED ---------------------------------";

使用示例

bash grafana-dashboard-importer.sh -t http:localhost:3000 -k eyJrIjoiZkhiQUVDczNFT0QyUaouefohaoeihoaefhaoiefiomFpaHVpX2FwaV9rZXkiLCJpZCI6MX0= -p ~/grafana-dashboard

KDP

灾备方案

数据备份

mysql备份

容器化xtrabackup

Grafana面板备份