分享知识，分享快乐

0%

hbase常用命令操作实战

发表于

hbase-hbck2

从官网下载hbck2 执行文件

http://hbase.apache.org/downloads.html

或 wget https://www.apache.org/dyn/closer.lua/hbase/hbase-operator-tools-1.0.0/hbase-operator-tools-1.0.0-bin.tar.gz

cdh官网的使用帮助

https://docs.cloudera.com/documentation/enterprise/6/6.3/topics/admin_hbase_hbck.html#concept_hkk_q25_llb

1
2
3
4
5
6
7
cd /opt/hbase-operator-tools-1.0.0/hbase-hbck2

hbase hbck -j hbase-hbck2-1.0.0.jar -s  assigns 1588230740

hbase hbck -j hbase-hbck2-1.0.0.jar -s  assigns hbase:namespace,,1594264903686.db55eec81d86ac0ae26eba718518ce26

hbase hbck -j hbase-hbck2-1.0.0.jar -s addFsRegionsMissingInMeta default:test n1:tbl_2 n2

hbase shell 实战

移动表的数据到另外一个服务器

1
2
3
# 语法：move 'encodeRegionName', 'ServerName'``
# encodeRegionName指的regioName后面的编码，ServerName指的是master-status的Region Servers列表
move '61a647884b761a785daf3b38049aaa27','bigdata-5.baofoo.cn,16020,1595233360076'
1
2


如何查看HBase的HFile

https://www.jianshu.com/p/49043e99795a

跨集群备份数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#正则查询快照
list_snapshots 'snapshot.*_20200902'
echo "list_snapshots 'snapshot.*_`date -d "1 days ago" +%Y%m%d`' " |sudo -u hbase hbase shell 


#同步快照到备份集群
curl "http://cdh85-49:20550/" > hbase_tbls.txt
cat hbase_tbls.txt | tr ':' '-' | while read tb; do 
sudo -u hbase hbase org.apache.hadoop.hbase.snapshot.ExportSnapshot \
-snapshot "snapshot_${tb}_20200902" \
-copy-from hdfs://cdh85-49:8020/hbase \
-copy-to hdfs://cdh85-106:8020/hbase ;
done


#还原快照
cat hbase_tbls.txt | tr ':' '-' | while read tb; do 
echo  "restore_snapshot 'snapshot_${tb}_20200902'" | sudo -u hbase hbase shell ;  
done

定时备份 批量脚本程序

hbase snapshots.sh root@cdh85-55

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
#!/bin/bash

PRG="${0}"
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/;pwd`

## 0. 列出所有hbase表 准备做快照
echo "######################### job start: `date '+%Y-%m%-d %H:%M:%S'`"
echo "######################### 0.list hbase tables;"
curl "http://bigdata-5.baofoo.cn:20550/" > ${BASEDIR}/tbls.txt

## 1. 生成新的当日的快照  快照命名方式  snapshot_NAMESPARCE-TABLE_YYYYMMDD
echo "######################### 1.create hbase table snapshots;"
cat  ${BASEDIR}/tbls.txt | while read tbls; do
echo  "######################### create '${tbls}' snapshot:`date '+%Y-%m%-d %H:%M:%S'`" 
echo  "snapshot '${tbls}','snapshot_${tbls}_`date +%Y%m%d`' " | awk -F, '{ gsub(":","-",$2) ;print $1","$2}' | sudo -u hbase hbase shell ;  done

## 2.删除指定hbase集群快照， 入参 { $1:n , 表示n天前的快照，比如 0：当天 1：一天前 2：两天前}
echo "######################### 2.delete hbase history snapshots;"
find_snapshot_list(){
echo "list_snapshots 'snapshot.*_`date -d "${1} days ago" +%Y%m%d`' " |sudo -u hbase hbase shell  ;
}

delete_snapshot_list(){
find_snapshot_list $1 | grep "\\[.*\\]"  | sed 's/[]["]//g' | tr ',' '\n' | while read word; do
echo  "delete_snapshot '${word}'" |sudo -u hbase hbase shell
done
}
#删除1天前快照
delete_snapshot_list 0

echo "######################### job end: `date '+%Y-%m%-d %H:%M:%S'`"
#!/bin/bash

cd /root/hbase_snapshots/

## 0. 列出所有hbase表 准备做快照
echo "######################### job start: `date '+%Y-%m%-d %H:%M:%S'`"
echo "######################### 0.list hbase tables;"
curl "http://cdh85-49:20550/" > tbls.txt

## 1. 生成新的当日的快照  快照命名方式  snapshot_NAMESPARCE-TABLE_YYYYMMDD
echo "######################### 1.create hbase table snapshots;"
cat tbls.txt | while read tbls; do
echo  "######################### create '${tbls}' snapshot:`date '+%Y-%m%-d %H:%M:%S'`" 
echo  "snapshot '${tbls}','snapshot_${tbls}_`date +%Y%m%d`'; sleep 1; " | awk -F, '{ gsub(":","-",$2) ;print $1","$2}' | sudo -u hbase hbase shell ;  done

## 2.删除指定hbase集群快照， 入参 { $1:n , 表示n天前的快照，比如 0：当天 1：一天前 2：两天前}
echo "######################### 2.delete hbase history snapshots;"
find_snapshot_list(){
echo "list_snapshots 'snapshot.*_`date -d "${1} days ago" +%Y%m%d`' " |sudo -u hbase hbase shell  ;
}

delete_snapshot_list(){
find_snapshot_list $1 | grep "\\[.*\\]"  | sed 's/[]["]//g' | tr ',' '\n' | while read word; do
echo  "delete_snapshot '${word}'" |sudo -u hbase hbase shell
done
}
#删除1天前快照
delete_snapshot_list 1

echo "######################### job end: `date '+%Y-%m%-d %H:%M:%S'`"
#!/bin/bash


## 0. 列出所有hbase表 准备做快照
#echo "list" | sudo -u hbase hbase shell |grep "\\[.*\\]" > a.log
curl "http://bigdata-5.baofoo.cn:20550/" > hbase_tbls.txt

## 1. 删除当前集群历史快照
#echo "delete_all_snapshot 'snapshot.*' " |sudo -u hbase hbase --config  /etc/hbase/conf.cloudera.hbase shell  ;
#cat a.log   | sed 's/[]["]//g' | tr ',' '\n' | while read word; do echo  "delete_snapshot   'snapshot_${word}_`date -d '1 days ago' +%Y%m%d`' " | awk -F"   " '{ gsub(":","-",$2) ;print $1" "$2}' |sudo -u hbase hbase --config  /etc/hbase/conf.cloudera.hbase shell  ; done

## 2. 生成新的当日的快照  快照命名方式  snapshot_NAMESPARCE-TABLE_YYYYMMDD
#cat a.log   | sed 's/[]["]//g' | tr ',' '\n' | while read word; do \
#echo  "snapshot '${word}' ,  'snapshot_${word}_`date +%Y%m%d`' " | awk -F, '{ gsub(":","-",$2) ;print $1","$2}' | sudo -u hbase hbase --config  /etc/hbase/conf.cloudera.hbase shell ;  done

cat hbase_tbls.txt | while read word; do \
echo  "snapshot '${word}' ,  'snapshot_${word}_`date +%Y%m%d`' " | awk -F, '{ gsub(":","-",$2) ;print $1","$2}' | sudo -u hbase hbase --config  /etc/hbase/conf.cloudera.hbase shell ;  done


##  3. 删除备份集群历史快照
#cat a.log |sed 's/"//g'   | sed 's/[][]//g' | tr ',' '\n' | while read word; do echo  "delete_snapshot   'snapshot_${word}_`date -d '1 days ago' +%Y%m%d`' " | awk -F"   " '{ gsub(":","-",$2) ;print $1" "$2}' |sudo -u hbase hbase --config  /etc/hbase/conf.cloudera.hbase2 shell  ; done
#echo " delete_all_snapshot 'snapshot.*' " |sudo -u hbase hbase --config  /etc/hbase/conf.cloudera.hbase2 shell  

## 4. 同步快照到备份集群
cat a.log |sed 's/"//g'   | sed 's/[][]//g' | tr ',' '\n' | tr ':' '-'  | while read word;do 
sudo -u hbase hbase org.apache.hadoop.hbase.snapshot.ExportSnapshot \
-Dmapreduce.job.queuename=bf_yarn_pool.development \
--snapshot 'snapshot_${word}_`date +%Y%m%d`'  \
--copy-from hdfs://ns1/hbase3  \
--copy-to hdfs://ns1/hbase9 \
--chuser hbase -chgroup hbase --overwrite
;
done

## 查找历史快照，   入参 2个 { $1:n , 表示n天前的快照，比如 0：当天 1：一天前 2：两天前；$2: hbase config路径}

find_snapshot_list(){
echo "list_snapshots 'snapshot.*_`date -d "${1} days ago" +%Y%m%d`' " |sudo -u hbase hbase --config  $2 shell  ;
}

## 查找一天前快照 hbase1集群
#find_snapshot_list 1 /etc/hbase/conf.cloudera.hbase


## 删除指定hbase集群快照， 入参 2个，{ $1:n , 表示n天前的快照，比如 0：当天 1：一天前 2：两天前；$2: hbase config路径 }
delete_snapshot_list(){
find_snapshot_list $1 $2 | grep "\\[.*\\]"  | sed 's/[]["]//g' | tr ',' '\n' | while read word; do
echo  "delete_snapshot '${word}'" |sudo -u hbase hbase --config $2  shell
done
}

delete_snapshot_list 1 /etc/hbase/conf.cloudera.hbase


cat a.log   | sed 's/[]["]//g' | tr ',' '\n' | while read word; do echo   '${word}' ;done

hbase-restore-snapshot.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
#!/bin/bash

PRG="${0}"
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/;pwd`

usage(){
read -p "DO YOU WANT RESTORE ALL HBASE TABLES? PLEASE INPUT[yes OR no]:" yn
case $yn in
        [Yy]* ) continue;;
        [Nn]* ) exit 1;;
        * ) echo "Please answer yes or no.";exit 2;;
esac
}
usage


## 0. 列出所有hbase表 准备还原快照
echo "######################### job start: `date '+%Y-%m%-d %H:%M:%S'`"
echo "######################### 0.list hbase tables;"
curl "http://bigdata-3.baofoo.cn:20550/" > ${BASEDIR}/tbls.txt

## 1.disable  hbase tables.
echo "######################### 1.disable hbase  table; "
cat  ${BASEDIR}/tbls.txt | while read tbls; do 
echo  "######################### disable '${tbls}' " 
echo  "disable '${tbls}'  " | sudo -u hbase hbase shell ;  done

## 2.恢复指定hbase集群快照， 入参 { $1:n , 表示n天前的快照，比如 0：当天 1：一天前 2：两天前}
echo "######################### 2.restore_snapshot "
find_snapshot_list(){
echo "list_snapshots 'snapshot.*_`date -d "${1} days ago" +%Y%m%d`' " |sudo -u hbase hbase shell  ;
}

restore_snapshot_(){
find_snapshot_list $1 | grep "\\[.*\\]"  | sed 's/[]["]//g' | tr ',' '\n' | while read word; do
echo "######################### 2.restore_snapshot '${word}'; "
echo  "restore_snapshot  '${word}'" |sudo -u hbase hbase shell
done
}
#恢复1天前快照
restore_snapshot_ 1

## 3.enable  hbase tables.
echo "######################### 3.enable hbase  table; "
cat  ${BASEDIR}/tbls.txt | while read tbls; do 
echo  "######################### enable '${tbls}' " 
echo  "enable '${tbls}'  "| sudo -u hbase hbase shell ;  done

echo "######################### job end: `date '+%Y-%m%-d %H:%M:%S'`"

hbase_restore_snashot_1.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
#!/bin/bash

PRG="${0}"
BASEDIR=`dirname ${PRG}`
BASEDIR=`cd ${BASEDIR}/;pwd`

usage(){
echo "RESTORE ALL HBASE TABLES,steps:1.disable hbase tables;2.restore_snapshot;3.enable hbase tables."
read -p "DO YOU WANT RESTORE ALL HBASE TABLES? PLEASE INPUT[yes OR no]:" yn
case $yn in
        [Yy]* ) continue;;
        [Nn]* ) exit 1;;
        * ) echo "Please answer yes or no.";exit 2;;
esac
}
usage


## 0. 列出所有hbase表 准备还原快照
echo "######################### job start: `date '+%Y-%m%-d %H:%M:%S'`"
echo "######################### 0.list hbase tables;"
##curl "http://bigdata-3.baofoo.cn:20550/" > ${BASEDIR}/tbls.txt
curl "http://cdh85-49:20550/" > ${BASEDIR}/tbls.txt


## 1.disable  hbase tables;  2.restore_snapshot;   3.enable hbase tables;

cat  ${BASEDIR}/tbls.txt | while read tbls; do 
echo  "######################### 1. disable '${tbls}' ：`date '+%Y-%m%-d %H:%M:%S'` " 
echo  "disable '${tbls}'" | sudo -u hbase hbase shell ;  

#deal_date=`date -d "0 days ago" +%Y%m%d`
deal_date=`date -d "1 days ago" +%Y%m%d`
tbls_r=`echo ${tbls}|tr ':' '-'`
echo  "######################### 2. restore_snapshot 'snapshot_${tbls_r}_${deal_date}'：`date '+%Y-%m%-d %H:%M:%S'`  "
echo  "restore_snapshot 'snapshot_${tbls_r}_${deal_date}'" | sudo -u hbase hbase shell ;  

echo  "######################### 3. enable '${tbls}' ：`date '+%Y-%m%-d %H:%M:%S'` " 
echo  "enable '${tbls}' " | sudo -u hbase hbase shell ;


done