分享知识，分享快乐

0%

distcp 文件迁移

发表于 更新于 
1
2
3
4
5
#!/bin/bash
for DB in `cat db_name.txt`
do
 hadoop distcp -D mapreduce.job.queuename=bf_yarn_pool.production -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/hive/warehouse/$DB.db hdfs://172.20.85.39:8020/user/hive/warehouse/$DB.db
done
1
2
3
4
5
6
7
8
9
10
11
12
13
hadoop distcp \
-Dmapred.jobtracker.maxtasks.per.job=1800000 \   #任务最大map数（数据分成多map任务）
-Dmapred.job.max.map.running=4000 \              #最大map并发
-Ddistcp.bandwidth=150000000 \                   #带宽
-Ddfs.replication=2 \                            #复制因子，两副本
-Ddistcp.skip.dir=$skipPath \                    #过滤的目录（不拷贝的目录）
-Dmapred.map.max.attempts=9 \                    #每个task最大尝试次数
-Dmapred.fairscheduler.pool=distcp \             #指定任务运行的pool
-pugp \                                          #保留属性（用户，组，权限）
-i \                                             #忽略失败的task
-skipcrccheck \                                  #忽略CRC校验（防止源，目标集群hdfs版本不一致导致任务失败。）
hdfs://clusterA:9000/AAA/data  \                 #源地址
hdfs://clusterB:9000/BBB/data                    #目标地址

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/xy_app_spark/tables/fi_gw_express_order_idcard1_encrypt/pk_year=2018/pk_month=2018-10 hdfs://172.20.85.39:8020/user/hive/warehouse/credit_mining.db/fi_gw_express_order_idcard1_encrypt/pk_year=2018/pk_month=2018-10

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/xy_app_spark/tables/fi_gw_express_order_idcard1_encrypt/pk_year=2018/pk_month=2018-11 hdfs://172.20.85.39:8020/user/hive/warehouse/credit_mining.db/fi_gw_express_order_idcard1_encrypt/pk_year=2018/pk_month=2018-11

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/xy_app_spark/tables/fo_payment_encrypt/pk_year=2018/pk_month=2018-10 hdfs://172.20.85.39:8020/user/hive/warehouse/credit_mining.db/fo_payment_encrypt/pk_year=2018/pk_month=2018-10

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/xy_app_spark/tables/fo_payment_encrypt/pk_year=2018/pk_month=2018-11 hdfs://172.20.85.39:8020/user/hive/warehouse/credit_mining.db/fo_payment_encrypt/pk_year=2018/pk_month=2018-11

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020//user/hive/warehouse/xy_ods.db/t_serve_business_order_real_time_v2_encrypt hdfs://172.20.85.39:8020/user/hive/warehouse/xy_ods.db/t_serve_business_order_real_time_v2_encrypt

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i hdfs://192.168.81.30:8020/user/hive/warehouse/xy_ods_db.db/credit_logprocessor_rocord hdfs://172.20.85.39:8020/user/hive/warehouse/xy_ods_db.db/credit_logprocessor_rocord

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/hive/warehouse/xy_ods_db.db/credit_logprocessor_rocord/pk_day=2018-11-11 hdfs://172.20.85.39:8020/user/hive/warehouse/xy_ods_db.db/credit_logprocessor_rocord/pk_day=2018-11-11

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/hive/warehouse/xy_ods.db/ods_verification_cardno_d_incr/pk_year=2017 hdfs://172.20.85.39:8020/user/hive/warehouse/xy_ods.db/ods_verification_cardno_d_incr/pk_year=2017

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -update -skipcrccheck hdfs://192.168.81.30:8020/user/hive/warehouse/xy_ods.db/ods_verification_cardno_d_incr/pk_year=2017/pk_month=2017-07/pk_day=2017-07-23/000011_0 hdfs://172.20.85.39:8020/user/hive/warehouse/xy_ods.db/ods_verification_cardno_d_incr/pk_year=2017/pk_month=2017-07/pk_day=2017-07-23/000011_0

sudo -u xy_app_spark hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -update -skipcrccheck hdfs://172.20.85.29:8020/user/xy_app_spark/bulkload/NORMAL/xy_app_spark-image_current_report hdfs://172.20.85.59:8020/user/xy_app_spark/bulkload/NORMAL/xy_app_spark-image_current_report

sudo -u hdfs hdfs ec -getPolicy -path /user

sudo -u xy_app_spark hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -update hdfs://172.20.85.29:8020/user/xy_app_spark/bulkload/NORMAL/xy_app_spark-image_current_report hdfs://172.20.85.59:8020/user/xy_app_spark/bulkload/NORMAL/xy_app_spark-image_current_report

hadoop fs -get /user/hive/warehouse/baofoo_cutpayment.db/protocol_payment_order /home/yarn/protocol_payment_order
hadoop fs -get /user/hive/warehouse/baofoo_cutpayment.db/protocol_payment_business_order /home/yarn/protocol_payment_business_order

baofoo_cutpayment.protocol_payment_order
baofoo_cutpayment.protocol_payment_business_order

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true hdfs://192.168.81.30:8020/user/hue/oozie/workspaces/ hdfs://172.20.85.39:8020/user/hue/oozie/workspaces/

Bad status for request TOpenSessionReq(username=‘hue’, password=None, client_protocol=6, configuration={‘idle_session_timeout’: ‘900’, ‘impala.doas.user’: u’hue’}): TOpenSessionResp(status=TStatus(errorCode=None, errorMessage=“User ‘yarn’ is not authorized to delegate to ‘hue’.\n”, sqlState=‘HY000’, infoMessages=None, statusCode=3), sessionHandle=TSessionHandle(sessionId=THandleIdentifier(secret=’\xdcs{\xf9A\x12N\xb1\x97\x18\xf4\xbb\xc8\x90#\xa7’, guid=’\x12\x13\x8b\xcd\xa1\xdfA\x07\x9c\xf9\x16i\x97\rU9’)), configuration=None, serverProtocolVersion=5)

/user/xy_app_spark/tables/fo_payment_encrypt
/user/xy_app_spark/tables/t_serve_business_order_real_time_encrypt
/user/xy_app_spark/tables/fi_gw_agrt_express_order_encrypt
/user/xy_app_spark/tables/fi_gw_express_order_idcard1_encrypt

ifactive=sudo -u hdfs hdfs haadmin -getServiceState namenode402
echo $ifactive
if [[ $ifactive =~ “active” ]]; then
  nameservice=172.20.85.29
else
nameservice=172.20.85.39
fi
echo $nameservice

hdfs://ns1/user/hive/warehouse/credit_dfp.db/t_device_access_d_incr

hdfs://ns1/user/hive/warehouse/credit_dfp.db/t_device_query_d_incr

hdfs://ns1/user/hive/warehouse/credit_dfp.db/t_device_access_d_incr

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/hive/warehouse/xy_ods.db/t_serve_business_order_real_time/pk_year=2019 hdfs://172.20.85.29:8020/user/hive/warehouse/xy_ods.db/t_serve_business_order_real_time/pk_year=2019

旧集群/user/hive/warehouse/xy_ods.db/t_serve_business_order_real_time/pk_year=2018/pk_month=2018-12
到新集群 旧集群/user/hive/warehouse/xy_ods.db/t_serve_business_order_real_time/pk_year=2019/pk_month=2018-12

hadoop distcp /user/hive/warehouse/sample_08 /user/hive/warehouse/t3

/etc/init.d/mysql

hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/hive/warehouse/xy_app_spark.db/snapshot/current/nono/washer_all/step4 hdfs://172.20.85.29:8020/user/hive/warehouse/xy_app_spark.db/snapshot/current/nono/washer_all/step4

#!/bin/bash
for tb in cat tb_name.txt
do
hadoop distcp -D mapreduce.job.queuename=xy_yarn_pool.development -D ipc.client.fallback-to-simple-auth-allowed=true -i -overwrite hdfs://192.168.81.30:8020/user/hive/warehouse/xy_wulichuang.db/$tb hdfs://172.20.85.29:8020/user/hive/warehouse/xy_wulichuang.db/$tb
done