Apache Kyuubi on CDH 部署
https://shmily-qjj.top/ee1c2df4/
Apache Kyuubi on Spark 在 CDH 上的深度实践
https://www.slidestalk.com/openLooKeng/22
vi kyuubi-env.sh
1
2
3
4
5
6
7
8
9
| export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
export SPARK_HOME=/opt/cloudera/parcels/CDH/lib/spark3
export FLINK_HOME=/opt/cloudera/parcels/FLINK
export HIVE_HOME=/opt/cloudera/parcels/CDH/lib/hive
export HADOOP_CONF_DIR=/etc/hadoop/conf
export YARN_CONF_DIR=/etc/hadoop/conf
export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled
-XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark -XX:MaxDirectMemorySize=1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:./logs/kyuubi-server-gc-%t.log -XX:+Us
eGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=5M -XX:NewRatio=3 -XX:MetaspaceSize=512m"
|
vi conf/kyuubi-defaults.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
| kyuubi.frontend.bind.host 10.0.0.1
kyuubi.frontend.protocols THRIFT_BINARY
kyuubi.frontend.thrift.binary.bind.port 10009
#
# kyuubi.engine.type SPARK_SQL
# kyuubi.engine.share.level USER
# kyuubi.session.engine.initialize.timeout PT3M
#
kyuubi.ha.addresses zk1:2181,zk2:2181,zk3:2181
kyuubi.ha.namespace kyuubi
# 不指定 spark就是local模式运行了
spark.master=yarn
spark.submit.deployMode=cluster
kyuubi.authentication=LDAP
kyuubi.authentication.ldap.baseDN=dc=org
kyuubi.authentication.ldap.domain=apache.org
kyuubi.authentication.ldap.binddn=uid=kyuubi,OU=Users,DC=apache,DC=org
kyuubi.authentication.ldap.bindpw=kyuubi123123
kyuubi.authentication.ldap.url=ldap://hostname.com:389/
|
重启
1
2
3
4
5
| 注意 ：kyuubi 停止的时候 spark on yarn的 kyuubi_application 并没有停掉， 需要在yarn里kill， 这样重启 才能加载新加入的jar包和新的配置文件。
sudo -u hdfs bin/kyuubi start
sudo -u hdfs bin/kyuubi stop
sudo -u hdfs bin/kyuubi restart
|
bin/beeline -u ‘jdbc:hive2://10.0.19.132:10009/’ -n hadoop
bin/beeline -u ‘jdbc:hive2://172.20.192.36:10009/’ -n hadoop
jdbc:hive2://172.20.192.36:10009/?spark.app.name=test_kyuubi_application;spark.sql.adaptive.enabled=true;spark.driver.memory=4G;spark.executor.instances=256;spark.executor.cores=4;spark.executor.memory=16g
1
| ?spark.app.name=test_kyuubi_application;spark.dynamicAllocation.enabled=true;spark.shuffle.service.enabled=true;spark.shuffle.useOldFetchProtocol=true;spark.dynamicAllocation.initialExecutors=10;spark.dynamicAllocation.minExecutors=5;spark.dynamicAllocation.maxExecutors=200;spark.dynamicAllocation.executorIdleTimeout=60s;spark.dynamicAllocation.cachedExecutorIdleTimeout=10min;spark.executor.cores=1;spark.executor.memory=2G;spark.yarn.queue=bf_yarn_pool.development
|