From 46b118bbbebd93098a1f7cb0ba763d644f36fb7e Mon Sep 17 00:00:00 2001 From: yuki Date: Mon, 29 Jan 2018 23:02:03 +0900 Subject: [PATCH 1/3] HAWQ-1536 work in progress --- pxf/pxf-service/src/scripts/pxf-service | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pxf/pxf-service/src/scripts/pxf-service b/pxf/pxf-service/src/scripts/pxf-service index d06127978e..62afcce578 100644 --- a/pxf/pxf-service/src/scripts/pxf-service +++ b/pxf/pxf-service/src/scripts/pxf-service @@ -267,6 +267,11 @@ function check_hadoop_install() fi ;; cdh|CDH) + if [ -d "/opt/cloudera/parcels/CDH/lib/hadoop" ]; then + DISTRO="cdh" + CDHMANAGER="manager" + return 0; + fi if [ -d "/usr/lib/hadoop/client" ]; then DISTRO="cdh" return 0; @@ -294,6 +299,7 @@ function check_hadoop_install() function determineHadoopDistro() { DISTRO="" + CDHMANAGER="" # check if the distro is explicitly specified if [ -z "${HADOOP_DISTRO}" ]; then # if distro is not specified, try checking for HDP and then CDH @@ -335,6 +341,13 @@ function generatePrivateClasspath() if [ -n "${HADOOP_ROOT}" ]; then sed -i -e "s|HADOOP_ROOT|${HADOOP_ROOT}|g" ${PXF_HOME}/conf/pxf-private.classpath fi + + # from clouderamanager + if ["${DISTRO}" -eq "cdh" ] && [ "${CDHMANAGER}" -eq "manager" ] ; then + sed -i -e "s|CDHJAR|'/opt/cloudera/parcels/CDH/lib/hadoop/'|g" ${PXF_HOME}/conf/pxf-private.classpath + else + sed -i -e "s|CDHJAR|'/usr/lib/hadoop/'|g" ${PXF_HOME}/conf/pxf-private.classpath + fi } function printUsage() From 8990a2259abd8328e55b214f0aefaa89991dc35d Mon Sep 17 00:00:00 2001 From: yuki saito Date: Wed, 31 Jan 2018 12:54:29 +0900 Subject: [PATCH 2/3] HAWQ-1536 add setting for cloudera manager --- .../pxf-private-cdh.classpath.template | 81 ++++++++++--------- pxf/pxf-service/src/scripts/pxf-service | 15 +++- 2 files changed, 54 insertions(+), 42 deletions(-) diff --git a/pxf/pxf-service/src/configs/templates/pxf-private-cdh.classpath.template b/pxf/pxf-service/src/configs/templates/pxf-private-cdh.classpath.template index 437ffaf47b..b48a293010 100644 --- a/pxf/pxf-service/src/configs/templates/pxf-private-cdh.classpath.template +++ b/pxf/pxf-service/src/configs/templates/pxf-private-cdh.classpath.template @@ -37,46 +37,49 @@ PXF_HOME/lib/pxf-jdbc.jar /etc/hbase/conf # Hadoop Libraries -/usr/lib/hadoop/client/hadoop-hdfs.jar -/usr/lib/hadoop/client/hadoop-mapreduce-client-core.jar -/usr/lib/hadoop/client/hadoop-mapreduce-client-common.jar -/usr/lib/hadoop/client/hadoop-auth.jar -/usr/lib/hadoop/client/hadoop-common.jar -/usr/lib/hadoop/client/avro.jar -/usr/lib/hadoop/client/commons-cli.jar -/usr/lib/hadoop/client/commons-codec.jar -/usr/lib/hadoop/client/commons-collections.jar -/usr/lib/hadoop/client/commons-configuration.jar -/usr/lib/hadoop/client/commons-io.jar -/usr/lib/hadoop/client/commons-lang.jar -/usr/lib/hadoop/client/commons-logging.jar -/usr/lib/hadoop/client/guava.jar -/usr/lib/hadoop/client/htrace-core4.jar -/usr/lib/hadoop/client/jackson-core.jar -/usr/lib/hadoop/client/jackson-mapper-asl.jar -/usr/lib/hadoop/client/jetty-*.jar -/usr/lib/hadoop/client/jersey-core.jar -/usr/lib/hadoop/client/log4j.jar -/usr/lib/hadoop/client/protobuf-java-*[0-9].jar -/usr/lib/hadoop/client/slf4j-api.jar -/usr/lib/hadoop/client/snappy-java.jar -/usr/lib/hadoop/lib/asm-*[0-9].jar -/usr/lib/hadoop/lib/jersey-server-*[0-9].jar +CDH_HADOOP_JAR/client/hadoop-hdfs.jar +CDH_HADOOP_JAR/client/hadoop-mapreduce-client-core.jar +CDH_HADOOP_JAR/client/hadoop-mapreduce-client-common.jar +CDH_HADOOP_JAR/client/hadoop-auth.jar +CDH_HADOOP_JAR/client/hadoop-common.jar +CDH_HADOOP_JAR/client/avro.jar +CDH_HADOOP_JAR/client/commons-cli.jar +CDH_HADOOP_JAR/client/commons-codec.jar +CDH_HADOOP_JAR/client/commons-collections.jar +CDH_HADOOP_JAR/client/commons-configuration.jar +CDH_HADOOP_JAR/client/commons-io.jar +CDH_HADOOP_JAR/client/commons-lang.jar +CDH_HADOOP_JAR/client/commons-logging.jar +CDH_HADOOP_JAR/client/guava.jar +CDH_HADOOP_JAR/client/htrace-core4.jar +CDH_HADOOP_JAR/client/jackson-core.jar +CDH_HADOOP_JAR/client/jackson-mapper-asl.jar +CDH_HADOOP_JAR/client/jetty-*.jar +CDH_HADOOP_JAR/client/jersey-core.jar +CDH_HADOOP_JAR/client/log4j.jar +CDH_HADOOP_JAR/client/protobuf-java-*[0-9].jar +CDH_HADOOP_JAR/client/slf4j-api.jar +CDH_HADOOP_JAR/client/snappy-java.jar +CDH_HADOOP_JAR/lib/asm-*[0-9].jar +CDH_HADOOP_JAR/lib/jersey-server-*[0-9].jar +CDH_YARN_JAR/hadoop-yarn*.jar +CDH_JETTY_JAR/jetty-*.jar + # Hive Libraries -/usr/lib/hive/lib/antlr-runtime-*[0-9].jar -/usr/lib/hive/lib/datanucleus-api-jdo-*[0-9].jar -/usr/lib/hive/lib/datanucleus-core-*[0-9].jar -/usr/lib/hive/lib/hive-exec.jar -/usr/lib/hive/lib/hive-metastore.jar -/usr/lib/hive/lib/jdo-api-*[0-9].jar -/usr/lib/hive/lib/libfb303-*[0-9].jar +CDH_HIVE_JAR/lib/antlr-runtime-*[0-9].jar +CDH_HIVE_JAR/lib/datanucleus-api-jdo-*[0-9].jar +CDH_HIVE_JAR/lib/datanucleus-core-*[0-9].jar +CDH_HIVE_JAR/lib/hive-exec.jar +CDH_HIVE_JAR/lib/hive-metastore.jar +CDH_HIVE_JAR/lib/jdo-api-*[0-9].jar +CDH_HIVE_JAR/lib/libfb303-*[0-9].jar # HBase Libraries -/usr/lib/hbase/hbase-client.jar -/usr/lib/hbase/hbase-common.jar -/usr/lib/hbase/hbase-protocol.jar -/usr/lib/hbase/lib/htrace-core.jar -/usr/lib/hbase/lib/netty-*[0-9].Final.jar -/usr/lib/hbase/lib/metrics-core*.jar -/usr/lib/zookeeper/zookeeper.jar +CDH_HBASE_JAR/hbase-client.jar +CDH_HBASE_JAR/hbase-common.jar +CDH_HBASE_JAR/hbase-protocol.jar +CDH_HBASE_JAR/lib/htrace-core.jar +CDH_HBASE_JAR/lib/netty-*[0-9].Final.jar +CDH_HBASE_JAR/lib/metrics-core*.jar +CDH_ZOO_JAR/zookeeper.jar diff --git a/pxf/pxf-service/src/scripts/pxf-service b/pxf/pxf-service/src/scripts/pxf-service index 62afcce578..2ea8bdd642 100644 --- a/pxf/pxf-service/src/scripts/pxf-service +++ b/pxf/pxf-service/src/scripts/pxf-service @@ -343,10 +343,19 @@ function generatePrivateClasspath() fi # from clouderamanager - if ["${DISTRO}" -eq "cdh" ] && [ "${CDHMANAGER}" -eq "manager" ] ; then - sed -i -e "s|CDHJAR|'/opt/cloudera/parcels/CDH/lib/hadoop/'|g" ${PXF_HOME}/conf/pxf-private.classpath + if [ "${DISTRO}" = "cdh" ] && [ "${CDHMANAGER}" = "manager" ]; then + sed -i -e "s|CDH_HADOOP_JAR|/opt/cloudera/parcels/CDH/lib/hadoop|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_YARN_JAR|/opt/cloudera/parcels/CDH/jars|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_JETTY_JAR|/opt/cloudera/parcels/CDH/jars|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_HIVE_JAR|/opt/cloudera/parcels/CDH/lib/hive|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_HBASE_JAR|/opt/cloudera/parcels/CDH/lib/hbase|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_ZOO_JAR|/opt/cloudera/parcels/CDH/lib/zookeeper|g" ${PXF_HOME}/conf/pxf-private.classpath else - sed -i -e "s|CDHJAR|'/usr/lib/hadoop/'|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_HADOOP_JAR|/usr/lib/hadoop|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_YARN_JAR|/usr/lib/hadoop-yarn|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_HIVE_JAR|/usr/lib/hive|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_HBASE_JAR|/usr/lib/hbase|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_ZOO_JAR|/usr/lib/zookeeper|g" ${PXF_HOME}/conf/pxf-private.classpath fi } From b4697bbf616aa24149da7a09a72b2398137b41e2 Mon Sep 17 00:00:00 2001 From: yuki saito Date: Wed, 31 Jan 2018 13:07:22 +0900 Subject: [PATCH 3/3] HAWQ-1536 add setting for cloudera manager --- pxf/pxf-service/src/scripts/pxf-service | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pxf/pxf-service/src/scripts/pxf-service b/pxf/pxf-service/src/scripts/pxf-service index 2ea8bdd642..96e5b8f047 100644 --- a/pxf/pxf-service/src/scripts/pxf-service +++ b/pxf/pxf-service/src/scripts/pxf-service @@ -344,18 +344,18 @@ function generatePrivateClasspath() # from clouderamanager if [ "${DISTRO}" = "cdh" ] && [ "${CDHMANAGER}" = "manager" ]; then - sed -i -e "s|CDH_HADOOP_JAR|/opt/cloudera/parcels/CDH/lib/hadoop|g" ${PXF_HOME}/conf/pxf-private.classpath - sed -i -e "s|CDH_YARN_JAR|/opt/cloudera/parcels/CDH/jars|g" ${PXF_HOME}/conf/pxf-private.classpath - sed -i -e "s|CDH_JETTY_JAR|/opt/cloudera/parcels/CDH/jars|g" ${PXF_HOME}/conf/pxf-private.classpath - sed -i -e "s|CDH_HIVE_JAR|/opt/cloudera/parcels/CDH/lib/hive|g" ${PXF_HOME}/conf/pxf-private.classpath - sed -i -e "s|CDH_HBASE_JAR|/opt/cloudera/parcels/CDH/lib/hbase|g" ${PXF_HOME}/conf/pxf-private.classpath - sed -i -e "s|CDH_ZOO_JAR|/opt/cloudera/parcels/CDH/lib/zookeeper|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_HADOOP_JAR|/opt/cloudera/parcels/CDH/lib/hadoop|g" ${PXF_HOME}/conf/pxf-private.classpath \ + -e "s|CDH_YARN_JAR|/opt/cloudera/parcels/CDH/jars|g" ${PXF_HOME}/conf/pxf-private.classpath \ + -e "s|CDH_JETTY_JAR|/opt/cloudera/parcels/CDH/jars|g" ${PXF_HOME}/conf/pxf-private.classpath \ + -e "s|CDH_HIVE_JAR|/opt/cloudera/parcels/CDH/lib/hive|g" ${PXF_HOME}/conf/pxf-private.classpath \ + -e "s|CDH_HBASE_JAR|/opt/cloudera/parcels/CDH/lib/hbase|g" ${PXF_HOME}/conf/pxf-private.classpath \ + -e "s|CDH_ZOO_JAR|/opt/cloudera/parcels/CDH/lib/zookeeper|g" ${PXF_HOME}/conf/pxf-private.classpath else - sed -i -e "s|CDH_HADOOP_JAR|/usr/lib/hadoop|g" ${PXF_HOME}/conf/pxf-private.classpath - sed -i -e "s|CDH_YARN_JAR|/usr/lib/hadoop-yarn|g" ${PXF_HOME}/conf/pxf-private.classpath - sed -i -e "s|CDH_HIVE_JAR|/usr/lib/hive|g" ${PXF_HOME}/conf/pxf-private.classpath - sed -i -e "s|CDH_HBASE_JAR|/usr/lib/hbase|g" ${PXF_HOME}/conf/pxf-private.classpath - sed -i -e "s|CDH_ZOO_JAR|/usr/lib/zookeeper|g" ${PXF_HOME}/conf/pxf-private.classpath + sed -i -e "s|CDH_HADOOP_JAR|/usr/lib/hadoop|g" ${PXF_HOME}/conf/pxf-private.classpath \ + -e "s|CDH_YARN_JAR|/usr/lib/hadoop-yarn|g" ${PXF_HOME}/conf/pxf-private.classpath \ + -e "s|CDH_HIVE_JAR|/usr/lib/hive|g" ${PXF_HOME}/conf/pxf-private.classpath \ + -e "s|CDH_HBASE_JAR|/usr/lib/hbase|g" ${PXF_HOME}/conf/pxf-private.classpath \ + -e "s|CDH_ZOO_JAR|/usr/lib/zookeeper|g" ${PXF_HOME}/conf/pxf-private.classpath fi }