快捷導(dǎo)航

Java JDBC連接Kerberos認(rèn)證的HIVE和Impala方式

更新時(shí)間：2025年02月11日 09:29:34 作者：Donnedaen

本文主要介紹了HiveJDBC和ImpalaJDBC的使用方法,包括版本對(duì)應(yīng)、Maven安裝、主機(jī)名配置、端口開通、JDBC連接和Kerberos認(rèn)證等

1. 版本

HIVE 2.1.1-cdh6.3.2版本需對(duì)應(yīng)版本的 Hive JDBC 和 Impala JDBC，如下：

注意：?jiǎn)为?dú)使用HIVE JDBC ，使用版本 3.1.0，使用 Impala JDBC 需要搭配 HIVE JDBC 1.1.0使用，不然會(huì)報(bào)錯(cuò)；搭配HIVE JDBC 3.1.0 報(bào)：

java.lang.NoClassDefFoundError:org/apache/hive/service/cli/thrift/TCLIService$Client

       <dependency>
         <groupId>org.apache.hive</groupId>
         <artifactId>hive-exec</artifactId>
         <version>3.1.0</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-common</artifactId>
         <version>3.1.0</version>
       </dependency>
       <dependency>
<!--        <dependency>-->
<!--            <groupId>org.apache.hive</groupId>-->
<!--            <artifactId>hive-jdbc</artifactId>-->
<!--            <version>3.1.0</version>-->
<!--        </dependency>-->
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-auth</artifactId>
         <version>3.1.0</version>
       </dependency>
       <dependency>
         <groupId>com.cloudera.impala</groupId>
         <artifactId>impala-jdbc41</artifactId>
         <version>2.5.28</version>
       </dependency>
       <dependency>
           <groupId>org.apache.hive</groupId>
           <artifactId>hive-jdbc</artifactId>
           <version>1.1.0</version>
       </dependency>

1.1 手動(dòng)安裝 maven

如果 Maven 庫沒有相應(yīng)JDBC，手動(dòng)安裝 maven，如下：

mvn install:install-file -D file=C:\Users\mx\Desktop\impala-jdbc41-0.0.1.jar -D groupId=com.cloudera.impala -D artifactId=impala-jdbc42 -D version=2.6.33 -D packaging=jar

2. 映射地址

因kerberos SPN配置必須使用主機(jī)名鏈接，Linux 和 Windows需要配置hosts，格式：IP地址主機(jī)名，配置完成之后 telnet 檢測(cè)

說明：如果是docker容器鏈接 Hive 或 Impala ，還需要在docker容器中配置 hosts，保證主機(jī)和容器 telent 都是通的

telnet hostname port

2.1 端口

開通 HIVE 和 Impala 相應(yīng)的端口，本地調(diào)試按需將 IP 和端口加入白名單

HIVE TCP端口：jdbc:hive2://hostname:10000
Impala TCP端口：jdbc:impala://hostname:21050
Hdfs TCP端口：hdfs://hostname:8020
Kerberos UDP端口：88

2.2 調(diào)試

使用 telnet hostname port ，調(diào)試以上 hostname 和端口，保障網(wǎng)絡(luò)連通

3. HIVE JDBC 連接 Kerberos 認(rèn)證的 HIVE 集群

package com.test;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;

public class HiveKerberosConnectionExample {

    private static String krb5 = "src/main/resources/krb5.conf";
    private static String keytab = "src/main/resources/hive.keytab";
    private static String principal = "hive@EXAMPLE.COM";
    private static String hiveUrl = "jdbc:hive2://host:10000/test;principal=hive/_HOST@EXAMPLE.COM";

    public static void main(String[] args) throws Exception {
        // 設(shè)置Kerberos配置文件路徑
        System.setProperty("java.security.krb5.conf", krb5);
        System.setProperty("sun.security.krb5.debug", "true"); // 可選，用于調(diào)試 Kerberos 認(rèn)證過程
        // 初始化Hadoop配置
        Configuration conf = new Configuration();
        conf.set("hadoop.security.authentication", "kerberos");
        // 使用Keytab進(jìn)行Kerberos身份驗(yàn)證
        UserGroupInformation.setConfiguration(conf);
        UserGroupInformation.loginUserFromKeytab(principal, keytab);
        
        // 建立Hive連接
        Class.forName("org.apache.hive.jdbc.HiveDriver");
        Connection connection = DriverManager.getConnection(hiveUrl, "", "");

        // 執(zhí)行查詢
        Statement stmt = connection.createStatement();
        ResultSet res = stmt.executeQuery("select * from test.tblname");

        // 處理結(jié)果
        while (res.next()) {
            //a,b,c,ds 為 tblname表字段
            String data = res.getString("a") + "," + res.getString("b") + "," + res.getString("c") + "," + res.getString("ds");
            System.out.println("************** 輸出 tblname***************************");
            System.out.println(data);
        }

        // 關(guān)閉資源
        res.close();
        stmt.close();
        connection.close();
    }
}

4. HIVE JDBC 連接 Kerberos 認(rèn)證的 Impala 集群

package com.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;

import java.security.PrivilegedAction;
import java.sql.*;


public class ImpalaKerberosConnectionExample {
    private static String krb5 = "src/main/resources/krb5.conf";
    private static String keytab = "src/main/resources/impala.keytab";
    private static String principal = "impala@EXAMPLE.COM";
    private static String impalaUrl = "jdbc:hive2://host:21050/test;principal=impala/_HOST@EXAMPLE.COM";
    public static void main(String[] args) throws Exception {
        // 設(shè)置Kerberos配置文件路徑
        System.setProperty("java.security.krb5.conf", krb5);
        System.setProperty("sun.security.krb5.debug", "true"); // 可選，用于調(diào)試 Kerberos 認(rèn)證過程
        // 初始化Hadoop配置
        Configuration conf = new Configuration();
        conf.set("hadoop.security.authentication", "kerberos");
        // 使用Keytab進(jìn)行Kerberos身份驗(yàn)證
        UserGroupInformation.setConfiguration(conf);
        UserGroupInformation.loginUserFromKeytab(principal, keytab);
        UserGroupInformation loginUser = UserGroupInformation.getLoginUser();

        //使用HIVE jdbc 建立Impala連接
        Class.forName("org.apache.hive.jdbc.HiveDriver");
        // Impala Jdbc 和 HIVE Jdbc 查詢
//        loginUser.doAs((PrivilegedAction<Void>) () -> {
//            try {
//                try (Connection connection = DriverManager.getConnection(impalaUrl,"","")) {
//                    try (Statement statement = connection.createStatement()) {
//                        ResultSet resultSet = statement.executeQuery("SELECT * FROM test.tblname LIMIT 10");
//                        while (resultSet.next()) {
//                            String data = resultSet.getString("a") + "," + resultSet.getString("b") + "," + resultSet.getString("c") + "," + resultSet.getString("ds");
//                            System.out.println("************** 輸出 tblname***************************");
//                            System.out.println(data);
//                        }
//                        resultSet.close();
//                    }
//                }
//            } catch (SQLException e) {
//                e.printStackTrace();
//            }
//            return null;
//        });

        // HIVE JDBC 查詢,Impala Jdbc 查詢報(bào) kerberos 認(rèn)證失敗 1312 碼
        Connection connection = DriverManager.getConnection(impalaUrl, "", "");
        Statement stmt = connection.createStatement();
        ResultSet res = stmt.executeQuery("SELECT * FROM test.tlbname LIMIT 10");

        // 處理結(jié)果
        while (res.next()) {
            String data = res.getString("a") + "," + res.getString("b") + "," + res.getString("c") + "," + res.getString("ds");
            System.out.println("************** Impala 輸出 tblname ***************************");
            System.out.println(data);
        }

        // 關(guān)閉資源
        res.close();
        stmt.close();
        connection.close();
    }
}

5. Impala JDBC 連接 Kerberos 認(rèn)證的 Impala 集群

package com.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;

import java.security.PrivilegedAction;
import java.sql.*;


public class ImpalaKerberosConnectionExample {
    private static String krb5 = "src/main/resources/krb5.conf";
    private static String keytab = "src/main/resources/impala.keytab";
    private static String principal = "impala@EXAMPLE.COM";
    private static String impalaUrl = "jdbc:impala://host:21050/test;AuthMech=1;KrbRealm=EXAMPLE.COM;KrbHostFQDN=host;KrbServiceName=impala";
    public static void main(String[] args) throws Exception {
        // 設(shè)置Kerberos配置文件路徑
        System.setProperty("java.security.krb5.conf", krb5);
        System.setProperty("sun.security.krb5.debug", "true"); // 可選，用于調(diào)試 Kerberos 認(rèn)證過程
        // 初始化Hadoop配置
        Configuration conf = new Configuration();
        conf.set("hadoop.security.authentication", "kerberos");
        // 使用Keytab進(jìn)行Kerberos身份驗(yàn)證
        UserGroupInformation.setConfiguration(conf);
        UserGroupInformation.loginUserFromKeytab(principal, keytab);
        UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
        // 使用 Impala jdbc 建立Impala連接
        Class.forName("com.cloudera.impala.jdbc41.Driver");// 搭配HIVE JDBC 3.1.0報(bào)：java.lang.NoClassDefFoundError: org/apache/hive/service/cli/thrift/TCLIService$Client

        //使用HIVE jdbc 建立Impala連接
//        Class.forName("org.apache.hive.jdbc.HiveDriver");
        // Impala查詢
        loginUser.doAs((PrivilegedAction<Void>) () -> {
            try {
                try (Connection connection = DriverManager.getConnection(impalaUrl,"","")) {
                    try (Statement statement = connection.createStatement()) {
                        ResultSet resultSet = statement.executeQuery("SELECT * FROM test.tblname LIMIT 10");
                        while (resultSet.next()) {
                            String data = resultSet.getString("a") + "," + resultSet.getString("b") + "," + resultSet.getString("c") + "," + resultSet.getString("ds");
                            System.out.println("************** Impala 輸出 tlbname ***************************");
                            System.out.println(data);
                        }
                        resultSet.close();
                    }
                }
            } catch (SQLException e) {
                e.printStackTrace();
            }
            return null;
        });
    }
}