Hbase won’t start after server reboot in Kerberized and Encryption enabled IOP cluster.

Hbase start is failing with the following error in Hbase log in Kerberized and Encryption enabled IOP cluster after server reboot.

2017-01-25 20:02:54,212 INFO
[master/atl-stg-big-01c.sec.ibm.com/207.231.140.182:16000]
regionserver.HRegionServer: ClusterId :
f5b2700e-c147-44e5-901b-dc3a09bada7b
2017-01-25 20:02:54,364 FATAL
[atl-stg-big-01c:16000.activeMasterManager] master.HMaster: Failed to
become active master
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:
350)
at
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImp
l.java:206)
at
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:18
8)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at sun.net.NetworkClient.doConnect(NetworkClient.java:175)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
at sun.net.www.http.HttpClient.(HttpClient.java:211)
at sun.net.www.http.HttpClient.New(HttpClient.java:308)
at sun.net.www.http.HttpClient.New(HttpClient.java:326)
at
sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient(HttpURLConnection.java:1169)
at
sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1105)
at
sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:999)
at
sun.net.www.protocol.http.HttpURLConnection.connect(HttpURLConnection.java:933)
at
org.apache.hadoop.security.authentication.client.KerberosAuthenticator.a
uthenticate(KerberosAuthenticator.java:190)
at
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthentic
ator.authenticate(DelegationTokenAuthenticator.java:128)
at
org.apache.hadoop.security.authentication.client.AuthenticatedURL.openCo
nnection(AuthenticatedURL.java:215)
at
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthentic
atedURL.openConnection(DelegationTokenAuthenticatedURL.java:322)
at
org.apache.hadoop.crypto.key.kms.KMSClientProvider$1.run(KMSClientProvid
er.java:483)
at
org.apache.hadoop.crypto.key.kms.KMSClientProvider$1.run(KMSClientProvid
er.java:478)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformatio
n.java:1657)
at
org.apache.hadoop.crypto.key.kms.KMSClientProvider.createConnection(KMSC
lientProvider.java:478)
at
org.apache.hadoop.crypto.key.kms.KMSClientProvider.decryptEncryptedKey(K
MSClientProvider.java:776)
at
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.decryptEncrypted
Key(KeyProviderCryptoExtension.java:388)
at
org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(DFSCl
ient.java:1381)
at
org.apache.hadoop.hdfs.DFSClient.createWrappedInputStream(DFSClient.java
:1451)
at
org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSys
tem.java:305)
at
org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSys
tem.java:299)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolv
er.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.
java:312)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:767)
at org.apache.hadoop.hbase.util.FSUtils.getVersion(FSUtils.java:572)
at
org.apache.hadoop.hbase.util.FSUtils.checkVersion(FSUtils.java:657)
at
org.apache.hadoop.hbase.master.MasterFileSystem.checkRootDir(MasterFileS
ystem.java:462)
at
org.apache.hadoop.hbase.master.MasterFileSystem.createInitialFileSystemL
ayout(MasterFileSystem.java:153)
at
org.apache.hadoop.hbase.master.MasterFileSystem.(MasterFileSystem.
java:128)
at
org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(
HMaster.java:638)
at
org.apache.hadoop.hbase.master.HMaster.access$500(HMaster.java:184)
at org.apache.hadoop.hbase.master.HMaster$1.run(HMaster.java:1729)
at java.lang.Thread.run(Thread.java:745)
2017-01-25 20:02:54,366 FATAL
[atl-stg-big-01c:16000.activeMasterManager] master.HMaster: Unhandled
exception. Starting shutdown.
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:
350)
at
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImp
l.java:206)
at
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:18
8)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at sun.net.NetworkClient.doConnect(NetworkClient.java:175)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
at sun.net.www.http.HttpClient.(HttpClient.java:211)
at sun.net.www.http.HttpClient.New(HttpClient.java:308)
at sun.net.www.http.HttpClient.New(HttpClient.java:326)
at
sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient(HttpURLConnection.java:1169)
at
sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1105)
at
sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:999)
at
sun.net.www.protocol.http.HttpURLConnection.connect(HttpURLConnection.java:933)
at
org.apache.hadoop.security.authentication.client.KerberosAuthenticator.a
uthenticate(KerberosAuthenticator.java:190)
at
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthentic
ator.authenticate(DelegationTokenAuthenticator.java:128)
at
org.apache.hadoop.security.authentication.client.AuthenticatedURL.openCo
nnection(AuthenticatedURL.java:215)
at
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthentic
atedURL.openConnection(DelegationTokenAuthenticatedURL.java:322)
at
org.apache.hadoop.crypto.key.kms.KMSClientProvider$1.run(KMSClientProvid
er.java:483)
at
org.apache.hadoop.crypto.key.kms.KMSClientProvider$1.run(KMSClientProvid
er.java:478)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformatio
n.java:1657)
at
org.apache.hadoop.crypto.key.kms.KMSClientProvider.createConnection(KMSC
lientProvider.java:478)
at
org.apache.hadoop.crypto.key.kms.KMSClientProvider.decryptEncryptedKey(K
MSClientProvider.java:776)
at
org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.decryptEncrypted
Key(KeyProviderCryptoExtension.java:388)
at
org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(DFSCl
ient.java:1381)
at
org.apache.hadoop.hdfs.DFSClient.createWrappedInputStream(DFSClient.java
:1451)
at
org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSys
tem.java:305)
at
org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSys
tem.java:299)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolv
er.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.
java:312)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:767)
at org.apache.hadoop.hbase.util.FSUtils.getVersion(FSUtils.java:572)
at
org.apache.hadoop.hbase.util.FSUtils.checkVersion(FSUtils.java:657)
at
org.apache.hadoop.hbase.master.MasterFileSystem.checkRootDir(MasterFileS
ystem.java:462)
at
org.apache.hadoop.hbase.master.MasterFileSystem.createInitialFileSystemL
ayout(MasterFileSystem.java:153)
at
org.apache.hadoop.hbase.master.MasterFileSystem.(MasterFileSystem.
java:128)
at
org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(
HMaster.java:638)
at
org.apache.hadoop.hbase.master.HMaster.access$500(HMaster.java:184)
at org.apache.hadoop.hbase.master.HMaster$1.run(HMaster.java:1729)
at java.lang.Thread.run(Thread.java:745)
2017-01-25 20:02:54,366 INFO
[atl-stg-big-01c:16000.activeMasterManager] regionserver.HRegionServer:
STOPPED: Unhandled exception. Starting shutdown.

How to fix this?

Related:

Leave a Reply