2024-11-15 23:42:02,797 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-15 23:42:02,808 main DEBUG Took 0.009299 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 23:42:02,809 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 23:42:02,809 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 23:42:02,810 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 23:42:02,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,820 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 23:42:02,833 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,835 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,837 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,837 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,838 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,839 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,840 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,841 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,841 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,842 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,842 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,843 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,843 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,844 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,844 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,845 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,845 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,846 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,846 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,847 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 23:42:02,847 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,848 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 23:42:02,850 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 23:42:02,851 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 23:42:02,853 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 23:42:02,854 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 23:42:02,855 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 23:42:02,855 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 23:42:02,863 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 23:42:02,866 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 23:42:02,868 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 23:42:02,868 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 23:42:02,869 main DEBUG createAppenders(={Console}) 2024-11-15 23:42:02,870 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 initialized 2024-11-15 23:42:02,870 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-15 23:42:02,871 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 OK. 2024-11-15 23:42:02,871 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 23:42:02,872 main DEBUG OutputStream closed 2024-11-15 23:42:02,872 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 23:42:02,873 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 23:42:02,873 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK 2024-11-15 23:42:02,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 23:42:02,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 23:42:02,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 23:42:02,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 23:42:02,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 23:42:02,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 23:42:02,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 23:42:02,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 23:42:02,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 23:42:02,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 23:42:02,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 23:42:02,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 23:42:02,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 23:42:02,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 23:42:02,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 23:42:02,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 23:42:02,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 23:42:02,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 23:42:02,953 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 23:42:02,954 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5812f68b) with optional ClassLoader: null 2024-11-15 23:42:02,954 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 23:42:02,955 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5812f68b] started OK. 2024-11-15T23:42:03,212 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8c43351e-72f6-e3ea-2f40-0dbc3152f640 2024-11-15 23:42:03,216 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 23:42:03,216 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T23:42:03,231 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestSecureIPC timeout: 52 mins 2024-11-15T23:42:03,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=13, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5959 2024-11-15T23:42:03,451 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T23:42:03,507 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:03,510 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:03,548 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714123545,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:04,214 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:04,311 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T23:42:04,352 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T23:42:04,363 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T23:42:04,384 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40271 2024-11-15T23:42:04,391 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:04,391 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:04,391 INFO [Listener,port=40271 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40271: starting 2024-11-15T23:42:04,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6539498c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:04,512 DEBUG [Thread-6 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:04,566 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40271 2024-11-15T23:42:04,575 DEBUG [Thread-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:04,623 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714124622,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:04,634 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:04,640 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:04,646 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:04,646 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:04,654 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:04,656 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:04,657 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:04,658 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:04,658 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:04,659 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:04,660 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:04,661 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:04,661 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-15T23:42:04,661 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:04,662 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:04,662 DEBUG [Thread-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:04,663 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:04,663 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:04,663 INFO [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:04,681 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-15T23:42:04,681 INFO [Reader=1,bindAddress=localhost,port=40271 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59996, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:04,686 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:246 2024-11-15T23:42:04,688 DEBUG [BRPC Connection (290841831) to localhost:40271 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-15T23:42:04,723 DEBUG [BRPC Connection (290841831) to localhost:40271 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:232 2024-11-15T23:42:04,727 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2231 2024-11-15T23:42:04,729 DEBUG [BRPC Connection (290841831) to localhost:40271 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-15T23:42:04,732 DEBUG [BRPC Connection (290841831) to localhost:40271 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2217 2024-11-15T23:42:04,745 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-15T23:42:04,747 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18147 2024-11-15T23:42:04,755 DEBUG [BRPC Connection (290841831) to localhost:40271 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-15T23:42:04,767 DEBUG [BRPC Connection (290841831) to localhost:40271 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22250 2024-11-15T23:42:04,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:04,773 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40271 2024-11-15T23:42:04,773 INFO [Listener,port=40271 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40271: stopping 2024-11-15T23:42:04,773 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:04,773 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:04,781 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 13) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-3-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=290 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5848 (was 5959) 2024-11-15T23:42:04,788 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5848 2024-11-15T23:42:04,808 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:04,808 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:04,816 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714124815,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:04,834 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:04,835 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36853 2024-11-15T23:42:04,836 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:04,836 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:04,836 INFO [Listener,port=36853 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36853: starting 2024-11-15T23:42:04,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7519c9bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-15T23:42:04,838 DEBUG [Thread-11 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:04,839 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36853 2024-11-15T23:42:04,840 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:04,853 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714124852,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:04,855 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:04,856 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:04,857 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:36853 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-15T23:42:04,861 WARN [Thread-11 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731714124818 2024-11-15T23:42:04,952 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:04,954 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:04,954 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:04,955 DEBUG [Thread-11 {}] ipc.FailedServers(52): Added failed server with address localhost:36853 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:04,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:04,956 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36853 2024-11-15T23:42:04,957 INFO [Listener,port=36853 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36853: stopping 2024-11-15T23:42:04,957 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:04,957 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:04,963 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5836 (was 5848) 2024-11-15T23:42:04,969 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5836 2024-11-15T23:42:04,983 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:04,983 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:04,991 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714124990,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:04,995 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:04,996 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39579 2024-11-15T23:42:04,997 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:04,997 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:04,997 INFO [Listener,port=39579 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39579: starting 2024-11-15T23:42:04,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26717c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:04,998 DEBUG [Thread-14 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:04,999 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39579 2024-11-15T23:42:05,000 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,008 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714125007,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,010 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext. 2024-11-15T23:42:05,010 DEBUG [Reader=1,bindAddress=localhost,port=39579 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,012 DEBUG [Reader=1,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,012 DEBUG [Reader=1,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=498 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,013 DEBUG [Reader=1,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 107 from saslServer. 2024-11-15T23:42:05,014 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 107 for processing by initSASLContext 2024-11-15T23:42:05,014 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,014 DEBUG [Reader=1,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,015 DEBUG [Reader=1,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,015 DEBUG [Reader=1,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,015 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,015 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:05,016 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:39579 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-15T23:42:05,016 WARN [Thread-14 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731714124993 2024-11-15T23:42:05,019 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,021 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext. 2024-11-15T23:42:05,021 DEBUG [Reader=2,bindAddress=localhost,port=39579 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,022 DEBUG [Reader=2,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,023 DEBUG [Reader=2,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=498 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,024 DEBUG [Reader=2,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 107 from saslServer. 2024-11-15T23:42:05,024 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 107 for processing by initSASLContext 2024-11-15T23:42:05,025 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,025 DEBUG [Reader=2,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,025 DEBUG [Reader=2,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,026 DEBUG [Reader=2,bindAddress=localhost,port=39579 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,026 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,026 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:05,026 DEBUG [Thread-14 {}] ipc.FailedServers(52): Added failed server with address localhost:39579 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,027 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39579 2024-11-15T23:42:05,027 INFO [Listener,port=39579 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39579: stopping 2024-11-15T23:42:05,027 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,028 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,034 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.fetchStackFrames(StackStreamFactory.java:397) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.getNextBatch(StackStreamFactory.java:322) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.peekFrame(StackStreamFactory.java:263) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.hasNext(StackStreamFactory.java:351) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:593) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007fc13418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5824 (was 5836) 2024-11-15T23:42:05,039 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5824 2024-11-15T23:42:05,052 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:05,053 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:05,059 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714125059,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:05,063 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,064 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34759 2024-11-15T23:42:05,065 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,067 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,067 INFO [Listener,port=34759 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34759: starting 2024-11-15T23:42:05,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1feb7ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,068 DEBUG [Thread-17 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:05,069 DEBUG [Thread-17 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34759 2024-11-15T23:42:05,071 DEBUG [Thread-17 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,079 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714125079,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,082 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:05,083 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,084 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,084 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,086 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:05,087 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:05,088 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,088 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,089 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,089 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,089 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,090 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:05,090 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:05,090 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,090 DEBUG [Thread-17 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:05,090 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,091 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,091 DEBUG [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:05,091 INFO [Reader=1,bindAddress=localhost,port=34759 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:05,091 INFO [Reader=1,bindAddress=localhost,port=34759 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37202, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:05,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,118 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34759 2024-11-15T23:42:05,118 INFO [Listener,port=34759 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34759: stopping 2024-11-15T23:42:05,118 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,118 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,119 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,120 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39343 2024-11-15T23:42:05,121 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,121 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,121 INFO [Listener,port=39343 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39343: starting 2024-11-15T23:42:05,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9d319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,123 DEBUG [Thread-20 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:05,123 DEBUG [Thread-20 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39343 2024-11-15T23:42:05,124 DEBUG [Thread-20 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,126 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:05,126 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,127 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,127 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,128 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:05,129 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:05,129 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,130 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,130 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,130 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,131 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,131 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:05,131 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:05,131 DEBUG [Thread-20 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:05,131 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,131 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,132 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,132 DEBUG [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:05,132 INFO [Reader=1,bindAddress=localhost,port=39343 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:05,133 INFO [Reader=1,bindAddress=localhost,port=39343 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42704, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:05,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,152 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39343 2024-11-15T23:42:05,152 INFO [Listener,port=39343 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39343: stopping 2024-11-15T23:42:05,152 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,152 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,152 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,154 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39295 2024-11-15T23:42:05,154 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,154 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,155 INFO [Listener,port=39295 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39295: starting 2024-11-15T23:42:05,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8c3651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,156 DEBUG [Thread-23 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:05,156 DEBUG [Thread-23 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39295 2024-11-15T23:42:05,157 DEBUG [Thread-23 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,159 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:05,159 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,160 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,160 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,162 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:05,162 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:05,162 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,163 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,163 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,163 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,164 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,164 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:05,164 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:05,164 DEBUG [Thread-23 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:05,164 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,165 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,165 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,165 DEBUG [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:05,166 INFO [Reader=1,bindAddress=localhost,port=39295 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:05,166 INFO [Reader=1,bindAddress=localhost,port=39295 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35862, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:05,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,179 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39295 2024-11-15T23:42:05,179 INFO [Listener,port=39295 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39295: stopping 2024-11-15T23:42:05,179 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,179 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,180 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,182 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34205 2024-11-15T23:42:05,182 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,182 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,182 INFO [Listener,port=34205 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34205: starting 2024-11-15T23:42:05,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268decc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,183 DEBUG [Thread-26 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:05,184 DEBUG [Thread-26 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34205 2024-11-15T23:42:05,185 DEBUG [Thread-26 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,186 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:05,187 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,188 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,188 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,189 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:05,189 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:05,190 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,190 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,190 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,190 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,191 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,191 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:05,191 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-15T23:42:05,191 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,191 DEBUG [Thread-26 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:05,191 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,192 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,192 DEBUG [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-15T23:42:05,192 INFO [Reader=1,bindAddress=localhost,port=34205 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:05,192 INFO [Reader=1,bindAddress=localhost,port=34205 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60512, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:05,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,204 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34205 2024-11-15T23:42:05,204 INFO [Listener,port=34205 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34205: stopping 2024-11-15T23:42:05,204 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,205 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,205 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,206 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34643 2024-11-15T23:42:05,207 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,207 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,207 INFO [Listener,port=34643 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34643: starting 2024-11-15T23:42:05,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4efc1118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,208 DEBUG [Thread-29 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:05,209 DEBUG [Thread-29 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34643 2024-11-15T23:42:05,210 DEBUG [Thread-29 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,211 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:05,211 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,212 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,212 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,214 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:05,214 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:05,214 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,214 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,214 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,215 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,215 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,216 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:05,216 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-15T23:42:05,216 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,216 DEBUG [Thread-29 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:05,216 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,216 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,216 DEBUG [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:05,217 INFO [Reader=1,bindAddress=localhost,port=34643 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:05,217 INFO [Reader=1,bindAddress=localhost,port=34643 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43386, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:05,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,234 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34643 2024-11-15T23:42:05,235 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,235 INFO [Listener,port=34643 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34643: stopping 2024-11-15T23:42:05,235 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/jdk.internal.misc.Unsafe.bool2byte(Unsafe.java:1777) java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetBoolean(Unsafe.java:1784) java.base@17.0.11/java.lang.invoke.VarHandleBooleans$FieldInstanceReadWrite.compareAndSet(VarHandleBooleans.java:178) java.base@17.0.11/java.lang.invoke.VarHandleGuards.guard_LII_Z(VarHandleGuards.java:137) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:125) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5809 (was 5824) 2024-11-15T23:42:05,246 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5809 2024-11-15T23:42:05,260 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:05,261 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:05,267 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714125266,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:05,270 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,272 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44001 2024-11-15T23:42:05,272 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,272 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,272 INFO [Listener,port=44001 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44001: starting 2024-11-15T23:42:05,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea118df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,274 DEBUG [Thread-32 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-15T23:42:05,275 DEBUG [Thread-32 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44001 2024-11-15T23:42:05,278 DEBUG [BRPC Connection (290841831) to localhost:44001 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-15T23:42:05,278 WARN [BRPC Connection (290841831) to localhost:44001 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-15T23:42:05,280 DEBUG [BRPC Connection (290841831) to localhost:44001 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-15T23:42:05,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,280 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44001 2024-11-15T23:42:05,281 INFO [Listener,port=44001 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44001: stopping 2024-11-15T23:42:05,281 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,281 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,290 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5799 (was 5809) 2024-11-15T23:42:05,296 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5799 2024-11-15T23:42:05,310 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:05,310 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:05,317 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714125316,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:05,320 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,322 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38083 2024-11-15T23:42:05,322 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-15T23:42:05,322 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-15T23:42:05,322 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-15T23:42:05,323 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-15T23:42:05,323 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-15T23:42:05,323 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-15T23:42:05,323 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-15T23:42:05,323 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-15T23:42:05,323 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,323 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,323 INFO [Listener,port=38083 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38083: starting 2024-11-15T23:42:05,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea8e507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,325 DEBUG [Thread-35 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-15T23:42:05,325 DEBUG [Thread-35 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38083 2024-11-15T23:42:05,326 WARN [Reader=1,bindAddress=localhost,port=38083 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-15T23:42:05,326 INFO [Reader=1,bindAddress=localhost,port=38083 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-15T23:42:05,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,334 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38083 2024-11-15T23:42:05,335 INFO [Listener,port=38083 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38083: stopping 2024-11-15T23:42:05,335 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,335 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,341 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5787 (was 5799) 2024-11-15T23:42:05,346 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5787 2024-11-15T23:42:05,359 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:05,359 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:05,365 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714125365,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:05,368 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,370 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43271 2024-11-15T23:42:05,373 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,373 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d680bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,373 INFO [Listener,port=43271 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43271: starting 2024-11-15T23:42:05,375 DEBUG [Thread-38 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:05,375 DEBUG [Thread-38 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43271 2024-11-15T23:42:05,377 DEBUG [Thread-38 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,385 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714125385,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,386 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:05,387 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,388 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,388 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,389 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:05,389 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:05,390 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,390 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,390 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,390 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,390 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,391 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:05,391 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:05,391 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,391 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,391 DEBUG [Thread-38 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:05,391 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,391 DEBUG [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:05,392 INFO [Reader=1,bindAddress=localhost,port=43271 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:05,392 INFO [Reader=1,bindAddress=localhost,port=43271 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41492, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:05,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,400 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43271 2024-11-15T23:42:05,400 INFO [Listener,port=43271 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43271: stopping 2024-11-15T23:42:05,400 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,401 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,408 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.fetchStackFrames(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.fetchStackFrames(StackStreamFactory.java:386) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.getNextBatch(StackStreamFactory.java:322) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.peekFrame(StackStreamFactory.java:263) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.hasNext(StackStreamFactory.java:351) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:593) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007fc13418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RpcServer idle connection scanner for port 43271 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Reader=2,bindAddress=localhost,port=43271 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Listener,port=43271 java.base@17.0.11/java.lang.ref.Reference.clear0(Native Method) java.base@17.0.11/java.lang.ref.Reference.clear(Reference.java:389) java.base@17.0.11/jdk.internal.ref.PhantomCleanable.clean(PhantomCleanable.java:132) java.base@17.0.11/java.util.Timer.cancel(Timer.java:443) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$ConnectionManager.stopIdleScan(SimpleRpcServer.java:633) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:265) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5772 (was 5787) 2024-11-15T23:42:05,414 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5772 2024-11-15T23:42:05,427 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:05,428 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:05,433 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714125433,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:05,436 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,437 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37617 2024-11-15T23:42:05,438 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,438 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,438 INFO [Listener,port=37617 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37617: starting 2024-11-15T23:42:05,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d90d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,439 DEBUG [Thread-41 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:05,440 DEBUG [Thread-41 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37617 2024-11-15T23:42:05,441 DEBUG [Thread-41 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,448 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714125448,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,449 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:05,450 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,451 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,451 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,452 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:05,452 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:05,452 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,452 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,452 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,453 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,453 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,453 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:05,453 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-15T23:42:05,453 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,454 DEBUG [Thread-41 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:05,454 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,454 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,454 DEBUG [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:05,454 INFO [Reader=1,bindAddress=localhost,port=37617 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:05,455 INFO [Reader=1,bindAddress=localhost,port=37617 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60922, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:05,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:05,467 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37617 2024-11-15T23:42:05,468 INFO [Listener,port=37617 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37617: stopping 2024-11-15T23:42:05,468 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:05,468 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:05,468 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:05,473 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46379 2024-11-15T23:42:05,476 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:05,477 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:05,477 INFO [Listener,port=46379 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46379: starting 2024-11-15T23:42:05,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@785d0423, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:05,478 DEBUG [Thread-44 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:05,479 DEBUG [Thread-44 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46379 2024-11-15T23:42:05,480 DEBUG [Thread-44 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,481 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:05,481 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,482 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,482 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,484 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:05,484 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:05,484 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:05,484 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,485 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,485 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:05,485 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:05,485 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:05,486 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-15T23:42:05,486 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:05,486 DEBUG [Thread-44 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:05,486 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:05,486 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:05,486 DEBUG [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:05,486 INFO [Reader=1,bindAddress=localhost,port=46379 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:05,487 INFO [Reader=1,bindAddress=localhost,port=46379 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47316, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:07,489 ERROR [Thread-44 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:47316 remote=localhost/127.0.0.1:46379] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-15T23:42:07,490 DEBUG [Thread-44 {}] ipc.FailedServers(52): Added failed server with address localhost:46379 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-15T23:42:07,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:07,490 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46379 2024-11-15T23:42:07,490 INFO [Listener,port=46379 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46379: stopping 2024-11-15T23:42:07,491 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:07,491 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:07,498 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.pattern.DatePatternConverter.format(DatePatternConverter.java:265) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:240) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:225) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5785 (was 5772) - AvailableMemoryMB LEAK? - 2024-11-15T23:42:07,505 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5785 2024-11-15T23:42:07,523 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:07,523 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:07,528 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714127528,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:07,531 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:07,532 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=32987 2024-11-15T23:42:07,533 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:07,533 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:07,533 INFO [Listener,port=32987 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=32987: starting 2024-11-15T23:42:07,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@430ca033, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:07,534 DEBUG [Thread-47 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:07,534 DEBUG [Thread-47 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:32987 2024-11-15T23:42:07,536 DEBUG [Thread-47 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,543 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714127542,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,544 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:07,545 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,546 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:07,546 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:07,548 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:07,548 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:07,548 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:07,549 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:07,549 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:07,549 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:07,549 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:07,550 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:07,550 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:07,550 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:07,550 DEBUG [Thread-47 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:07,550 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:07,550 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,551 DEBUG [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:07,551 INFO [Reader=1,bindAddress=localhost,port=32987 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:07,551 INFO [Reader=1,bindAddress=localhost,port=32987 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59504, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:07,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:07,561 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 32987 2024-11-15T23:42:07,561 INFO [Listener,port=32987 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=32987: stopping 2024-11-15T23:42:07,562 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:07,562 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:07,569 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 19) Potentially hanging thread: RpcServer idle connection scanner for port 32987 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.util.StackLocator$$Lambda$126/0x00007fc13418c4a8.(Unknown Source) java.base@17.0.11/java.lang.invoke.DirectMethodHandle$Holder.newInvokeSpecial(DirectMethodHandle$Holder) java.base@17.0.11/java.lang.invoke.Invokers$Holder.linkToTargetMethod(Invokers$Holder) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:98) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007fc13418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5768 (was 5785) 2024-11-15T23:42:07,575 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5768 2024-11-15T23:42:07,593 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:07,593 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:07,599 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714127599,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:07,602 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:07,603 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35089 2024-11-15T23:42:07,604 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:07,604 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:07,604 INFO [Listener,port=35089 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35089: starting 2024-11-15T23:42:07,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc832f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:07,605 DEBUG [Thread-50 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:07,606 DEBUG [Thread-50 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35089 2024-11-15T23:42:07,607 DEBUG [Thread-50 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,614 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714127613,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,615 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:07,616 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,617 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:07,617 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:07,618 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:07,618 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:07,618 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:07,619 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:07,619 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:07,619 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:07,619 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:07,619 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:07,620 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:07,620 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:07,620 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:07,620 DEBUG [Thread-50 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:07,620 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,620 DEBUG [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:07,620 INFO [Reader=1,bindAddress=localhost,port=35089 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:07,620 INFO [Reader=1,bindAddress=localhost,port=35089 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34274, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:07,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:07,628 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35089 2024-11-15T23:42:07,629 INFO [Listener,port=35089 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35089: stopping 2024-11-15T23:42:07,629 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:07,629 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:07,636 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19) Potentially hanging thread: Reader=0,bindAddress=localhost,port=35089 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.impl.ContextDataFactory.emptyFrozenContextData(ContextDataFactory.java:142) app//org.apache.logging.log4j.core.impl.ThreadContextDataInjector$ForDefaultThreadContextMap.injectContextData(ThreadContextDataInjector.java:134) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.createContextData(Log4jLogEvent.java:497) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.(Log4jLogEvent.java:355) app//org.apache.logging.log4j.core.impl.DefaultLogEventFactory.createEvent(DefaultLogEventFactory.java:74) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:550) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: Reader=1,bindAddress=localhost,port=35089 java.base@17.0.11/java.lang.ThreadLocal.get(ThreadLocal.java:163) java.base@17.0.11/jdk.internal.misc.TerminatingThreadLocal._threadTerminated(TerminatingThreadLocal.java:74) java.base@17.0.11/jdk.internal.misc.TerminatingThreadLocal.threadTerminated(TerminatingThreadLocal.java:70) java.base@17.0.11/java.lang.Thread.exit(Thread.java:850) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5751 (was 5768) 2024-11-15T23:42:07,642 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5750 2024-11-15T23:42:07,660 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:07,661 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:07,666 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714127665,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:07,669 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:07,673 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42989 2024-11-15T23:42:07,674 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:07,674 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:07,674 INFO [Listener,port=42989 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42989: starting 2024-11-15T23:42:07,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f9d890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:07,676 DEBUG [Thread-53 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:07,676 DEBUG [Thread-53 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42989 2024-11-15T23:42:07,677 DEBUG [Thread-53 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,684 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714127684,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,686 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:07,686 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-15T23:42:07,686 DEBUG [Thread-53 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:07,686 INFO [Reader=1,bindAddress=localhost,port=42989 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-15T23:42:07,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:07,698 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42989 2024-11-15T23:42:07,698 INFO [Listener,port=42989 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42989: stopping 2024-11-15T23:42:07,701 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:07,701 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:07,707 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19) Potentially hanging thread: RpcServer idle connection scanner for port 42989 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Reader=2,bindAddress=localhost,port=42989 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) - Thread LEAK? -, OpenFileDescriptor=294 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5733 (was 5750) 2024-11-15T23:42:07,714 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5732 2024-11-15T23:42:07,732 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:07,732 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:07,738 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714127738,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:07,742 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:07,760 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:07,765 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T23:42:07,808 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3474 (auto-detected) 2024-11-15T23:42:07,809 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T23:42:07,830 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45997 2024-11-15T23:42:07,831 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:07,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d45ef82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:07,832 DEBUG [Thread-54 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:07,833 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45997 2024-11-15T23:42:07,834 DEBUG [Thread-54 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,840 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714127840,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,842 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:07,891 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:07,891 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,893 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:07,896 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:07,897 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:07,897 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:07,898 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:07,898 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:07,898 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:07,899 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-15T23:42:07,899 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:07,899 DEBUG [Thread-54 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:07,899 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,899 DEBUG [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:07,899 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:07,906 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40570, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:07,906 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-15T23:42:07,906 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:250 2024-11-15T23:42:07,907 DEBUG [BRPC Connection (290841831) to localhost:45997 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-15T23:42:07,909 DEBUG [BRPC Connection (290841831) to localhost:45997 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:236 2024-11-15T23:42:07,910 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2243 2024-11-15T23:42:07,910 DEBUG [BRPC Connection (290841831) to localhost:45997 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-15T23:42:07,912 DEBUG [BRPC Connection (290841831) to localhost:45997 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2229 2024-11-15T23:42:07,914 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-15T23:42:07,914 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18311 2024-11-15T23:42:07,915 DEBUG [BRPC Connection (290841831) to localhost:45997 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-15T23:42:07,922 DEBUG [BRPC Connection (290841831) to localhost:45997 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22414 2024-11-15T23:42:07,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:07,923 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45997 2024-11-15T23:42:07,933 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21 (was 19) Potentially hanging thread: NettyRpcServer-2-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-2-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=299 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5668 (was 5732) 2024-11-15T23:42:07,938 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5667 2024-11-15T23:42:07,956 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:07,956 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:07,962 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714127962,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:07,966 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:07,966 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:07,967 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43051 2024-11-15T23:42:07,967 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:07,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442792ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-15T23:42:07,968 DEBUG [Thread-55 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:07,968 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43051 2024-11-15T23:42:07,969 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,975 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714127975,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:07,976 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:07,977 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:07,977 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:43051 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-15T23:42:07,977 WARN [Thread-55 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731714127964 2024-11-15T23:42:08,041 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,043 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,043 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:08,043 DEBUG [Thread-55 {}] ipc.FailedServers(52): Added failed server with address localhost:43051 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,044 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43051 2024-11-15T23:42:08,050 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24 (was 21) Potentially hanging thread: NettyRpcServer-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=308 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5650 (was 5667) 2024-11-15T23:42:08,055 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5649 2024-11-15T23:42:08,070 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:08,070 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:08,075 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714128075,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:08,078 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,078 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,080 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46435 2024-11-15T23:42:08,080 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5608b9e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,081 DEBUG [Thread-56 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:08,081 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46435 2024-11-15T23:42:08,082 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,088 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714128087,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,089 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,089 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,089 DEBUG [NettyRpcServer-4-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,091 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,091 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,092 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,092 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,092 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,093 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,093 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:08,093 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:46435 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-15T23:42:08,093 WARN [Thread-56 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731714128076 2024-11-15T23:42:08,141 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,142 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,143 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,143 DEBUG [NettyRpcServer-4-3 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,144 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,145 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,145 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,145 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,145 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,146 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,146 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:08,146 DEBUG [Thread-56 {}] ipc.FailedServers(52): Added failed server with address localhost:46435 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,147 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46435 2024-11-15T23:42:08,154 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27 (was 24) Potentially hanging thread: NettyRpcServer-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=317 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5631 (was 5649) 2024-11-15T23:42:08,160 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5630 2024-11-15T23:42:08,173 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:08,173 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:08,178 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714128178,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:08,181 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,182 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,183 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40897 2024-11-15T23:42:08,183 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e94926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,184 DEBUG [Thread-57 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:08,185 DEBUG [Thread-57 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40897 2024-11-15T23:42:08,186 DEBUG [Thread-57 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,192 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714128192,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,194 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext. 2024-11-15T23:42:08,194 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=498 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,194 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,196 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,196 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,197 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,197 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,197 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,197 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,198 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:08,198 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:08,198 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,198 DEBUG [Thread-57 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:08,198 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,198 DEBUG [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:08,198 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:08,199 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38004, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:08,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,210 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40897 2024-11-15T23:42:08,211 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,212 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,213 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42207 2024-11-15T23:42:08,213 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6125d6f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,214 DEBUG [Thread-58 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:08,214 DEBUG [Thread-58 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42207 2024-11-15T23:42:08,216 DEBUG [Thread-58 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,217 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,217 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,217 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,219 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,219 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,220 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,220 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,220 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,220 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,221 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:08,221 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:08,221 DEBUG [Thread-58 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:08,221 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,221 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,221 DEBUG [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:08,221 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:08,222 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39420, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:08,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,230 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42207 2024-11-15T23:42:08,231 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,231 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,232 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38707 2024-11-15T23:42:08,232 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@352f737e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,234 DEBUG [Thread-59 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:08,234 DEBUG [Thread-59 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38707 2024-11-15T23:42:08,235 DEBUG [Thread-59 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,236 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,236 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,236 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,238 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,239 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,239 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,239 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,239 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,240 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,240 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:08,240 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:08,240 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,240 DEBUG [Thread-59 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:08,240 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,241 DEBUG [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:08,241 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:08,241 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43872, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:08,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,249 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38707 2024-11-15T23:42:08,250 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,250 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,251 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34829 2024-11-15T23:42:08,251 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28385864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,252 DEBUG [Thread-60 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:08,252 DEBUG [Thread-60 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34829 2024-11-15T23:42:08,253 DEBUG [Thread-60 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,254 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,254 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,255 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,257 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,257 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,257 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,258 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,258 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,258 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,258 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:08,260 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-15T23:42:08,260 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,260 DEBUG [Thread-60 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:08,260 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,260 DEBUG [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-15T23:42:08,260 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:08,261 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59616, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:08,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,272 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34829 2024-11-15T23:42:08,272 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,273 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,274 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38567 2024-11-15T23:42:08,274 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4cec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,275 DEBUG [Thread-61 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:08,275 DEBUG [Thread-61 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38567 2024-11-15T23:42:08,276 DEBUG [Thread-61 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,278 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,278 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,278 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,280 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,280 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,280 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,283 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,283 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,283 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,284 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:08,284 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-15T23:42:08,284 DEBUG [Thread-61 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:08,286 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,286 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,286 DEBUG [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:08,287 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:08,287 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47298, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:08,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,300 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38567 2024-11-15T23:42:08,307 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37 (was 27) Potentially hanging thread: NettyRpcServer-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=362 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5595 (was 5630) 2024-11-15T23:42:08,314 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5594 2024-11-15T23:42:08,328 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:08,329 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:08,334 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714128333,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:08,336 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,337 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,338 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42471 2024-11-15T23:42:08,338 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4fb2e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,339 DEBUG [Thread-62 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-15T23:42:08,340 DEBUG [Thread-62 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42471 2024-11-15T23:42:08,342 DEBUG [BRPC Connection (290841831) to localhost:42471 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-15T23:42:08,342 WARN [BRPC Connection (290841831) to localhost:42471 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-15T23:42:08,342 DEBUG [BRPC Connection (290841831) to localhost:42471 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-15T23:42:08,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,343 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42471 2024-11-15T23:42:08,349 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 37) Potentially hanging thread: NettyRpcServer-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=371 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5583 (was 5594) 2024-11-15T23:42:08,355 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5583 2024-11-15T23:42:08,369 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:08,370 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:08,375 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714128374,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:08,377 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,378 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,379 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43077 2024-11-15T23:42:08,379 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-15T23:42:08,379 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-15T23:42:08,379 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-15T23:42:08,379 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-15T23:42:08,379 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-15T23:42:08,380 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-15T23:42:08,380 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-15T23:42:08,380 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-15T23:42:08,380 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3211499d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,381 DEBUG [Thread-63 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-15T23:42:08,381 DEBUG [Thread-63 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43077 2024-11-15T23:42:08,389 WARN [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-15T23:42:08,389 INFO [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-15T23:42:08,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,398 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43077 2024-11-15T23:42:08,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41 (was 39) Potentially hanging thread: NettyRpcServer-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=380 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5570 (was 5583) 2024-11-15T23:42:08,411 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5569 2024-11-15T23:42:08,425 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:08,426 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:08,431 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714128431,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:08,433 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,434 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,435 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33859 2024-11-15T23:42:08,435 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b6d599, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,436 DEBUG [Thread-64 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:08,437 DEBUG [Thread-64 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33859 2024-11-15T23:42:08,438 DEBUG [Thread-64 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,444 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714128444,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,445 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,446 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,446 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,447 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,448 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,448 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,448 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,449 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,449 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,449 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:08,449 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:08,449 DEBUG [Thread-64 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:08,449 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,450 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,450 DEBUG [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:08,450 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:08,450 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50204, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:08,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,464 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33859 2024-11-15T23:42:08,473 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43 (was 41) Potentially hanging thread: NettyRpcServer-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=389 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5553 (was 5569) 2024-11-15T23:42:08,483 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5553 2024-11-15T23:42:08,501 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:08,501 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:08,506 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714128506,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:08,508 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,509 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,510 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37525 2024-11-15T23:42:08,510 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f903f37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,514 DEBUG [Thread-65 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:08,514 DEBUG [Thread-65 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37525 2024-11-15T23:42:08,515 DEBUG [Thread-65 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,522 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714128522,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,524 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,524 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,524 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,526 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,527 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,527 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,527 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,527 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,528 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,528 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:08,528 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-15T23:42:08,528 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,528 DEBUG [Thread-65 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:08,528 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,528 DEBUG [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:08,529 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:08,529 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53580, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:08,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:08,541 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37525 2024-11-15T23:42:08,542 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:08,543 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:08,544 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38201 2024-11-15T23:42:08,544 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:08,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d32025f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:08,546 DEBUG [Thread-66 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:08,546 DEBUG [Thread-66 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38201 2024-11-15T23:42:08,547 DEBUG [Thread-66 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,549 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:08,551 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,551 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,553 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:08,553 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:08,554 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:08,554 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,554 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:08,555 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:08,555 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:08,555 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-15T23:42:08,555 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:08,555 DEBUG [Thread-66 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:08,556 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:08,556 DEBUG [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:08,556 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:08,556 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41886, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:10,558 ERROR [Thread-66 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41886 remote=localhost/127.0.0.1:38201] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-15T23:42:10,558 DEBUG [Thread-66 {}] ipc.FailedServers(52): Added failed server with address localhost:38201 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-15T23:42:10,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:10,559 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38201 2024-11-15T23:42:10,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47 (was 43) Potentially hanging thread: NettyRpcServer-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=407 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5520 (was 5553) 2024-11-15T23:42:10,571 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5520 2024-11-15T23:42:10,585 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:10,585 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:10,589 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714130589,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:10,592 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:10,592 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:10,593 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36457 2024-11-15T23:42:10,593 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:10,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53511d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:10,594 DEBUG [Thread-67 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:10,595 DEBUG [Thread-67 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36457 2024-11-15T23:42:10,595 DEBUG [Thread-67 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,602 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714130601,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,603 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:10,603 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,603 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,605 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:10,605 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:10,606 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:10,606 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,606 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:10,606 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:10,607 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:10,607 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:10,607 DEBUG [Thread-67 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:10,607 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,607 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,607 DEBUG [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:10,607 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:10,608 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47446, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:10,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:10,615 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36457 2024-11-15T23:42:10,621 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49 (was 47) Potentially hanging thread: NettyRpcServer-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=416 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5505 (was 5520) 2024-11-15T23:42:10,627 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5505 2024-11-15T23:42:10,642 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:10,643 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:10,647 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714130647,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:10,649 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:10,649 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:10,650 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40397 2024-11-15T23:42:10,651 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:10,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d4b109a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:10,652 DEBUG [Thread-68 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:10,652 DEBUG [Thread-68 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40397 2024-11-15T23:42:10,653 DEBUG [Thread-68 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,658 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714130657,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,659 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:10,659 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,659 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,661 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:10,661 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-15T23:42:10,661 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-15T23:42:10,662 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,662 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:10,662 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-15T23:42:10,662 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-15T23:42:10,662 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-15T23:42:10,662 DEBUG [Thread-68 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:10,662 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,663 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,663 DEBUG [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:10,663 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:10,663 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51296, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:10,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:10,670 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40397 2024-11-15T23:42:10,677 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51 (was 49) Potentially hanging thread: NettyRpcServer-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=425 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5490 (was 5505) 2024-11-15T23:42:10,683 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5489 2024-11-15T23:42:10,700 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:10,701 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:10,705 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714130705,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:10,708 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:10,709 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:10,710 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40291 2024-11-15T23:42:10,710 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:10,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad3efb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:10,712 DEBUG [Thread-69 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:10,712 DEBUG [Thread-69 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40291 2024-11-15T23:42:10,713 DEBUG [Thread-69 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,718 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714130717,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,719 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-15T23:42:10,719 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-15T23:42:10,719 DEBUG [Thread-69 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:10,719 INFO [NettyRpcServer-17-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-15T23:42:10,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:10,726 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40291 2024-11-15T23:42:10,732 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=53 (was 51) Potentially hanging thread: NettyRpcServer-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=434 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5474 (was 5489) 2024-11-15T23:42:10,737 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=53, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5474 2024-11-15T23:42:10,750 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:10,750 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:10,754 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714130754,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:10,756 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:10,757 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44841 2024-11-15T23:42:10,758 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:10,758 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:10,758 INFO [Listener,port=44841 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44841: starting 2024-11-15T23:42:10,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f2a098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:10,762 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T23:42:10,762 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T23:42:10,767 DEBUG [Thread-72 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:10,782 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,789 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714130788,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,791 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,792 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,792 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,793 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:10,794 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,794 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,794 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:10,795 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,795 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,795 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,795 DEBUG [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:10,795 INFO [Reader=1,bindAddress=localhost,port=44841 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:10,800 INFO [Reader=1,bindAddress=localhost,port=44841 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48550, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:10,801 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:10,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:10,814 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44841 2024-11-15T23:42:10,814 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:10,814 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:10,814 INFO [Listener,port=44841 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44841: stopping 2024-11-15T23:42:10,822 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=56 (was 53) Potentially hanging thread: RpcServer idle connection scanner for port 44841 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Listener,port=44841 java.base@17.0.11/sun.nio.ch.EventFD.set0(Native Method) java.base@17.0.11/sun.nio.ch.EventFD.set(EventFD.java:48) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:251) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:183) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:256) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-1 java.base@17.0.11/java.lang.ClassLoader.defineClass1(Native Method) java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1017) java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:294) app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:290) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.initialize(FastThreadLocal.java:177) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.get(FastThreadLocal.java:142) app//org.apache.hadoop.hbase.ipc.IPCUtil.execute(IPCUtil.java:304) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown(NettyRpcConnection.java:146) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:161) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 434) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5456 (was 5474) 2024-11-15T23:42:10,829 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5454 2024-11-15T23:42:10,841 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:10,842 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:10,846 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714130846,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:10,848 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:10,849 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33819 2024-11-15T23:42:10,849 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:10,849 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:10,849 INFO [Listener,port=33819 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33819: starting 2024-11-15T23:42:10,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@114c390c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-15T23:42:10,850 DEBUG [Thread-75 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:10,852 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,857 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714130857,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,859 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:10,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:10,860 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:33819 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-15T23:42:10,860 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33819 2024-11-15T23:42:10,860 INFO [Listener,port=33819 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33819: stopping 2024-11-15T23:42:10,860 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:10,860 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:10,867 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=56 (was 54) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:216) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5440 (was 5454) 2024-11-15T23:42:10,872 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5440 2024-11-15T23:42:10,885 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:10,885 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:10,890 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714130889,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:10,892 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:10,894 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38325 2024-11-15T23:42:10,894 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:10,894 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:10,894 INFO [Listener,port=38325 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38325: starting 2024-11-15T23:42:10,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bed3424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:10,896 DEBUG [Thread-78 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:10,898 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,904 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714130904,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,906 DEBUG [Reader=1,bindAddress=localhost,port=38325 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,907 DEBUG [Reader=1,bindAddress=localhost,port=38325 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,907 DEBUG [Reader=1,bindAddress=localhost,port=38325 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,907 DEBUG [Reader=1,bindAddress=localhost,port=38325 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:10,908 DEBUG [Reader=1,bindAddress=localhost,port=38325 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,908 DEBUG [Reader=1,bindAddress=localhost,port=38325 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,908 DEBUG [Reader=1,bindAddress=localhost,port=38325 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:10,910 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:10,910 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:38325 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-15T23:42:10,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:10,910 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38325 2024-11-15T23:42:10,911 INFO [Listener,port=38325 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38325: stopping 2024-11-15T23:42:10,911 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:10,911 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:10,917 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=58 (was 55) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Relogin-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.util.datetime.FixedDateFormat.formatInstant(FixedDateFormat.java:589) app//org.apache.logging.log4j.core.util.datetime.FixedDateFormat.formatInstant(FixedDateFormat.java:584) app//org.apache.logging.log4j.core.pattern.DatePatternConverter$FixedFormatter.format(DatePatternConverter.java:100) app//org.apache.logging.log4j.core.pattern.DatePatternConverter$CachedTime.(DatePatternConverter.java:155) app//org.apache.logging.log4j.core.pattern.DatePatternConverter.formatWithoutThreadLocals(DatePatternConverter.java:310) app//org.apache.logging.log4j.core.pattern.DatePatternConverter.format(DatePatternConverter.java:290) app//org.apache.logging.log4j.core.pattern.DatePatternConverter.format(DatePatternConverter.java:265) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:375) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:216) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:59) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.toByteArray(AbstractStringLayout.java:294) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:145) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5424 (was 5440) 2024-11-15T23:42:10,923 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5423 2024-11-15T23:42:10,936 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:10,936 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:10,949 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714130948,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:10,951 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:10,952 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36831 2024-11-15T23:42:10,952 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:10,952 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:10,952 INFO [Listener,port=36831 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36831: starting 2024-11-15T23:42:10,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23afe04e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:10,953 DEBUG [Thread-81 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:10,957 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,960 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714130960,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,962 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,962 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,962 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,963 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:10,964 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,964 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,964 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:10,964 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,964 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,964 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:10,965 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,965 DEBUG [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:10,965 INFO [Reader=1,bindAddress=localhost,port=36831 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:10,965 INFO [Reader=1,bindAddress=localhost,port=36831 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36598, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:10,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:10,971 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36831 2024-11-15T23:42:10,971 INFO [Listener,port=36831 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36831: stopping 2024-11-15T23:42:10,971 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:10,971 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:10,972 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:10,973 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39059 2024-11-15T23:42:10,973 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:10,973 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:10,973 INFO [Listener,port=39059 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39059: starting 2024-11-15T23:42:10,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36dabdde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:10,974 DEBUG [Thread-84 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:10,975 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,977 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,977 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,977 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,978 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:10,978 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,978 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,978 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:10,979 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,979 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,979 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:10,979 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,979 DEBUG [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:10,979 INFO [Reader=1,bindAddress=localhost,port=39059 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:10,980 INFO [Reader=1,bindAddress=localhost,port=39059 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42714, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:10,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:10,986 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39059 2024-11-15T23:42:10,986 INFO [Listener,port=39059 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39059: stopping 2024-11-15T23:42:10,986 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:10,986 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:10,986 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:10,987 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42945 2024-11-15T23:42:10,988 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:10,988 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:10,988 INFO [Listener,port=42945 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42945: starting 2024-11-15T23:42:10,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c6c61d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:10,989 DEBUG [Thread-87 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:10,990 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,991 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,992 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,992 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,992 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:10,993 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731714130950 2024-11-15T23:42:10,993 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,993 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,993 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:10,994 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:10,994 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:10,994 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:10,994 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:10,994 DEBUG [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:10,994 INFO [Reader=1,bindAddress=localhost,port=42945 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:10,994 INFO [Reader=1,bindAddress=localhost,port=42945 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35936, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:11,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:11,000 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42945 2024-11-15T23:42:11,000 INFO [Listener,port=42945 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42945: stopping 2024-11-15T23:42:11,000 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:11,001 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:11,001 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:11,002 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33669 2024-11-15T23:42:11,002 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:11,002 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:11,002 INFO [Listener,port=33669 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33669: starting 2024-11-15T23:42:11,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6fdbb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:11,003 DEBUG [Thread-90 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:11,004 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,006 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,006 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,006 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=498 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,007 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 107 from saslServer. 2024-11-15T23:42:11,008 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,008 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,008 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:11,008 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,008 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,009 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,009 DEBUG [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-15T23:42:11,009 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:11,009 INFO [Reader=1,bindAddress=localhost,port=33669 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:11,010 INFO [Reader=1,bindAddress=localhost,port=33669 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59348, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:11,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:11,017 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33669 2024-11-15T23:42:11,017 INFO [Listener,port=33669 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33669: stopping 2024-11-15T23:42:11,017 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:11,017 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:11,018 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:11,019 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46675 2024-11-15T23:42:11,020 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:11,020 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:11,020 INFO [Listener,port=46675 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46675: starting 2024-11-15T23:42:11,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292dea84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:11,021 DEBUG [Thread-93 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:11,022 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,023 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,024 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,024 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=498 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,025 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 107 from saslServer. 2024-11-15T23:42:11,026 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,026 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,026 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:11,026 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,026 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,026 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:11,026 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,027 DEBUG [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:11,027 INFO [Reader=1,bindAddress=localhost,port=46675 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:11,027 INFO [Reader=1,bindAddress=localhost,port=46675 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35390, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:11,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:11,035 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46675 2024-11-15T23:42:11,035 INFO [Listener,port=46675 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46675: stopping 2024-11-15T23:42:11,035 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:11,037 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:11,046 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 57) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser$StackFrameBuffer.resize(StackStreamFactory.java:479) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.fetchStackFrames(StackStreamFactory.java:384) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.getNextBatch(StackStreamFactory.java:322) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.peekFrame(StackStreamFactory.java:263) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.hasNext(StackStreamFactory.java:351) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:593) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007fc13418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RpcServer idle connection scanner for port 46675 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Listener,port=46675 java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.close0(Native Method) java.base@17.0.11/sun.nio.ch.SocketDispatcher.close(SocketDispatcher.java:70) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.tryClose(ServerSocketChannelImpl.java:554) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.tryFinishClose(ServerSocketChannelImpl.java:568) java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.kill(ServerSocketChannelImpl.java:631) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:194) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:256) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5398 (was 5423) 2024-11-15T23:42:11,054 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5398 2024-11-15T23:42:11,069 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:11,069 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:11,072 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714131071,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:11,074 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:11,075 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34139 2024-11-15T23:42:11,076 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:11,076 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:11,076 INFO [Listener,port=34139 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34139: starting 2024-11-15T23:42:11,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3846534f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:11,077 DEBUG [Thread-96 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-15T23:42:11,080 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-15T23:42:11,080 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-15T23:42:11,080 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-15T23:42:11,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:11,081 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34139 2024-11-15T23:42:11,081 INFO [Listener,port=34139 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34139: stopping 2024-11-15T23:42:11,081 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:11,081 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:11,089 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64 (was 62) Potentially hanging thread: Reader=2,bindAddress=localhost,port=34139 Potentially hanging thread: RPCClient-NioEventLoopGroup-18-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5398 (was 5398) 2024-11-15T23:42:11,095 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5398 2024-11-15T23:42:11,105 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:11,105 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:11,107 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714131107,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:11,110 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:11,111 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45723 2024-11-15T23:42:11,111 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-15T23:42:11,111 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-15T23:42:11,111 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-15T23:42:11,111 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-15T23:42:11,111 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-15T23:42:11,111 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-15T23:42:11,111 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-15T23:42:11,111 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-15T23:42:11,112 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:11,112 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:11,112 INFO [Listener,port=45723 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45723: starting 2024-11-15T23:42:11,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c85b17d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:11,113 DEBUG [Thread-99 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-15T23:42:11,115 WARN [Reader=1,bindAddress=localhost,port=45723 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-15T23:42:11,116 INFO [Reader=1,bindAddress=localhost,port=45723 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-15T23:42:11,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:11,121 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45723 2024-11-15T23:42:11,121 INFO [Listener,port=45723 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45723: stopping 2024-11-15T23:42:11,121 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:11,122 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:11,127 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 63) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndExchangeByte(Unsafe.java:1527) java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetByte(Unsafe.java:1544) java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetBoolean(Unsafe.java:1784) java.base@17.0.11/java.lang.invoke.VarHandleBooleans$FieldInstanceReadWrite.compareAndSet(VarHandleBooleans.java:178) java.base@17.0.11/java.lang.invoke.VarHandleGuards.guard_LII_Z(VarHandleGuards.java:137) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:125) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5396 (was 5398) 2024-11-15T23:42:11,133 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5396 2024-11-15T23:42:11,143 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:11,143 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:11,145 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714131145,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:11,148 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:11,149 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43905 2024-11-15T23:42:11,149 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:11,151 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:11,151 INFO [Listener,port=43905 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43905: starting 2024-11-15T23:42:11,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2645b2d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:11,152 DEBUG [Thread-102 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:11,154 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,157 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714131157,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,159 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,159 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,159 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,160 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:11,161 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,161 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,161 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:11,162 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,162 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:11,162 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,162 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,162 DEBUG [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:11,162 INFO [Reader=1,bindAddress=localhost,port=43905 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:11,162 INFO [Reader=1,bindAddress=localhost,port=43905 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48400, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:11,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:11,171 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43905 2024-11-15T23:42:11,172 INFO [Listener,port=43905 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43905: stopping 2024-11-15T23:42:11,172 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:11,172 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:11,181 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 64) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5394 (was 5396) 2024-11-15T23:42:11,188 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5394 2024-11-15T23:42:11,199 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:11,199 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:11,201 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714131201,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:11,204 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:11,205 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35755 2024-11-15T23:42:11,205 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:11,206 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:11,206 INFO [Listener,port=35755 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35755: starting 2024-11-15T23:42:11,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf9d72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:11,207 DEBUG [Thread-105 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:11,208 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,211 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714131211,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,212 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,213 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,213 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,214 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:11,215 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,215 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,215 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:11,216 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,216 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,216 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,216 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:11,216 DEBUG [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:11,216 INFO [Reader=1,bindAddress=localhost,port=35755 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:11,217 INFO [Reader=1,bindAddress=localhost,port=35755 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53346, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:11,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:11,228 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35755 2024-11-15T23:42:11,228 INFO [Listener,port=35755 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35755: stopping 2024-11-15T23:42:11,228 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:11,228 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:11,228 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:11,233 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34233 2024-11-15T23:42:11,237 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:11,238 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:11,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d2ffad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:11,239 INFO [Listener,port=34233 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34233: starting 2024-11-15T23:42:11,240 DEBUG [Thread-108 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:11,241 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,243 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,244 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,244 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,245 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:11,246 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,246 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,246 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:11,247 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:11,247 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:11,247 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:11,247 DEBUG [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:11,247 INFO [Reader=1,bindAddress=localhost,port=34233 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:11,248 INFO [Reader=1,bindAddress=localhost,port=34233 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:32946, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,251 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:13,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,251 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:34233 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-15T23:42:13,251 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34233 2024-11-15T23:42:13,251 INFO [Listener,port=34233 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34233: stopping 2024-11-15T23:42:13,251 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:13,252 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:13,260 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69 (was 65) Potentially hanging thread: Listener,port=34233 app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:268) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StringCoding.implEncodeAsciiArray(StringCoding.java:65) java.base@17.0.11/java.lang.System$2.encodeASCII(System.java:2418) java.base@17.0.11/sun.nio.cs.UTF_8$Encoder.encodeArrayLoop(UTF_8.java:456) java.base@17.0.11/sun.nio.cs.UTF_8$Encoder.encodeLoop(UTF_8.java:564) java.base@17.0.11/java.nio.charset.CharsetEncoder.encode(CharsetEncoder.java:585) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:64) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5389 (was 5394) 2024-11-15T23:42:13,266 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5389 2024-11-15T23:42:13,276 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,276 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,279 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133279,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,281 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,281 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35947 2024-11-15T23:42:13,282 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:13,282 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,282 INFO [Listener,port=35947 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35947: starting 2024-11-15T23:42:13,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545b8e55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,283 DEBUG [Thread-111 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,285 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,288 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714133288,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,289 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,290 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:13,290 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,291 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,291 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:13,291 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,291 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,292 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:13,292 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,292 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,292 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,292 DEBUG [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:13,292 INFO [Reader=1,bindAddress=localhost,port=35947 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,293 INFO [Reader=1,bindAddress=localhost,port=35947 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50602, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,298 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35947 2024-11-15T23:42:13,299 INFO [Listener,port=35947 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35947: stopping 2024-11-15T23:42:13,299 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:13,299 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:13,305 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 67) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5388 (was 5389) 2024-11-15T23:42:13,311 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5388 2024-11-15T23:42:13,321 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,321 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,324 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133324,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,326 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,327 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33285 2024-11-15T23:42:13,327 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:13,327 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,327 INFO [Listener,port=33285 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33285: starting 2024-11-15T23:42:13,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d379e32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,328 DEBUG [Thread-114 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,330 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,332 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714133332,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,334 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,334 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:13,334 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,335 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,336 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:13,336 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,336 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,336 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-15T23:42:13,336 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,336 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,336 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,336 DEBUG [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:13,336 INFO [Reader=1,bindAddress=localhost,port=33285 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,337 INFO [Reader=1,bindAddress=localhost,port=33285 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45900, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,342 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731714133325 2024-11-15T23:42:13,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,343 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33285 2024-11-15T23:42:13,343 INFO [Listener,port=33285 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33285: stopping 2024-11-15T23:42:13,343 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:13,344 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:13,350 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 68) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.util.stream.WhileOps$1Op$1OpSink.accept(WhileOps.java:376) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:595) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007fc13418c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5387 (was 5388) 2024-11-15T23:42:13,356 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5386 2024-11-15T23:42:13,366 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,366 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,368 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133368,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,370 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,371 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35207 2024-11-15T23:42:13,371 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-15T23:42:13,371 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,371 INFO [Listener,port=35207 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35207: starting 2024-11-15T23:42:13,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dac645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,372 DEBUG [Thread-117 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,374 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,377 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714133377,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,379 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,379 INFO [Reader=1,bindAddress=localhost,port=35207 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-15T23:42:13,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,385 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35207 2024-11-15T23:42:13,385 INFO [Listener,port=35207 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35207: stopping 2024-11-15T23:42:13,385 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-15T23:42:13,385 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-15T23:42:13,393 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 69) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5385 (was 5386) 2024-11-15T23:42:13,399 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=70, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5385 2024-11-15T23:42:13,410 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,410 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,412 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133412,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,414 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,415 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,416 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41563 2024-11-15T23:42:13,416 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30206332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,417 DEBUG [Thread-118 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,418 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,421 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714133421,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,422 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,422 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,423 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,424 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,424 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,425 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,425 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,425 DEBUG [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:13,425 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,426 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43040, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,427 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,435 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41563 2024-11-15T23:42:13,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72 (was 70) Potentially hanging thread: NettyRpcServer-19-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-19-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=475 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5383 (was 5385) 2024-11-15T23:42:13,450 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72, OpenFileDescriptor=475, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5383 2024-11-15T23:42:13,461 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,461 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,464 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133464,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,466 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,466 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,467 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46675 2024-11-15T23:42:13,467 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c0291d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-15T23:42:13,468 DEBUG [Thread-119 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,469 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,472 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714133472,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,474 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:13,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,474 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:46675 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-15T23:42:13,474 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46675 2024-11-15T23:42:13,482 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74 (was 72) Potentially hanging thread: NettyRpcServer-20-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-20-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=484 (was 475) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5382 (was 5383) 2024-11-15T23:42:13,489 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74, OpenFileDescriptor=484, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5381 2024-11-15T23:42:13,499 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,499 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,502 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133502,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,504 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,504 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,505 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33789 2024-11-15T23:42:13,505 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa8b665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,506 DEBUG [Thread-120 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,507 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,510 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714133510,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,512 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,512 DEBUG [NettyRpcServer-21-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,514 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,514 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,514 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,515 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:13,515 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731714133503 2024-11-15T23:42:13,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,515 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:33789 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-15T23:42:13,515 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33789 2024-11-15T23:42:13,522 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76 (was 74) Potentially hanging thread: NettyRpcServer-21-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-21-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=493 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5379 (was 5381) 2024-11-15T23:42:13,528 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5379 2024-11-15T23:42:13,539 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,540 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,542 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133542,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,544 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,544 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,545 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37943 2024-11-15T23:42:13,545 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c73e0fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,546 DEBUG [Thread-121 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,548 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,551 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714133550,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,552 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,552 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,554 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,554 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,554 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,555 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,555 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,555 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,555 DEBUG [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:13,555 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,556 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60576, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,562 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37943 2024-11-15T23:42:13,563 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,563 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,564 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34813 2024-11-15T23:42:13,564 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e564d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,594 DEBUG [Thread-122 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,595 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,596 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,597 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,598 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,598 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,598 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,599 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,599 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,599 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,599 DEBUG [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:13,599 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,600 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45756, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,606 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34813 2024-11-15T23:42:13,607 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,607 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,609 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38575 2024-11-15T23:42:13,609 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4025105d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,610 DEBUG [Thread-123 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,611 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,613 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,613 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,614 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,615 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,615 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,616 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,616 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,616 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,616 DEBUG [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:13,616 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,617 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47638, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,624 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38575 2024-11-15T23:42:13,625 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,625 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,626 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35287 2024-11-15T23:42:13,626 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733c3a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,627 DEBUG [Thread-124 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,628 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,629 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,629 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,630 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,631 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,631 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,632 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,632 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,632 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,632 DEBUG [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-15T23:42:13,632 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,633 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57838, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,642 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35287 2024-11-15T23:42:13,643 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,643 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,645 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40919 2024-11-15T23:42:13,645 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384237c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,646 DEBUG [Thread-125 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,647 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,649 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,649 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,650 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,651 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,651 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,652 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,652 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,652 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,652 DEBUG [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:13,652 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,653 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53976, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,662 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40919 2024-11-15T23:42:13,670 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86 (was 76) Potentially hanging thread: NettyRpcServer-24-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-22-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-25-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-26-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-23-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-22-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-24-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-26-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-23-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-25-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=538 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5370 (was 5379) 2024-11-15T23:42:13,676 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86, OpenFileDescriptor=538, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5369 2024-11-15T23:42:13,687 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,687 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,690 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133690,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,692 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,692 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,693 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37443 2024-11-15T23:42:13,693 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35542d11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,694 DEBUG [Thread-126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-15T23:42:13,696 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-15T23:42:13,697 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-15T23:42:13,697 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-15T23:42:13,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,697 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37443 2024-11-15T23:42:13,705 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88 (was 86) Potentially hanging thread: NettyRpcServer-27-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-27-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=547 (was 538) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5367 (was 5369) 2024-11-15T23:42:13,711 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88, OpenFileDescriptor=547, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5367 2024-11-15T23:42:13,721 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,721 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,724 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133723,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,725 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,725 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,726 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39499 2024-11-15T23:42:13,726 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-15T23:42:13,726 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-15T23:42:13,726 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-15T23:42:13,726 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-15T23:42:13,726 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-15T23:42:13,726 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-15T23:42:13,726 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-15T23:42:13,726 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-15T23:42:13,726 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6567ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,727 DEBUG [Thread-127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-15T23:42:13,729 WARN [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-15T23:42:13,729 INFO [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-15T23:42:13,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,735 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39499 2024-11-15T23:42:13,742 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90 (was 88) Potentially hanging thread: NettyRpcServer-28-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-28-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=556 (was 547) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5365 (was 5367) 2024-11-15T23:42:13,749 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90, OpenFileDescriptor=556, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5365 2024-11-15T23:42:13,760 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,760 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,762 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133762,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,764 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,764 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,765 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44619 2024-11-15T23:42:13,765 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca039e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,766 DEBUG [Thread-128 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,767 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,770 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714133770,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,771 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,771 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,772 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,773 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,773 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,773 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,773 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,774 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,774 DEBUG [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:13,774 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,774 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50238, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,780 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44619 2024-11-15T23:42:13,787 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92 (was 90) Potentially hanging thread: NettyRpcServer-29-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-29-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=565 (was 556) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5363 (was 5365) 2024-11-15T23:42:13,793 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92, OpenFileDescriptor=565, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5362 2024-11-15T23:42:13,803 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:13,803 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:13,806 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714133805,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:13,807 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,808 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,809 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44745 2024-11-15T23:42:13,809 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc1b4da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,810 DEBUG [Thread-129 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,810 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,813 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714133813,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,814 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=498 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,815 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,816 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,817 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,817 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,817 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,818 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:13,818 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,818 DEBUG [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:13,818 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,819 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58512, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:13,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:13,827 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44745 2024-11-15T23:42:13,828 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:13,828 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:13,829 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39763 2024-11-15T23:42:13,829 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:13,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ef0723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:13,830 DEBUG [Thread-130 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:13,831 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,833 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,833 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,834 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:13,834 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,835 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:13,835 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:13,835 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:13,835 DEBUG [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-15T23:42:13,835 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:13,836 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41430, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:15,838 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-15T23:42:15,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:15,838 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:39763 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-15T23:42:15,838 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39763 2024-11-15T23:42:15,846 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96 (was 92) Potentially hanging thread: NettyRpcServer-31-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-31-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-30-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-30-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=583 (was 565) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5355 (was 5362) 2024-11-15T23:42:15,851 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731714133806 2024-11-15T23:42:15,852 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5355 2024-11-15T23:42:15,863 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:15,863 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:15,865 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714135865,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:15,867 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:15,867 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:15,868 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44865 2024-11-15T23:42:15,868 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:15,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a66c2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:15,869 DEBUG [Thread-131 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:15,870 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,873 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714135872,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,874 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:15,874 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,875 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:15,876 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:15,876 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:15,877 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:15,877 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:15,877 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,877 DEBUG [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:15,877 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:15,878 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35988, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:15,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:15,884 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44865 2024-11-15T23:42:15,891 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98 (was 96) Potentially hanging thread: NettyRpcServer-32-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-32-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=592 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5354 (was 5355) 2024-11-15T23:42:15,898 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5354 2024-11-15T23:42:15,908 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:15,908 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:15,910 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714135910,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:15,912 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:15,912 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:15,913 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40299 2024-11-15T23:42:15,913 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:15,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183e6541, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:15,914 DEBUG [Thread-132 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:15,915 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,918 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714135918,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,920 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:15,920 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,921 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-15T23:42:15,921 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:15,922 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-15T23:42:15,922 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:15,922 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-15T23:42:15,922 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,922 DEBUG [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-15T23:42:15,922 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-15T23:42:15,923 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35286, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-15T23:42:15,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:15,929 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40299 2024-11-15T23:42:15,936 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100 (was 98) Potentially hanging thread: NettyRpcServer-33-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-33-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=601 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5352 (was 5354) 2024-11-15T23:42:15,942 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100, OpenFileDescriptor=601, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=5351 2024-11-15T23:42:15,952 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-15T23:42:15,952 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-15T23:42:15,954 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731714135954,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-15T23:42:15,956 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-15T23:42:15,956 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T23:42:15,957 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40851 2024-11-15T23:42:15,957 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-15T23:42:15,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2149a29b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T23:42:15,958 DEBUG [Thread-133 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-15T23:42:15,959 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,962 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731714135962,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-15T23:42:15,964 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-15T23:42:15,964 INFO [NettyRpcServer-34-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-15T23:42:15,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T23:42:15,970 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40851 2024-11-15T23:42:15,977 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=102 (was 100) Potentially hanging thread: NettyRpcServer-34-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-34-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=610 (was 601) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=5350 (was 5351) 2024-11-15T23:42:15,978 INFO [Time-limited test {}] impl.DefaultInternalKdcServerImpl(102): Default Internal kdc server stopped.