2024-11-26 09:45:38,945 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-26 09:45:38,960 main DEBUG Took 0.012026 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 09:45:38,960 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 09:45:38,961 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 09:45:38,962 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 09:45:38,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:38,973 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 09:45:38,988 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:38,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:38,991 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:38,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:38,992 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:38,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:38,994 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:38,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:38,995 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:38,995 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:38,996 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:38,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:38,997 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:38,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:38,998 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:38,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:38,999 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:38,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:39,000 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:39,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:39,001 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:39,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:39,002 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:39,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 09:45:39,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:39,003 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 09:45:39,005 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 09:45:39,007 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 09:45:39,009 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 09:45:39,010 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 09:45:39,011 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 09:45:39,012 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 09:45:39,020 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 09:45:39,022 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 09:45:39,025 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 09:45:39,025 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 09:45:39,026 main DEBUG createAppenders(={Console}) 2024-11-26 09:45:39,027 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 initialized 2024-11-26 09:45:39,027 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-26 09:45:39,027 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 OK. 2024-11-26 09:45:39,028 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 09:45:39,028 main DEBUG OutputStream closed 2024-11-26 09:45:39,029 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 09:45:39,029 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 09:45:39,029 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK 2024-11-26 09:45:39,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 09:45:39,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 09:45:39,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 09:45:39,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 09:45:39,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 09:45:39,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 09:45:39,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 09:45:39,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 09:45:39,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 09:45:39,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 09:45:39,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 09:45:39,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 09:45:39,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 09:45:39,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 09:45:39,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 09:45:39,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 09:45:39,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 09:45:39,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 09:45:39,111 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 09:45:39,111 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5812f68b) with optional ClassLoader: null 2024-11-26 09:45:39,112 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 09:45:39,112 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5812f68b] started OK. 2024-11-26T09:45:39,330 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5eff2940-55cf-0a00-258f-5db839160379 2024-11-26 09:45:39,333 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 09:45:39,333 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T09:45:39,346 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestSecureIPC timeout: 52 mins 2024-11-26T09:45:39,508 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=13, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4482 2024-11-26T09:45:39,589 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T09:45:39,655 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:39,660 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:39,698 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614339696,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:40,340 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:40,439 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T09:45:40,480 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T09:45:40,491 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T09:45:40,514 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42611 2024-11-26T09:45:40,521 INFO [Listener,port=42611 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42611: starting 2024-11-26T09:45:40,521 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:40,521 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:40,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc7794a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:40,644 DEBUG [Thread-6 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:40,702 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42611 2024-11-26T09:45:40,710 DEBUG [Thread-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:40,747 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614340746,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:40,756 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:40,769 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:40,775 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:40,775 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:40,783 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:40,785 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:40,786 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:40,786 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:40,787 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:40,788 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:40,788 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:40,789 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:40,789 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-26T09:45:40,789 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:40,790 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:40,790 DEBUG [Thread-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:40,790 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:40,791 DEBUG [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:40,791 INFO [Reader=1,bindAddress=localhost,port=42611 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:40,813 INFO [Reader=1,bindAddress=localhost,port=42611 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33514, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:40,813 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-26T09:45:40,818 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:256 2024-11-26T09:45:40,820 DEBUG [BRPC Connection (2096510094) to localhost:42611 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-26T09:45:40,854 DEBUG [BRPC Connection (2096510094) to localhost:42611 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:242 2024-11-26T09:45:40,858 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2264 2024-11-26T09:45:40,859 DEBUG [BRPC Connection (2096510094) to localhost:42611 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-26T09:45:40,864 DEBUG [BRPC Connection (2096510094) to localhost:42611 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2250 2024-11-26T09:45:40,874 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-26T09:45:40,875 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18246 2024-11-26T09:45:40,879 DEBUG [BRPC Connection (2096510094) to localhost:42611 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-26T09:45:40,894 DEBUG [BRPC Connection (2096510094) to localhost:42611 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22349 2024-11-26T09:45:40,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:40,899 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42611 2024-11-26T09:45:40,899 INFO [Listener,port=42611 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42611: stopping 2024-11-26T09:45:40,899 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:40,900 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:40,907 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 13) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: pool-3-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=290 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4362 (was 4482) 2024-11-26T09:45:40,913 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4362 2024-11-26T09:45:40,927 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:40,927 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:40,934 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614340933,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:40,948 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:40,950 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34063 2024-11-26T09:45:40,950 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:40,950 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:40,950 INFO [Listener,port=34063 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34063: starting 2024-11-26T09:45:40,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447a3d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T09:45:40,952 DEBUG [Thread-11 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:40,952 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34063 2024-11-26T09:45:40,954 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:40,965 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614340964,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:40,967 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:40,968 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:40,969 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:34063 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-26T09:45:40,973 WARN [Thread-11 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732614340936 2024-11-26T09:45:41,009 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,011 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext. 2024-11-26T09:45:41,012 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:41,012 DEBUG [Thread-11 {}] ipc.FailedServers(52): Added failed server with address localhost:34063 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,014 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34063 2024-11-26T09:45:41,014 INFO [Listener,port=34063 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34063: stopping 2024-11-26T09:45:41,014 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,015 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,020 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.Class.isAssignableFrom(Native Method) java.base@17.0.11/java.lang.StackStreamFactory.isReflectionFrame(StackStreamFactory.java:989) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.peekFrame(StackStreamFactory.java:266) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.hasNext(StackStreamFactory.java:351) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.nextStackFrame(StackStreamFactory.java:520) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:594) java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007f683018c260.apply(Unknown Source) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4349 (was 4362) 2024-11-26T09:45:41,026 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4349 2024-11-26T09:45:41,040 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:41,040 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:41,047 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614341046,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:41,050 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,052 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40271 2024-11-26T09:45:41,053 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,053 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,053 INFO [Listener,port=40271 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40271: starting 2024-11-26T09:45:41,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60a0d4df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,054 DEBUG [Thread-14 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:41,055 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40271 2024-11-26T09:45:41,056 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,064 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614341063,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,066 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,067 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,068 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,068 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,070 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:41,070 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:41,070 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,071 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,071 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,071 DEBUG [Reader=1,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,071 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,072 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:41,072 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:40271 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-26T09:45:41,073 WARN [Thread-14 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732614341048 2024-11-26T09:45:41,091 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,093 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,094 DEBUG [Reader=2,bindAddress=localhost,port=40271 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,095 DEBUG [Reader=2,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,095 DEBUG [Reader=2,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,096 DEBUG [Reader=2,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:41,097 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:41,097 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,098 DEBUG [Reader=2,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,098 DEBUG [Reader=2,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,098 DEBUG [Reader=2,bindAddress=localhost,port=40271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,098 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,099 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:41,099 DEBUG [Thread-14 {}] ipc.FailedServers(52): Added failed server with address localhost:40271 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,100 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40271 2024-11-26T09:45:41,100 INFO [Listener,port=40271 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40271: stopping 2024-11-26T09:45:41,100 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,101 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,106 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackTraceElement.of(StackTraceElement.java:554) java.base@17.0.11/java.lang.StackFrameInfo.toStackTraceElement(StackFrameInfo.java:130) app//org.apache.logging.log4j.util.StackLocator$$Lambda$128/0x00007f683018c958.apply(Unknown Source) java.base@17.0.11/java.util.Optional.map(Optional.java:260) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4336 (was 4349) 2024-11-26T09:45:41,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4336 2024-11-26T09:45:41,125 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:41,126 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:41,132 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614341131,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:41,136 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,138 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37719 2024-11-26T09:45:41,139 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,139 INFO [Listener,port=37719 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37719: starting 2024-11-26T09:45:41,139 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f42f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,140 DEBUG [Thread-17 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:41,141 DEBUG [Thread-17 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37719 2024-11-26T09:45:41,142 DEBUG [Thread-17 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,149 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614341149,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,151 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,151 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,153 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,153 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,154 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:41,154 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:41,155 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,155 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,155 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,155 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,156 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,156 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:41,156 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:41,156 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,156 DEBUG [Thread-17 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:41,157 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,157 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,157 DEBUG [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:41,157 INFO [Reader=1,bindAddress=localhost,port=37719 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:41,158 INFO [Reader=1,bindAddress=localhost,port=37719 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49298, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:41,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,185 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37719 2024-11-26T09:45:41,185 INFO [Listener,port=37719 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37719: stopping 2024-11-26T09:45:41,185 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,185 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,186 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,188 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41545 2024-11-26T09:45:41,189 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,189 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,189 INFO [Listener,port=41545 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41545: starting 2024-11-26T09:45:41,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8c3651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,191 DEBUG [Thread-20 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:41,192 DEBUG [Thread-20 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41545 2024-11-26T09:45:41,193 DEBUG [Thread-20 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,196 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,196 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,197 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,197 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,199 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:41,199 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:41,199 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,200 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,200 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,201 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,201 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,201 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:41,202 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:41,202 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,202 DEBUG [Thread-20 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:41,202 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,202 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,203 DEBUG [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:41,203 INFO [Reader=1,bindAddress=localhost,port=41545 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:41,203 INFO [Reader=1,bindAddress=localhost,port=41545 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51264, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:41,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,220 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41545 2024-11-26T09:45:41,221 INFO [Listener,port=41545 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41545: stopping 2024-11-26T09:45:41,221 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,221 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,221 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,223 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45353 2024-11-26T09:45:41,224 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,224 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,224 INFO [Listener,port=45353 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45353: starting 2024-11-26T09:45:41,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268decc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,225 DEBUG [Thread-23 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:41,226 DEBUG [Thread-23 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45353 2024-11-26T09:45:41,227 DEBUG [Thread-23 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,229 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,229 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,230 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,230 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,232 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:41,233 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:41,233 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,234 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,234 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,234 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,235 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,235 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:41,236 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:41,236 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,236 DEBUG [Thread-23 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:41,236 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,236 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,237 DEBUG [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:41,237 INFO [Reader=1,bindAddress=localhost,port=45353 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:41,237 INFO [Reader=1,bindAddress=localhost,port=45353 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43606, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:41,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,250 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45353 2024-11-26T09:45:41,250 INFO [Listener,port=45353 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45353: stopping 2024-11-26T09:45:41,250 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,250 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,251 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,252 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34599 2024-11-26T09:45:41,253 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,253 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,253 INFO [Listener,port=34599 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34599: starting 2024-11-26T09:45:41,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea53fb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,255 DEBUG [Thread-26 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:41,255 DEBUG [Thread-26 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34599 2024-11-26T09:45:41,256 DEBUG [Thread-26 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,258 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,258 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,259 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,259 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,261 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:41,261 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:41,262 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,262 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,262 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,262 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,263 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,263 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:41,263 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-26T09:45:41,263 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,263 DEBUG [Thread-26 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:41,263 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,264 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,264 DEBUG [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-26T09:45:41,264 INFO [Reader=1,bindAddress=localhost,port=34599 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:41,264 INFO [Reader=1,bindAddress=localhost,port=34599 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43926, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:41,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,277 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34599 2024-11-26T09:45:41,277 INFO [Listener,port=34599 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34599: stopping 2024-11-26T09:45:41,277 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,278 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,278 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,281 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43887 2024-11-26T09:45:41,281 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,282 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,282 INFO [Listener,port=43887 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43887: starting 2024-11-26T09:45:41,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3acf87b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,283 DEBUG [Thread-29 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:41,283 DEBUG [Thread-29 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43887 2024-11-26T09:45:41,284 DEBUG [Thread-29 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,286 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,286 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,287 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,287 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,289 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:41,289 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:41,289 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,289 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,290 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,290 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,290 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,290 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:41,291 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-26T09:45:41,291 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,291 DEBUG [Thread-29 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:41,291 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,291 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,291 DEBUG [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:41,291 INFO [Reader=1,bindAddress=localhost,port=43887 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:41,292 INFO [Reader=1,bindAddress=localhost,port=43887 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35454, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:41,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,308 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43887 2024-11-26T09:45:41,308 INFO [Listener,port=43887 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43887: stopping 2024-11-26T09:45:41,308 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,308 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,315 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:162) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4321 (was 4336) 2024-11-26T09:45:41,321 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4321 2024-11-26T09:45:41,333 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:41,334 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:41,339 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614341339,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:41,344 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,346 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46723 2024-11-26T09:45:41,347 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,347 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,347 INFO [Listener,port=46723 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46723: starting 2024-11-26T09:45:41,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86dd8b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,349 DEBUG [Thread-32 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-26T09:45:41,349 DEBUG [Thread-32 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46723 2024-11-26T09:45:41,353 DEBUG [BRPC Connection (2096510094) to localhost:46723 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-26T09:45:41,353 WARN [BRPC Connection (2096510094) to localhost:46723 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-26T09:45:41,355 DEBUG [BRPC Connection (2096510094) to localhost:46723 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-26T09:45:41,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,356 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46723 2024-11-26T09:45:41,356 INFO [Listener,port=46723 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46723: stopping 2024-11-26T09:45:41,356 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,356 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,365 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4312 (was 4321) 2024-11-26T09:45:41,370 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=291, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4312 2024-11-26T09:45:41,383 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:41,383 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:41,390 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614341389,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:41,394 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,395 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41503 2024-11-26T09:45:41,396 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-26T09:45:41,396 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-26T09:45:41,396 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-26T09:45:41,396 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-26T09:45:41,396 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-26T09:45:41,396 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-26T09:45:41,396 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-26T09:45:41,396 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-26T09:45:41,399 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,399 INFO [Listener,port=41503 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41503: starting 2024-11-26T09:45:41,399 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ef5ea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,401 DEBUG [Thread-35 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-26T09:45:41,401 DEBUG [Thread-35 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41503 2024-11-26T09:45:41,402 WARN [Reader=1,bindAddress=localhost,port=41503 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-26T09:45:41,403 INFO [Reader=1,bindAddress=localhost,port=41503 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-26T09:45:41,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,413 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41503 2024-11-26T09:45:41,414 INFO [Listener,port=41503 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41503: stopping 2024-11-26T09:45:41,414 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,414 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,419 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 291), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4302 (was 4312) 2024-11-26T09:45:41,424 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4302 2024-11-26T09:45:41,437 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:41,438 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:41,443 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614341443,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:41,446 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,447 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45239 2024-11-26T09:45:41,448 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,448 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,448 INFO [Listener,port=45239 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45239: starting 2024-11-26T09:45:41,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505094d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,450 DEBUG [Thread-38 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:41,450 DEBUG [Thread-38 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45239 2024-11-26T09:45:41,451 DEBUG [Thread-38 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,458 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614341457,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,459 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,460 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,461 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,461 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,463 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:41,463 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:41,463 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,464 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,464 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,464 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,465 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,465 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:41,465 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:41,465 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,465 DEBUG [Thread-38 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:41,465 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,466 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,466 DEBUG [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:41,466 INFO [Reader=1,bindAddress=localhost,port=45239 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:41,466 INFO [Reader=1,bindAddress=localhost,port=45239 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33674, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:41,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,477 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45239 2024-11-26T09:45:41,478 INFO [Listener,port=45239 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45239: stopping 2024-11-26T09:45:41,478 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,478 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,484 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/jdk.internal.misc.Unsafe.getAndAddLong(Unsafe.java:2482) java.base@17.0.11/java.util.concurrent.atomic.AtomicLong.getAndAdd(AtomicLong.java:221) app//org.apache.maven.surefire.api.util.internal.Channels$4.writeImpl(Channels.java:200) app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.write(AbstractNoninterruptibleWritableChannel.java:66) app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.writeBuffered(AbstractNoninterruptibleWritableChannel.java:49) app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.write(AbstractStreamEncoder.java:69) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.write(EventChannelEncoder.java:288) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:173) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4285 (was 4302) 2024-11-26T09:45:41,491 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4285 2024-11-26T09:45:41,509 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:41,510 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:41,516 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614341515,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:41,519 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,520 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38115 2024-11-26T09:45:41,521 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,521 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,521 INFO [Listener,port=38115 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38115: starting 2024-11-26T09:45:41,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42853030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,522 DEBUG [Thread-41 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:41,522 DEBUG [Thread-41 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38115 2024-11-26T09:45:41,523 DEBUG [Thread-41 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,530 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614341529,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,531 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,531 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,532 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,532 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,534 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 107 from saslServer. 2024-11-26T09:45:41,534 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(119): Will read input token of size 107 for processing by initSASLContext 2024-11-26T09:45:41,534 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,534 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,534 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,535 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,535 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,535 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:41,535 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-26T09:45:41,535 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,535 DEBUG [Thread-41 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:41,535 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,536 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,536 DEBUG [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:41,536 INFO [Reader=1,bindAddress=localhost,port=38115 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:41,537 INFO [Reader=1,bindAddress=localhost,port=38115 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58932, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:41,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:41,549 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38115 2024-11-26T09:45:41,549 INFO [Listener,port=38115 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38115: stopping 2024-11-26T09:45:41,550 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:41,550 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:41,550 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:41,551 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46653 2024-11-26T09:45:41,554 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:41,554 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:41,554 INFO [Listener,port=46653 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46653: starting 2024-11-26T09:45:41,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ba83cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:41,556 DEBUG [Thread-44 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:41,556 DEBUG [Thread-44 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46653 2024-11-26T09:45:41,557 DEBUG [Thread-44 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,560 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:41,560 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,561 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,561 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,563 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:41,563 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:41,564 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:41,564 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,564 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,564 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:41,564 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:41,565 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:41,565 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-26T09:45:41,565 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:41,565 DEBUG [Thread-44 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:41,565 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:41,566 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:41,566 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:41,566 INFO [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:41,567 INFO [Reader=1,bindAddress=localhost,port=46653 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46542, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:43,568 ERROR [Thread-44 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46542 remote=localhost/127.0.0.1:46653] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-26T09:45:43,569 DEBUG [Thread-44 {}] ipc.FailedServers(52): Added failed server with address localhost:46653 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-26T09:45:43,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:43,570 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46653 2024-11-26T09:45:43,570 INFO [Listener,port=46653 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46653: stopping 2024-11-26T09:45:43,570 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:43,570 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:43,577 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.pattern.MdcPatternConverter.format(MdcPatternConverter.java:92) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.pattern.VariablesNotEmptyReplacementConverter.format(VariablesNotEmptyReplacementConverter.java:83) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:375) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:216) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:59) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.toByteArray(AbstractStringLayout.java:294) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:145) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4276 (was 4285) 2024-11-26T09:45:43,583 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4276 2024-11-26T09:45:43,596 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:43,596 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:43,601 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614343601,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:43,604 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:43,605 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46097 2024-11-26T09:45:43,605 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:43,605 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:43,605 INFO [Listener,port=46097 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46097: starting 2024-11-26T09:45:43,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf341da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:43,607 DEBUG [Thread-47 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:43,607 DEBUG [Thread-47 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46097 2024-11-26T09:45:43,608 DEBUG [Thread-47 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,616 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614343615,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,617 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:43,617 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,618 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:43,618 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:43,620 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:43,620 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:43,620 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:43,620 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:43,621 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:43,621 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:43,621 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:43,621 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:43,621 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:43,622 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:43,622 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:43,622 DEBUG [Thread-47 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:43,622 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,622 DEBUG [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:43,622 INFO [Reader=1,bindAddress=localhost,port=46097 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:43,622 INFO [Reader=1,bindAddress=localhost,port=46097 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49368, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:43,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:43,630 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46097 2024-11-26T09:45:43,630 INFO [Listener,port=46097 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46097: stopping 2024-11-26T09:45:43,631 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:43,631 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:43,636 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.util.StackLocator$$Lambda$128/0x00007f683018c958.apply(Unknown Source) java.base@17.0.11/java.util.Optional.map(Optional.java:260) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:100) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: RpcServer idle connection scanner for port 46097 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4260 (was 4276) 2024-11-26T09:45:43,642 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4260 2024-11-26T09:45:43,655 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:43,655 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:43,660 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614343660,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:43,663 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:43,664 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34549 2024-11-26T09:45:43,665 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:43,665 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:43,665 INFO [Listener,port=34549 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34549: starting 2024-11-26T09:45:43,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f98a34a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:43,666 DEBUG [Thread-50 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:43,666 DEBUG [Thread-50 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34549 2024-11-26T09:45:43,667 DEBUG [Thread-50 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,673 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614343673,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,674 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:43,674 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,676 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:43,676 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:43,677 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:43,677 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:43,677 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:43,678 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:43,678 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:43,678 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:43,678 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:43,678 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:43,678 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:43,678 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:43,678 DEBUG [Thread-50 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:43,678 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:43,679 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,679 DEBUG [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:43,679 INFO [Reader=1,bindAddress=localhost,port=34549 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:43,679 INFO [Reader=1,bindAddress=localhost,port=34549 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47520, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:43,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:43,687 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34549 2024-11-26T09:45:43,687 INFO [Listener,port=34549 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34549: stopping 2024-11-26T09:45:43,687 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:43,687 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:43,693 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4245 (was 4260) 2024-11-26T09:45:43,698 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4245 2024-11-26T09:45:43,711 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:43,711 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:43,716 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614343716,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:43,719 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:43,720 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36945 2024-11-26T09:45:43,721 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:43,721 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:43,721 INFO [Listener,port=36945 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36945: starting 2024-11-26T09:45:43,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f9d890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:43,723 DEBUG [Thread-53 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:43,723 DEBUG [Thread-53 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36945 2024-11-26T09:45:43,724 DEBUG [Thread-53 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,730 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614343729,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,731 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:43,731 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-26T09:45:43,731 DEBUG [Thread-53 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:43,731 INFO [Reader=1,bindAddress=localhost,port=36945 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-26T09:45:43,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:43,738 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36945 2024-11-26T09:45:43,739 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:43,739 INFO [Listener,port=36945 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36945: stopping 2024-11-26T09:45:43,739 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:43,745 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.impl.ThreadContextDataInjector$ForDefaultThreadContextMap.injectContextData(ThreadContextDataInjector.java:118) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.createContextData(Log4jLogEvent.java:497) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.(Log4jLogEvent.java:355) app//org.apache.logging.log4j.core.impl.DefaultLogEventFactory.createEvent(DefaultLogEventFactory.java:74) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:550) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RpcServer idle connection scanner for port 36945 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Listener,port=36945 java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method) java.base@17.0.11/sun.nio.ch.EventFD.close(EventFD.java:56) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:219) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:256) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=953 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4229 (was 4245) 2024-11-26T09:45:43,750 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=953, ProcessCount=11, AvailableMemoryMB=4229 2024-11-26T09:45:43,763 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:43,763 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:43,768 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614343768,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:43,772 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:43,790 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:43,794 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T09:45:43,833 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15829 (auto-detected) 2024-11-26T09:45:43,834 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T09:45:43,854 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44905 2024-11-26T09:45:43,854 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:43,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f017c5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:43,856 DEBUG [Thread-54 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:43,856 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44905 2024-11-26T09:45:43,857 DEBUG [Thread-54 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,864 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614343864,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,866 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:43,888 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:43,889 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,891 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:43,894 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:43,894 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:43,895 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:43,895 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:43,896 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:43,896 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:43,896 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-26T09:45:43,896 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:43,896 DEBUG [Thread-54 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:43,896 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,897 DEBUG [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:43,897 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:43,902 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-26T09:45:43,902 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35276, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:43,903 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:250 2024-11-26T09:45:43,903 DEBUG [BRPC Connection (2096510094) to localhost:44905 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-26T09:45:43,905 DEBUG [BRPC Connection (2096510094) to localhost:44905 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:236 2024-11-26T09:45:43,906 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2248 2024-11-26T09:45:43,906 DEBUG [BRPC Connection (2096510094) to localhost:44905 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-26T09:45:43,908 DEBUG [BRPC Connection (2096510094) to localhost:44905 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2234 2024-11-26T09:45:43,910 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-26T09:45:43,910 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18204 2024-11-26T09:45:43,911 DEBUG [BRPC Connection (2096510094) to localhost:44905 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-26T09:45:43,919 DEBUG [BRPC Connection (2096510094) to localhost:44905 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22307 2024-11-26T09:45:43,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:43,920 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44905 2024-11-26T09:45:43,929 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21 (was 19) Potentially hanging thread: NettyRpcServer-2-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-2-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=299 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 953), ProcessCount=11 (was 11), AvailableMemoryMB=4201 (was 4229) 2024-11-26T09:45:43,934 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4200 2024-11-26T09:45:43,948 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:43,948 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:43,953 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614343953,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:43,956 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:43,957 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:43,958 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43121 2024-11-26T09:45:43,958 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:43,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2796a315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T09:45:43,960 DEBUG [Thread-55 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:43,960 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43121 2024-11-26T09:45:43,961 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,966 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614343966,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:43,968 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:43,968 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:43,968 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:43121 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-26T09:45:43,969 WARN [Thread-55 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732614343954 2024-11-26T09:45:44,022 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,023 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext. 2024-11-26T09:45:44,024 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:44,024 DEBUG [Thread-55 {}] ipc.FailedServers(52): Added failed server with address localhost:43121 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,024 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43121 2024-11-26T09:45:44,031 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24 (was 21) Potentially hanging thread: NettyRpcServer-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=308 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4184 (was 4200) 2024-11-26T09:45:44,036 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4184 2024-11-26T09:45:44,049 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:44,049 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:44,054 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614344053,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:44,056 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,056 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,057 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33527 2024-11-26T09:45:44,057 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5bc4cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,059 DEBUG [Thread-56 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:44,059 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33527 2024-11-26T09:45:44,060 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,065 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614344065,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,066 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,066 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,066 DEBUG [NettyRpcServer-4-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,068 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:44,069 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:44,069 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,069 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,069 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,069 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,070 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:44,070 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:33527 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-26T09:45:44,070 WARN [Thread-56 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732614344054 2024-11-26T09:45:44,166 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,168 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,168 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,168 DEBUG [NettyRpcServer-4-3 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,170 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:44,170 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:44,170 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,171 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,171 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,171 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,171 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:44,172 DEBUG [Thread-56 {}] ipc.FailedServers(52): Added failed server with address localhost:33527 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,172 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33527 2024-11-26T09:45:44,178 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27 (was 24) Potentially hanging thread: NettyRpcServer-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=317 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4167 (was 4184) 2024-11-26T09:45:44,183 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4167 2024-11-26T09:45:44,196 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:44,197 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:44,201 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614344201,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:44,204 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,204 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,206 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44941 2024-11-26T09:45:44,206 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d155806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,207 DEBUG [Thread-57 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:44,207 DEBUG [Thread-57 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44941 2024-11-26T09:45:44,208 DEBUG [Thread-57 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,214 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614344214,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,215 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,215 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,215 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,217 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:44,217 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:44,218 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,218 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,218 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,218 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,219 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:44,219 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:44,219 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,219 DEBUG [Thread-57 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:44,219 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,219 DEBUG [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:44,219 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:44,220 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58192, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:44,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,228 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44941 2024-11-26T09:45:44,230 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,230 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,231 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35815 2024-11-26T09:45:44,231 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e94926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,232 DEBUG [Thread-58 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:44,232 DEBUG [Thread-58 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35815 2024-11-26T09:45:44,233 DEBUG [Thread-58 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,235 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,235 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,235 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,237 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 107 from saslServer. 2024-11-26T09:45:44,237 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(119): Will read input token of size 107 for processing by initSASLContext 2024-11-26T09:45:44,237 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,238 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,238 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,238 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,239 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:44,239 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:44,239 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,239 DEBUG [Thread-58 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:44,239 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,239 DEBUG [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:44,239 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:44,240 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41970, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:44,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,247 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35815 2024-11-26T09:45:44,248 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,248 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,249 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46369 2024-11-26T09:45:44,249 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6125d6f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,250 DEBUG [Thread-59 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:44,251 DEBUG [Thread-59 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46369 2024-11-26T09:45:44,252 DEBUG [Thread-59 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,253 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,253 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,253 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,255 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:44,255 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:44,256 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,256 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,256 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,256 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,257 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:44,257 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:44,257 DEBUG [Thread-59 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:44,257 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,257 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,257 DEBUG [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:44,257 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:44,258 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52172, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:44,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,266 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46369 2024-11-26T09:45:44,267 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,267 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,268 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33627 2024-11-26T09:45:44,268 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@352f737e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,269 DEBUG [Thread-60 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:44,269 DEBUG [Thread-60 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33627 2024-11-26T09:45:44,270 DEBUG [Thread-60 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,271 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,271 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,271 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,273 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:44,273 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:44,274 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,274 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,274 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,274 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,275 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:44,275 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-26T09:45:44,275 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,275 DEBUG [Thread-60 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:44,275 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,275 DEBUG [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-26T09:45:44,275 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:44,276 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57802, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:44,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,285 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33627 2024-11-26T09:45:44,286 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,286 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,287 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33731 2024-11-26T09:45:44,287 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28385864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,288 DEBUG [Thread-61 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:44,288 DEBUG [Thread-61 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33731 2024-11-26T09:45:44,289 DEBUG [Thread-61 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,290 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,290 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,291 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,292 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:44,292 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:44,292 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,293 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,293 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,293 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,293 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:44,293 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-26T09:45:44,293 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,293 DEBUG [Thread-61 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:44,294 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,294 DEBUG [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:44,294 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:44,294 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49532, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:44,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,304 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33731 2024-11-26T09:45:44,311 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37 (was 27) Potentially hanging thread: NettyRpcServer-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=362 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4143 (was 4167) 2024-11-26T09:45:44,316 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4142 2024-11-26T09:45:44,330 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:44,330 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:44,335 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614344334,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:44,337 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,337 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,338 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39843 2024-11-26T09:45:44,338 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23799907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,339 DEBUG [Thread-62 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-26T09:45:44,340 DEBUG [Thread-62 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39843 2024-11-26T09:45:44,342 DEBUG [BRPC Connection (2096510094) to localhost:39843 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-26T09:45:44,342 WARN [BRPC Connection (2096510094) to localhost:39843 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-26T09:45:44,342 DEBUG [BRPC Connection (2096510094) to localhost:39843 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-26T09:45:44,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,343 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39843 2024-11-26T09:45:44,348 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 37) Potentially hanging thread: NettyRpcServer-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=371 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4131 (was 4142) 2024-11-26T09:45:44,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4130 2024-11-26T09:45:44,366 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:44,366 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:44,371 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614344371,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:44,373 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,374 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,375 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40489 2024-11-26T09:45:44,375 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-26T09:45:44,375 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-26T09:45:44,375 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-26T09:45:44,375 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-26T09:45:44,375 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-26T09:45:44,375 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-26T09:45:44,375 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-26T09:45:44,375 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-26T09:45:44,375 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478a8195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,376 DEBUG [Thread-63 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-26T09:45:44,376 DEBUG [Thread-63 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40489 2024-11-26T09:45:44,378 WARN [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-26T09:45:44,378 INFO [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-26T09:45:44,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,386 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40489 2024-11-26T09:45:44,392 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41 (was 39) Potentially hanging thread: NettyRpcServer-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=380 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4119 (was 4130) 2024-11-26T09:45:44,397 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4118 2024-11-26T09:45:44,410 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:44,410 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:44,415 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614344414,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:44,417 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,417 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,418 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33179 2024-11-26T09:45:44,418 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631d51b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,419 DEBUG [Thread-64 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:44,419 DEBUG [Thread-64 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33179 2024-11-26T09:45:44,420 DEBUG [Thread-64 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,427 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614344426,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,428 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,429 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,429 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,430 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:44,431 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:44,431 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,431 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,431 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,431 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,432 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:44,432 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:44,432 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,432 DEBUG [Thread-64 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:44,432 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,432 DEBUG [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:44,432 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:44,433 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60224, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:44,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,440 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33179 2024-11-26T09:45:44,448 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43 (was 41) Potentially hanging thread: NettyRpcServer-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=389 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4102 (was 4118) 2024-11-26T09:45:44,455 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4101 2024-11-26T09:45:44,468 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:44,468 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:44,473 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614344473,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:44,475 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,475 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,476 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37225 2024-11-26T09:45:44,477 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f50ad15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,478 DEBUG [Thread-65 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:44,478 DEBUG [Thread-65 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37225 2024-11-26T09:45:44,479 DEBUG [Thread-65 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,485 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614344485,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,486 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,486 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,487 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,488 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:44,488 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:44,489 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,489 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,489 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,489 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,490 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:44,490 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-26T09:45:44,490 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,490 DEBUG [Thread-65 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:44,490 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,490 DEBUG [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:44,490 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:44,491 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49142, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:44,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:44,501 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37225 2024-11-26T09:45:44,502 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:44,502 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:44,503 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36955 2024-11-26T09:45:44,504 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:44,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f903f37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:44,505 DEBUG [Thread-66 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:44,505 DEBUG [Thread-66 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36955 2024-11-26T09:45:44,506 DEBUG [Thread-66 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,508 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:44,508 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,508 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,509 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:44,510 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:44,510 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:44,510 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,510 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:44,510 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:44,511 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:44,511 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-26T09:45:44,511 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:44,511 DEBUG [Thread-66 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:44,511 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:44,511 DEBUG [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:44,511 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:44,512 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36872, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:46,514 ERROR [Thread-66 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36872 remote=localhost/127.0.0.1:36955] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-26T09:45:46,514 DEBUG [Thread-66 {}] ipc.FailedServers(52): Added failed server with address localhost:36955 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-26T09:45:46,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,515 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36955 2024-11-26T09:45:46,522 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47 (was 43) Potentially hanging thread: NettyRpcServer-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=407 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4080 (was 4101) 2024-11-26T09:45:46,528 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4080 2024-11-26T09:45:46,540 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:46,540 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:46,545 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614346545,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:46,547 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,547 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:46,549 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38267 2024-11-26T09:45:46,549 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c0e405f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,550 DEBUG [Thread-67 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,550 DEBUG [Thread-67 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38267 2024-11-26T09:45:46,551 DEBUG [Thread-67 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,558 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614346558,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,559 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:46,559 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,559 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,561 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:46,561 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:46,561 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:46,562 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,562 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:46,562 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:46,562 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:46,562 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:46,562 DEBUG [Thread-67 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:46,562 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,562 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,563 DEBUG [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:46,563 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:46,563 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44528, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:46,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,570 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38267 2024-11-26T09:45:46,577 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49 (was 47) Potentially hanging thread: NettyRpcServer-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=416 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4063 (was 4080) 2024-11-26T09:45:46,583 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4063 2024-11-26T09:45:46,595 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:46,595 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:46,600 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614346600,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:46,602 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,602 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:46,603 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35071 2024-11-26T09:45:46,604 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ec6e08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,605 DEBUG [Thread-68 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,605 DEBUG [Thread-68 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35071 2024-11-26T09:45:46,606 DEBUG [Thread-68 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,611 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614346610,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,612 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:46,612 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,612 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,613 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:46,613 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-26T09:45:46,614 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-26T09:45:46,614 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,614 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:46,614 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-26T09:45:46,614 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-26T09:45:46,614 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-26T09:45:46,614 DEBUG [Thread-68 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:46,614 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,615 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,615 DEBUG [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:46,615 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:46,615 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39160, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:46,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,622 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35071 2024-11-26T09:45:46,629 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51 (was 49) Potentially hanging thread: NettyRpcServer-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=425 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4048 (was 4063) 2024-11-26T09:45:46,636 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4048 2024-11-26T09:45:46,651 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:46,652 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:46,657 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614346656,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:46,659 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,660 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:46,661 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37481 2024-11-26T09:45:46,661 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3015b643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,662 DEBUG [Thread-69 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,662 DEBUG [Thread-69 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37481 2024-11-26T09:45:46,663 DEBUG [Thread-69 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,669 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614346669,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,670 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-26T09:45:46,670 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-26T09:45:46,670 DEBUG [Thread-69 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:46,670 INFO [NettyRpcServer-17-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-26T09:45:46,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,677 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37481 2024-11-26T09:45:46,683 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=53 (was 51) Potentially hanging thread: NettyRpcServer-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=434 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4031 (was 4048) 2024-11-26T09:45:46,689 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=53, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4031 2024-11-26T09:45:46,702 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:46,702 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:46,706 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614346706,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:46,708 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,709 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45269 2024-11-26T09:45:46,710 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:46,710 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,710 INFO [Listener,port=45269 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45269: starting 2024-11-26T09:45:46,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63fc13ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,714 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T09:45:46,714 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T09:45:46,719 DEBUG [Thread-72 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,734 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,741 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614346741,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,743 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,744 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,744 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,745 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:46,746 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,746 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,746 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:46,746 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,746 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,747 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,747 DEBUG [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:46,747 INFO [Reader=1,bindAddress=localhost,port=45269 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:46,752 INFO [Reader=1,bindAddress=localhost,port=45269 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59206, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:46,754 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:46,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,768 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45269 2024-11-26T09:45:46,769 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:46,769 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:46,769 INFO [Listener,port=45269 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45269: stopping 2024-11-26T09:45:46,776 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=56 (was 53) Potentially hanging thread: RpcServer idle connection scanner for port 45269 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-1 java.base@17.0.11/java.lang.ClassLoader.defineClass1(Native Method) java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1017) java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:294) app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:290) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.initialize(FastThreadLocal.java:177) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.get(FastThreadLocal.java:142) app//org.apache.hadoop.hbase.ipc.IPCUtil.execute(IPCUtil.java:304) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown(NettyRpcConnection.java:146) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:161) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Listener,port=45269 java.base@17.0.11/java.lang.Class.isArray(Native Method) java.base@17.0.11/java.lang.Class.getComponentType(Class.java:1227) java.base@17.0.11/jdk.internal.misc.Unsafe.checkPrimitiveArray(Unsafe.java:558) java.base@17.0.11/jdk.internal.misc.Unsafe.checkPrimitivePointer(Unsafe.java:579) java.base@17.0.11/jdk.internal.misc.Unsafe.copyMemoryChecks(Unsafe.java:832) java.base@17.0.11/jdk.internal.misc.Unsafe.copyMemory(Unsafe.java:800) java.base@17.0.11/jdk.internal.misc.ScopedMemoryAccess.copyMemoryInternal(ScopedMemoryAccess.java:167) java.base@17.0.11/jdk.internal.misc.ScopedMemoryAccess.copyMemory(ScopedMemoryAccess.java:149) java.base@17.0.11/java.nio.ByteBuffer.putBuffer(ByteBuffer.java:1102) java.base@17.0.11/java.nio.ByteBuffer.put(ByteBuffer.java:1009) java.base@17.0.11/java.nio.HeapByteBuffer.put(HeapByteBuffer.java:243) app//org.apache.logging.log4j.core.layout.ByteBufferDestinationHelper.writeToUnsynchronized(ByteBufferDestinationHelper.java:47) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeBytes(OutputStreamManager.java:353) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:96) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252) - Thread LEAK? -, OpenFileDescriptor=466 (was 434) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=4010 (was 4031) 2024-11-26T09:45:46,782 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=4009 2024-11-26T09:45:46,794 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:46,794 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:46,799 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614346799,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:46,801 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,802 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38669 2024-11-26T09:45:46,802 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,802 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:46,802 INFO [Listener,port=38669 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38669: starting 2024-11-26T09:45:46,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@561e04f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T09:45:46,803 DEBUG [Thread-75 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,805 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,810 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614346810,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,812 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:46,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,813 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:38669 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-26T09:45:46,813 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38669 2024-11-26T09:45:46,813 INFO [Listener,port=38669 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38669: stopping 2024-11-26T09:45:46,813 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:46,813 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:46,819 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=56 (was 54) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushDestination(OutputStreamManager.java:261) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:295) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=3996 (was 4009) 2024-11-26T09:45:46,825 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=3995 2024-11-26T09:45:46,838 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:46,838 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:46,842 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614346842,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:46,844 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,845 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44037 2024-11-26T09:45:46,846 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,846 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:46,846 INFO [Listener,port=44037 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44037: starting 2024-11-26T09:45:46,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c4d6a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,847 DEBUG [Thread-78 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,849 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,854 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614346853,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,855 DEBUG [Reader=1,bindAddress=localhost,port=44037 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,856 DEBUG [Reader=1,bindAddress=localhost,port=44037 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,856 DEBUG [Reader=1,bindAddress=localhost,port=44037 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,857 DEBUG [Reader=1,bindAddress=localhost,port=44037 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:46,858 DEBUG [Reader=1,bindAddress=localhost,port=44037 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,858 DEBUG [Reader=1,bindAddress=localhost,port=44037 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,858 DEBUG [Reader=1,bindAddress=localhost,port=44037 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:46,859 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:46,860 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:44037 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-26T09:45:46,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,860 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44037 2024-11-26T09:45:46,860 INFO [Listener,port=44037 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44037: stopping 2024-11-26T09:45:46,860 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:46,860 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:46,867 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=58 (was 55) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.io.BufferedOutputStream.write(BufferedOutputStream.java:118) app//org.apache.maven.surefire.api.util.internal.Channels$4.writeImpl(Channels.java:199) app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.write(AbstractNoninterruptibleWritableChannel.java:66) app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.writeBuffered(AbstractNoninterruptibleWritableChannel.java:49) app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.write(AbstractStreamEncoder.java:69) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.write(EventChannelEncoder.java:288) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:173) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: Relogin-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=3980 (was 3995) 2024-11-26T09:45:46,873 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=3979 2024-11-26T09:45:46,885 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:46,885 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:46,889 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614346889,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:46,891 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,892 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42119 2024-11-26T09:45:46,892 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,892 INFO [Listener,port=42119 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42119: starting 2024-11-26T09:45:46,892 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:46,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656f7f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,893 DEBUG [Thread-81 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,895 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,905 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614346905,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,907 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,908 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,908 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,909 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:46,909 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,909 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,909 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:46,910 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,910 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,910 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:46,910 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,910 DEBUG [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:46,910 INFO [Reader=1,bindAddress=localhost,port=42119 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:46,911 INFO [Reader=1,bindAddress=localhost,port=42119 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51602, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:46,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,917 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42119 2024-11-26T09:45:46,917 INFO [Listener,port=42119 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42119: stopping 2024-11-26T09:45:46,917 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:46,917 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:46,917 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,918 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42303 2024-11-26T09:45:46,919 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:46,919 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,919 INFO [Listener,port=42303 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42303: starting 2024-11-26T09:45:46,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f78068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,920 DEBUG [Thread-84 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,921 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,923 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,923 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,923 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,924 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:46,925 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,925 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,925 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:46,925 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,925 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,925 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:46,925 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,925 DEBUG [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:46,925 INFO [Reader=1,bindAddress=localhost,port=42303 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:46,926 INFO [Reader=1,bindAddress=localhost,port=42303 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42530, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:46,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,931 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42303 2024-11-26T09:45:46,932 INFO [Listener,port=42303 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42303: stopping 2024-11-26T09:45:46,932 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:46,932 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:46,932 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,933 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39363 2024-11-26T09:45:46,934 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:46,934 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,934 INFO [Listener,port=39363 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39363: starting 2024-11-26T09:45:46,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f75d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,935 DEBUG [Thread-87 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,936 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,936 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732614346890 2024-11-26T09:45:46,937 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,938 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,938 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,939 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:46,940 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,940 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,940 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:46,940 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,940 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:46,940 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,941 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,941 DEBUG [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:46,941 INFO [Reader=1,bindAddress=localhost,port=39363 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:46,941 INFO [Reader=1,bindAddress=localhost,port=39363 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55898, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:46,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,947 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39363 2024-11-26T09:45:46,947 INFO [Listener,port=39363 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39363: stopping 2024-11-26T09:45:46,947 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:46,947 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:46,947 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,948 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42537 2024-11-26T09:45:46,948 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,948 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:46,949 INFO [Listener,port=42537 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42537: starting 2024-11-26T09:45:46,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4f68fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,949 DEBUG [Thread-90 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,951 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,952 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,953 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,953 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,954 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:46,954 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,955 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,955 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:46,955 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,955 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,955 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:46,956 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,956 DEBUG [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-26T09:45:46,956 INFO [Reader=1,bindAddress=localhost,port=42537 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:46,956 INFO [Reader=1,bindAddress=localhost,port=42537 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40324, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:46,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,967 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42537 2024-11-26T09:45:46,967 INFO [Listener,port=42537 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42537: stopping 2024-11-26T09:45:46,967 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:46,967 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:46,967 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:46,972 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41745 2024-11-26T09:45:46,972 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:46,972 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:46,972 INFO [Listener,port=41745 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41745: starting 2024-11-26T09:45:46,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce33334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:46,973 DEBUG [Thread-93 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:46,975 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,976 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,977 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,977 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,978 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:46,979 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,979 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,979 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:46,979 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:46,979 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:46,980 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:46,980 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:46,980 DEBUG [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:46,980 INFO [Reader=1,bindAddress=localhost,port=41745 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:46,981 INFO [Reader=1,bindAddress=localhost,port=41745 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58016, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:46,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:46,990 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41745 2024-11-26T09:45:46,990 INFO [Listener,port=41745 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41745: stopping 2024-11-26T09:45:46,991 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:46,992 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:47,000 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63 (was 57) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StringBuilder.append(StringBuilder.java:179) app//org.apache.logging.log4j.core.pattern.LineSeparatorPatternConverter.format(LineSeparatorPatternConverter.java:60) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:375) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:216) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:59) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.toByteArray(AbstractStringLayout.java:294) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:145) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=3952 (was 3979) 2024-11-26T09:45:47,006 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=3952 2024-11-26T09:45:47,016 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:47,016 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:47,019 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614347019,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:47,021 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:47,021 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35099 2024-11-26T09:45:47,022 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:47,022 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:47,022 INFO [Listener,port=35099 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35099: starting 2024-11-26T09:45:47,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11e0d790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:47,023 DEBUG [Thread-96 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-26T09:45:47,026 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-26T09:45:47,026 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-26T09:45:47,026 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-26T09:45:47,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:47,026 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35099 2024-11-26T09:45:47,026 INFO [Listener,port=35099 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35099: stopping 2024-11-26T09:45:47,027 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:47,027 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:47,033 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63 (was 62) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=3952 (was 3952) 2024-11-26T09:45:47,039 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=3952 2024-11-26T09:45:47,048 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:47,048 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:47,050 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614347050,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:47,052 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:47,054 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44649 2024-11-26T09:45:47,054 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-26T09:45:47,054 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-26T09:45:47,055 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-26T09:45:47,055 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-26T09:45:47,055 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-26T09:45:47,055 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-26T09:45:47,055 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-26T09:45:47,055 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-26T09:45:47,055 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:47,055 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:47,055 INFO [Listener,port=44649 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44649: starting 2024-11-26T09:45:47,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4cf6f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:47,056 DEBUG [Thread-99 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-26T09:45:47,058 WARN [Reader=1,bindAddress=localhost,port=44649 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-26T09:45:47,058 INFO [Reader=1,bindAddress=localhost,port=44649 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-26T09:45:47,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:47,064 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44649 2024-11-26T09:45:47,064 INFO [Listener,port=44649 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44649: stopping 2024-11-26T09:45:47,065 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:47,065 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:47,071 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64 (was 63) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=3952 (was 3952) 2024-11-26T09:45:47,077 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=3951 2024-11-26T09:45:47,086 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:47,087 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:47,089 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614347089,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:47,091 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:47,092 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36715 2024-11-26T09:45:47,096 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:47,096 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:47,096 INFO [Listener,port=36715 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36715: starting 2024-11-26T09:45:47,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d5bc80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:47,097 DEBUG [Thread-102 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:47,098 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,102 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614347102,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,103 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,104 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:47,104 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:47,105 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:47,106 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:47,106 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:47,106 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:47,106 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:47,107 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:47,107 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:47,107 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,107 DEBUG [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:47,107 INFO [Reader=1,bindAddress=localhost,port=36715 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:47,107 INFO [Reader=1,bindAddress=localhost,port=36715 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54562, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:47,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:47,114 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36715 2024-11-26T09:45:47,114 INFO [Listener,port=36715 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36715: stopping 2024-11-26T09:45:47,114 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:47,114 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:47,121 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=66 (was 64) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.encodeString(AbstractStreamEncoder.java:125) app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.encodeStringData(AbstractStreamEncoder.java:156) app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.encode(AbstractStreamEncoder.java:145) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.encodeMessage(EventChannelEncoder.java:378) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:172) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=892 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=3950 (was 3951) 2024-11-26T09:45:47,127 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=892, ProcessCount=11, AvailableMemoryMB=3950 2024-11-26T09:45:47,137 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:47,137 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:47,140 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614347139,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:47,141 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:47,142 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39971 2024-11-26T09:45:47,143 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:47,146 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:47,146 INFO [Listener,port=39971 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39971: starting 2024-11-26T09:45:47,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5e8e59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:47,147 DEBUG [Thread-105 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:47,151 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,155 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614347154,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,156 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,157 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:47,157 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:47,158 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:47,158 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:47,158 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:47,159 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:47,159 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:47,159 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:47,159 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,159 DEBUG [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:47,159 INFO [Reader=1,bindAddress=localhost,port=39971 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:47,160 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:47,161 INFO [Reader=1,bindAddress=localhost,port=39971 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41858, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:47,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:47,173 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39971 2024-11-26T09:45:47,173 INFO [Listener,port=39971 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39971: stopping 2024-11-26T09:45:47,173 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:47,173 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:47,173 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:47,174 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36833 2024-11-26T09:45:47,176 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:47,176 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:47,176 INFO [Listener,port=36833 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36833: starting 2024-11-26T09:45:47,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6351588d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:47,177 DEBUG [Thread-108 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:47,181 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,183 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,183 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:47,183 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:47,184 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:47,185 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:47,185 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:47,185 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:47,186 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:47,186 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:47,186 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:47,186 DEBUG [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:47,186 INFO [Reader=1,bindAddress=localhost,port=36833 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:47,187 INFO [Reader=1,bindAddress=localhost,port=36833 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41062, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,189 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:49,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,190 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:36833 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-26T09:45:49,190 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36833 2024-11-26T09:45:49,190 INFO [Listener,port=36833 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36833: stopping 2024-11-26T09:45:49,190 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:49,190 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:49,199 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 65) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.Thread.interrupt0(Native Method) java.base@17.0.11/java.lang.Thread.interrupt(Thread.java:1011) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.write(EventChannelEncoder.java:306) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:173) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 892), ProcessCount=11 (was 11), AvailableMemoryMB=3942 (was 3950) 2024-11-26T09:45:49,205 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3942 2024-11-26T09:45:49,215 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,215 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,217 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349217,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,219 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,220 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45369 2024-11-26T09:45:49,220 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:49,220 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,220 INFO [Listener,port=45369 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45369: starting 2024-11-26T09:45:49,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@307c34f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,221 DEBUG [Thread-111 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,223 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,224 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732614349218 2024-11-26T09:45:49,226 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614349226,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,228 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,228 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:49,228 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,229 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,230 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:49,230 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,230 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,230 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:49,230 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,230 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,230 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,230 DEBUG [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:49,230 INFO [Reader=1,bindAddress=localhost,port=45369 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,231 INFO [Reader=1,bindAddress=localhost,port=45369 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54710, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,237 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45369 2024-11-26T09:45:49,237 INFO [Listener,port=45369 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45369: stopping 2024-11-26T09:45:49,237 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:49,238 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:49,244 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69 (was 67) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3942 (was 3942) 2024-11-26T09:45:49,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3942 2024-11-26T09:45:49,261 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,261 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,264 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349264,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,266 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,267 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45907 2024-11-26T09:45:49,268 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:49,268 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,268 INFO [Listener,port=45907 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45907: starting 2024-11-26T09:45:49,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c08e3fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,269 DEBUG [Thread-114 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,271 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,275 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614349275,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,276 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,277 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:49,277 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,278 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,279 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:49,279 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,279 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,279 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-26T09:45:49,279 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,279 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,280 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,280 DEBUG [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:49,280 INFO [Reader=1,bindAddress=localhost,port=45907 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,280 INFO [Reader=1,bindAddress=localhost,port=45907 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41016, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,288 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45907 2024-11-26T09:45:49,288 INFO [Listener,port=45907 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45907: stopping 2024-11-26T09:45:49,288 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:49,288 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:49,297 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69 (was 68) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3942 (was 3942) 2024-11-26T09:45:49,305 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3941 2024-11-26T09:45:49,316 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,316 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,319 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349319,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,321 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,322 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41863 2024-11-26T09:45:49,322 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-26T09:45:49,322 INFO [Listener,port=41863 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41863: starting 2024-11-26T09:45:49,322 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a8319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,324 DEBUG [Thread-117 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,326 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,329 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614349329,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,331 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,331 INFO [Reader=1,bindAddress=localhost,port=41863 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-26T09:45:49,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,338 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41863 2024-11-26T09:45:49,338 INFO [Listener,port=41863 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41863: stopping 2024-11-26T09:45:49,338 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-26T09:45:49,338 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-26T09:45:49,344 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=71 (was 69) Potentially hanging thread: RpcServer.responder java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndExchangeByte(Unsafe.java:1535) java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetByte(Unsafe.java:1544) java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetBoolean(Unsafe.java:1784) java.base@17.0.11/java.lang.invoke.VarHandleBooleans$FieldInstanceReadWrite.compareAndSet(VarHandleBooleans.java:178) java.base@17.0.11/java.lang.invoke.VarHandleGuards.guard_LII_Z(VarHandleGuards.java:137) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:125) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3940 (was 3941) 2024-11-26T09:45:49,351 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=70, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3939 2024-11-26T09:45:49,361 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,361 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,363 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349363,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,365 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,365 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,366 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41575 2024-11-26T09:45:49,366 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b6d1c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,367 DEBUG [Thread-118 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,368 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,371 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614349371,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,372 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,372 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,373 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,374 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,374 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,375 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,375 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,375 DEBUG [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:49,375 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,376 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46226, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,376 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,384 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41575 2024-11-26T09:45:49,391 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72 (was 70) Potentially hanging thread: NettyRpcServer-19-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-19-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=475 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3939 (was 3939) 2024-11-26T09:45:49,397 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72, OpenFileDescriptor=475, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3939 2024-11-26T09:45:49,407 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,407 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,409 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349409,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,411 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,411 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,412 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37567 2024-11-26T09:45:49,412 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777a76a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-26T09:45:49,413 DEBUG [Thread-119 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,414 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,417 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614349417,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,419 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:49,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,419 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:37567 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-26T09:45:49,419 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37567 2024-11-26T09:45:49,426 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74 (was 72) Potentially hanging thread: NettyRpcServer-20-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-20-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=484 (was 475) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3938 (was 3939) 2024-11-26T09:45:49,432 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74, OpenFileDescriptor=484, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3938 2024-11-26T09:45:49,442 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,442 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,444 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349444,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,446 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,446 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,447 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36009 2024-11-26T09:45:49,447 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b5f774d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,448 DEBUG [Thread-120 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,449 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,452 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614349451,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,453 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,453 DEBUG [NettyRpcServer-21-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,454 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,455 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,455 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,455 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:49,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,456 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:36009 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-26T09:45:49,456 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36009 2024-11-26T09:45:49,463 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76 (was 74) Potentially hanging thread: NettyRpcServer-21-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-21-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=493 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3936 (was 3938) 2024-11-26T09:45:49,469 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3935 2024-11-26T09:45:49,479 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,479 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,482 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349481,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,483 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,484 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,485 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42879 2024-11-26T09:45:49,485 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f757f1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,486 DEBUG [Thread-121 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,486 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732614349482 2024-11-26T09:45:49,487 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,490 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614349490,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,491 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,491 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,492 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,493 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,493 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,494 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,494 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,494 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,494 DEBUG [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:49,494 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,494 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56236, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,501 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42879 2024-11-26T09:45:49,502 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,502 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,503 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46645 2024-11-26T09:45:49,503 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f54674e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,540 DEBUG [Thread-122 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,542 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,543 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,543 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,544 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,545 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,545 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,545 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,546 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,546 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,546 DEBUG [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:49,546 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,546 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37384, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,553 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46645 2024-11-26T09:45:49,554 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,554 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,555 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43131 2024-11-26T09:45:49,555 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4051982a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,556 DEBUG [Thread-123 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,557 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,558 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,558 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,559 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,560 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,560 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,560 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,560 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,560 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,561 DEBUG [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:49,561 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,561 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42508, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,567 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43131 2024-11-26T09:45:49,567 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,568 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,568 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34347 2024-11-26T09:45:49,568 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a020e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,569 DEBUG [Thread-124 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,570 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,571 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,572 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,573 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,574 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,574 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,575 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,575 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,575 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,575 DEBUG [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-26T09:45:49,575 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,576 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54872, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,584 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34347 2024-11-26T09:45:49,584 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,585 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,586 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46739 2024-11-26T09:45:49,586 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8e5713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,587 DEBUG [Thread-125 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,588 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,590 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,590 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,591 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,591 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,591 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,592 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,592 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,592 DEBUG [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:49,592 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,592 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,593 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:32958, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,602 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46739 2024-11-26T09:45:49,609 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86 (was 76) Potentially hanging thread: NettyRpcServer-25-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-26-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-22-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-23-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-22-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-24-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-26-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-24-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-25-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-23-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=538 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3927 (was 3935) 2024-11-26T09:45:49,616 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86, OpenFileDescriptor=538, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3926 2024-11-26T09:45:49,626 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,626 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,628 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349628,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,630 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,630 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,631 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40509 2024-11-26T09:45:49,631 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b6b2283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,632 DEBUG [Thread-126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-26T09:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-26T09:45:49,634 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-26T09:45:49,634 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-26T09:45:49,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,635 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40509 2024-11-26T09:45:49,643 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88 (was 86) Potentially hanging thread: NettyRpcServer-27-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-27-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=547 (was 538) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3924 (was 3926) 2024-11-26T09:45:49,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88, OpenFileDescriptor=547, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3924 2024-11-26T09:45:49,659 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,659 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,661 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349661,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,663 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,663 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,664 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40901 2024-11-26T09:45:49,664 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-26T09:45:49,664 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-26T09:45:49,664 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-26T09:45:49,664 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-26T09:45:49,664 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-26T09:45:49,664 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-26T09:45:49,664 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-26T09:45:49,664 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-26T09:45:49,664 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588db403, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,665 DEBUG [Thread-127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-26T09:45:49,667 WARN [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-26T09:45:49,667 INFO [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-26T09:45:49,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,674 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40901 2024-11-26T09:45:49,681 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90 (was 88) Potentially hanging thread: NettyRpcServer-28-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-28-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=556 (was 547) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3920 (was 3924) 2024-11-26T09:45:49,687 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90, OpenFileDescriptor=556, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3920 2024-11-26T09:45:49,698 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,698 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,700 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349700,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,701 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,702 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,703 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45155 2024-11-26T09:45:49,703 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665af319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,704 DEBUG [Thread-128 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,704 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,707 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614349707,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,708 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,708 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,709 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,710 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,710 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,711 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,711 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,711 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,711 DEBUG [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:49,711 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,711 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51756, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,717 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45155 2024-11-26T09:45:49,724 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92 (was 90) Potentially hanging thread: NettyRpcServer-29-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-29-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=565 (was 556) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3918 (was 3920) 2024-11-26T09:45:49,731 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92, OpenFileDescriptor=565, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3917 2024-11-26T09:45:49,741 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:49,741 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:49,743 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614349743,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:49,744 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,745 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,745 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42345 2024-11-26T09:45:49,746 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1afbebff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,746 DEBUG [Thread-129 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,747 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,750 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614349750,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,751 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,751 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,753 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,753 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,753 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,754 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,754 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,754 DEBUG [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:49,754 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:49,754 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,755 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36248, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:49,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:49,764 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42345 2024-11-26T09:45:49,765 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:49,765 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:49,766 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39343 2024-11-26T09:45:49,766 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:49,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dfe8685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:49,767 DEBUG [Thread-130 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:49,768 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,770 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,770 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,771 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:49,772 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,772 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:49,772 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:49,772 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:49,773 DEBUG [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-26T09:45:49,773 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:49,773 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36288, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:51,775 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-26T09:45:51,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:51,775 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:39343 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-26T09:45:51,775 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39343 2024-11-26T09:45:51,783 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96 (was 92) Potentially hanging thread: NettyRpcServer-31-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-30-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-30-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-31-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=583 (was 565) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3913 (was 3917) 2024-11-26T09:45:51,790 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3913 2024-11-26T09:45:51,791 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732614349743 2024-11-26T09:45:51,800 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:51,800 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:51,802 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614351802,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:51,804 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:51,804 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:51,805 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39959 2024-11-26T09:45:51,805 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:51,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1462322f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:51,807 DEBUG [Thread-131 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:51,808 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,811 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614351811,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,812 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:51,812 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,814 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:51,814 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:51,814 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:51,815 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:51,815 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:51,815 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,815 DEBUG [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:51,815 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:51,815 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37294, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:51,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:51,821 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39959 2024-11-26T09:45:51,829 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98 (was 96) Potentially hanging thread: NettyRpcServer-32-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-32-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=592 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3911 (was 3913) 2024-11-26T09:45:51,835 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3910 2024-11-26T09:45:51,845 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:51,845 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:51,847 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614351847,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:51,849 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:51,849 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:51,850 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46357 2024-11-26T09:45:51,850 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:51,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cd8d52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:51,851 DEBUG [Thread-132 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:51,852 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,855 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614351855,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,856 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:51,856 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,857 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-26T09:45:51,858 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:51,858 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-26T09:45:51,859 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-26T09:45:51,859 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:51,859 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,859 DEBUG [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-26T09:45:51,859 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-26T09:45:51,859 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48098, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-26T09:45:51,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:51,866 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46357 2024-11-26T09:45:51,874 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100 (was 98) Potentially hanging thread: NettyRpcServer-33-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-33-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=601 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3907 (was 3910) 2024-11-26T09:45:51,881 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100, OpenFileDescriptor=601, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=3907 2024-11-26T09:45:51,891 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-26T09:45:51,891 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-26T09:45:51,893 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732614351893,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-26T09:45:51,895 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-26T09:45:51,895 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T09:45:51,896 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43733 2024-11-26T09:45:51,896 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-26T09:45:51,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63fb59e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T09:45:51,897 DEBUG [Thread-133 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-26T09:45:51,898 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,900 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732614351900,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-26T09:45:51,902 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-26T09:45:51,902 INFO [NettyRpcServer-34-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-26T09:45:51,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T09:45:51,908 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43733 2024-11-26T09:45:51,916 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=102 (was 100) Potentially hanging thread: NettyRpcServer-34-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-34-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=610 (was 601) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=821 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=3904 (was 3907) 2024-11-26T09:45:51,916 INFO [Time-limited test {}] impl.DefaultInternalKdcServerImpl(102): Default Internal kdc server stopped.