2024-11-23 18:41:28,646 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-23 18:41:28,660 main DEBUG Took 0.012039 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 18:41:28,661 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 18:41:28,661 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 18:41:28,663 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 18:41:28,664 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,673 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 18:41:28,684 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,686 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,687 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,687 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,688 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,688 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,688 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,689 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,690 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,690 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,690 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,691 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,692 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,693 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,693 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,693 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,693 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,694 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 18:41:28,695 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,695 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 18:41:28,696 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 18:41:28,697 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 18:41:28,699 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 18:41:28,699 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 18:41:28,701 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 18:41:28,701 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 18:41:28,708 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 18:41:28,710 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 18:41:28,712 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 18:41:28,713 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 18:41:28,713 main DEBUG createAppenders(={Console}) 2024-11-23 18:41:28,714 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 initialized 2024-11-23 18:41:28,714 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 2024-11-23 18:41:28,714 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 OK. 2024-11-23 18:41:28,715 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 18:41:28,715 main DEBUG OutputStream closed 2024-11-23 18:41:28,715 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 18:41:28,715 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 18:41:28,716 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK 2024-11-23 18:41:28,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 18:41:28,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 18:41:28,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 18:41:28,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 18:41:28,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 18:41:28,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 18:41:28,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 18:41:28,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 18:41:28,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 18:41:28,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 18:41:28,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 18:41:28,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 18:41:28,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 18:41:28,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 18:41:28,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 18:41:28,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 18:41:28,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 18:41:28,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 18:41:28,792 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 18:41:28,793 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5812f68b) with optional ClassLoader: null 2024-11-23 18:41:28,793 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 18:41:28,794 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5812f68b] started OK. 2024-11-23T18:41:29,055 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/57f569e4-94f2-9bc3-61f7-0df603b187a9 2024-11-23 18:41:29,059 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 18:41:29,059 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T18:41:29,076 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestSecureIPC timeout: 52 mins 2024-11-23T18:41:29,296 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=13, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=9759 2024-11-23T18:41:29,401 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T18:41:29,479 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:29,484 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:29,541 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387289537,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:30,275 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:30,364 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T18:41:30,398 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T18:41:30,409 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T18:41:30,427 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43557 2024-11-23T18:41:30,432 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:30,432 INFO [Listener,port=43557 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43557: starting 2024-11-23T18:41:30,432 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:30,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e24486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:30,610 DEBUG [Thread-6 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:30,663 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43557 2024-11-23T18:41:30,674 DEBUG [Thread-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:30,719 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387290717,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:30,726 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:30,730 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:30,736 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:30,736 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:30,750 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:30,752 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:30,753 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:30,754 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:30,754 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:30,755 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:30,756 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:30,757 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:30,757 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-23T18:41:30,757 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:30,758 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:30,758 DEBUG [Thread-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:30,758 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:30,759 DEBUG [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:30,759 INFO [Reader=1,bindAddress=localhost,port=43557 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:30,782 INFO [Reader=1,bindAddress=localhost,port=43557 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33814, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:30,783 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-23T18:41:30,788 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:246 2024-11-23T18:41:30,791 DEBUG [BRPC Connection (465831974) to localhost:43557 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-23T18:41:30,823 DEBUG [BRPC Connection (465831974) to localhost:43557 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:232 2024-11-23T18:41:30,828 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2277 2024-11-23T18:41:30,830 DEBUG [BRPC Connection (465831974) to localhost:43557 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-23T18:41:30,834 DEBUG [BRPC Connection (465831974) to localhost:43557 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2263 2024-11-23T18:41:30,854 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-23T18:41:30,856 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18224 2024-11-23T18:41:30,862 DEBUG [BRPC Connection (465831974) to localhost:43557 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-23T18:41:30,883 DEBUG [BRPC Connection (465831974) to localhost:43557 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22327 2024-11-23T18:41:30,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:30,890 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43557 2024-11-23T18:41:30,891 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:30,891 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:30,891 INFO [Listener,port=43557 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43557: stopping 2024-11-23T18:41:30,897 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 13) Potentially hanging thread: pool-3-thread-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Listener,port=43557 java.base@17.0.11/java.lang.StringCoding.implEncodeAsciiArray(StringCoding.java:65) java.base@17.0.11/java.lang.System$2.encodeASCII(System.java:2418) java.base@17.0.11/sun.nio.cs.UTF_8$Encoder.encodeArrayLoop(UTF_8.java:456) java.base@17.0.11/sun.nio.cs.UTF_8$Encoder.encodeLoop(UTF_8.java:564) java.base@17.0.11/java.nio.charset.CharsetEncoder.encode(CharsetEncoder.java:585) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:64) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer idle connection scanner for port 43557 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=290 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=9646 (was 9759) 2024-11-23T18:41:30,903 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=9646 2024-11-23T18:41:30,913 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:30,914 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:30,921 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387290920,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:30,933 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:30,934 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44053 2024-11-23T18:41:30,935 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:30,935 INFO [Listener,port=44053 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44053: starting 2024-11-23T18:41:30,935 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:30,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@434b602e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-23T18:41:30,938 DEBUG [Thread-11 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:30,938 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44053 2024-11-23T18:41:30,940 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:30,952 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387290951,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:30,956 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:30,957 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:30,958 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:44053 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-23T18:41:30,963 WARN [Thread-11 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732387290923 2024-11-23T18:41:31,002 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,006 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext. 2024-11-23T18:41:31,006 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:31,007 DEBUG [Thread-11 {}] ipc.FailedServers(52): Added failed server with address localhost:44053 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,008 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44053 2024-11-23T18:41:31,008 INFO [Listener,port=44053 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44053: stopping 2024-11-23T18:41:31,008 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,008 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,013 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=9636 (was 9646) 2024-11-23T18:41:31,019 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=9636 2024-11-23T18:41:31,031 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:31,032 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:31,039 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387291038,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:31,043 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,045 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40277 2024-11-23T18:41:31,045 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,046 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,046 INFO [Listener,port=40277 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40277: starting 2024-11-23T18:41:31,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa89978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,047 DEBUG [Thread-14 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:31,047 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40277 2024-11-23T18:41:31,049 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,056 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387291055,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,059 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,059 DEBUG [Reader=1,bindAddress=localhost,port=40277 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,061 DEBUG [Reader=1,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,061 DEBUG [Reader=1,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,064 DEBUG [Reader=1,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,064 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,065 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,065 DEBUG [Reader=1,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,065 DEBUG [Reader=1,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,065 DEBUG [Reader=1,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,066 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,066 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:31,066 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:40277 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-23T18:41:31,067 WARN [Thread-14 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732387291041 2024-11-23T18:41:31,135 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,138 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,138 DEBUG [Reader=2,bindAddress=localhost,port=40277 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,139 DEBUG [Reader=2,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,139 DEBUG [Reader=2,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,141 DEBUG [Reader=2,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,142 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,142 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,143 DEBUG [Reader=2,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,143 DEBUG [Reader=2,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,143 DEBUG [Reader=2,bindAddress=localhost,port=40277 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,143 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,144 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:31,144 DEBUG [Thread-14 {}] ipc.FailedServers(52): Added failed server with address localhost:40277 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,145 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40277 2024-11-23T18:41:31,145 INFO [Listener,port=40277 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40277: stopping 2024-11-23T18:41:31,145 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,145 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,150 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.pattern.VariablesNotEmptyReplacementConverter.format(VariablesNotEmptyReplacementConverter.java:80) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:375) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:216) app//org.apache.logging.log4j.core.layout.PatternLayout.toSerializable(PatternLayout.java:59) app//org.apache.logging.log4j.core.layout.AbstractStringLayout.toByteArray(AbstractStringLayout.java:294) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:145) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=9621 (was 9636) 2024-11-23T18:41:31,156 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=9621 2024-11-23T18:41:31,170 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:31,170 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:31,175 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387291175,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:31,180 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,181 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45793 2024-11-23T18:41:31,182 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,182 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,182 INFO [Listener,port=45793 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45793: starting 2024-11-23T18:41:31,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@930e413, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,183 DEBUG [Thread-17 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:31,184 DEBUG [Thread-17 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45793 2024-11-23T18:41:31,185 DEBUG [Thread-17 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,193 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387291192,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,196 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,196 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,198 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,198 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,200 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,201 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,201 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,201 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,202 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,202 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,202 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,203 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:31,203 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:31,203 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,203 DEBUG [Thread-17 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:31,203 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,204 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,204 DEBUG [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:31,204 INFO [Reader=1,bindAddress=localhost,port=45793 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:31,205 INFO [Reader=1,bindAddress=localhost,port=45793 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47454, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:31,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,237 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45793 2024-11-23T18:41:31,237 INFO [Listener,port=45793 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45793: stopping 2024-11-23T18:41:31,237 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,237 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,238 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,240 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40981 2024-11-23T18:41:31,240 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,240 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,240 INFO [Listener,port=40981 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40981: starting 2024-11-23T18:41:31,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110107c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,242 DEBUG [Thread-20 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:31,243 DEBUG [Thread-20 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40981 2024-11-23T18:41:31,244 DEBUG [Thread-20 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,246 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,246 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,247 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,247 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,249 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,249 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,250 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,250 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,250 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,251 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,251 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,251 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:31,252 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:31,252 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,252 DEBUG [Thread-20 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:31,252 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,252 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,253 DEBUG [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:31,253 INFO [Reader=1,bindAddress=localhost,port=40981 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:31,253 INFO [Reader=1,bindAddress=localhost,port=40981 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47912, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:31,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,277 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40981 2024-11-23T18:41:31,278 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,278 INFO [Listener,port=40981 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40981: stopping 2024-11-23T18:41:31,278 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,278 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,279 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34741 2024-11-23T18:41:31,280 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,280 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,280 INFO [Listener,port=34741 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34741: starting 2024-11-23T18:41:31,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f3ea459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,281 DEBUG [Thread-23 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:31,282 DEBUG [Thread-23 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34741 2024-11-23T18:41:31,283 DEBUG [Thread-23 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,285 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,285 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,286 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,286 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,288 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,289 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,289 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,289 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,290 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,290 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,290 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,291 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:31,291 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:31,291 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,291 DEBUG [Thread-23 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:31,291 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,291 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,291 DEBUG [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:31,292 INFO [Reader=1,bindAddress=localhost,port=34741 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:31,292 INFO [Reader=1,bindAddress=localhost,port=34741 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48068, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:31,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,312 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34741 2024-11-23T18:41:31,313 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,313 INFO [Listener,port=34741 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34741: stopping 2024-11-23T18:41:31,313 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,313 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,315 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45429 2024-11-23T18:41:31,315 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,315 INFO [Listener,port=45429 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45429: starting 2024-11-23T18:41:31,315 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d844286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,316 DEBUG [Thread-26 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:31,317 DEBUG [Thread-26 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45429 2024-11-23T18:41:31,318 DEBUG [Thread-26 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,319 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,319 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,320 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,320 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,322 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,322 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,322 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,323 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,323 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,323 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,323 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,324 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:31,324 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-23T18:41:31,324 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,324 DEBUG [Thread-26 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:31,324 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,324 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,324 DEBUG [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-23T18:41:31,325 INFO [Reader=1,bindAddress=localhost,port=45429 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:31,325 INFO [Reader=1,bindAddress=localhost,port=45429 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34884, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:31,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,338 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45429 2024-11-23T18:41:31,338 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,338 INFO [Listener,port=45429 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45429: stopping 2024-11-23T18:41:31,338 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,339 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,340 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46421 2024-11-23T18:41:31,340 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,340 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,340 INFO [Listener,port=46421 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46421: starting 2024-11-23T18:41:31,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9d74ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,342 DEBUG [Thread-29 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:31,342 DEBUG [Thread-29 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46421 2024-11-23T18:41:31,343 DEBUG [Thread-29 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,345 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,346 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,346 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,347 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,348 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,348 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,348 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,348 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,349 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,349 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,349 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,349 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:31,350 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-23T18:41:31,350 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,350 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,350 DEBUG [Thread-29 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:31,350 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,350 DEBUG [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:31,350 INFO [Reader=1,bindAddress=localhost,port=46421 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:31,351 INFO [Reader=1,bindAddress=localhost,port=46421 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55896, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:31,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,368 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46421 2024-11-23T18:41:31,368 INFO [Listener,port=46421 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46421: stopping 2024-11-23T18:41:31,368 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,369 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,376 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.ThreadLocal$ThreadLocalMap.cleanSomeSlots(ThreadLocal.java:669) java.base@17.0.11/java.lang.ThreadLocal$ThreadLocalMap.set(ThreadLocal.java:500) java.base@17.0.11/java.lang.ThreadLocal.setInitialValue(ThreadLocal.java:199) java.base@17.0.11/java.lang.ThreadLocal.get(ThreadLocal.java:172) app//org.apache.maven.surefire.report.ClassMethodIndexer.getLocalIndex(ClassMethodIndexer.java:56) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:201) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=9604 (was 9621) 2024-11-23T18:41:31,382 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=9604 2024-11-23T18:41:31,393 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:31,394 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:31,399 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387291398,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:31,404 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,406 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38059 2024-11-23T18:41:31,406 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,406 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,406 INFO [Listener,port=38059 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38059: starting 2024-11-23T18:41:31,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d312e0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,408 DEBUG [Thread-32 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-23T18:41:31,408 DEBUG [Thread-32 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38059 2024-11-23T18:41:31,411 DEBUG [BRPC Connection (465831974) to localhost:38059 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-23T18:41:31,412 WARN [BRPC Connection (465831974) to localhost:38059 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-23T18:41:31,413 DEBUG [BRPC Connection (465831974) to localhost:38059 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-23T18:41:31,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,414 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38059 2024-11-23T18:41:31,414 INFO [Listener,port=38059 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38059: stopping 2024-11-23T18:41:31,414 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,414 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,422 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=9597 (was 9604) 2024-11-23T18:41:31,426 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=9596 2024-11-23T18:41:31,437 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:31,437 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:31,442 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387291441,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:31,445 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,446 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34437 2024-11-23T18:41:31,447 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-23T18:41:31,447 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-23T18:41:31,447 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-23T18:41:31,447 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-23T18:41:31,447 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-23T18:41:31,447 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-23T18:41:31,447 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-23T18:41:31,447 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-23T18:41:31,447 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,447 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,447 INFO [Listener,port=34437 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34437: starting 2024-11-23T18:41:31,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca57f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,449 DEBUG [Thread-35 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-23T18:41:31,449 DEBUG [Thread-35 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34437 2024-11-23T18:41:31,450 WARN [Reader=1,bindAddress=localhost,port=34437 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-23T18:41:31,450 INFO [Reader=1,bindAddress=localhost,port=34437 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-23T18:41:31,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,459 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34437 2024-11-23T18:41:31,459 INFO [Listener,port=34437 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34437: stopping 2024-11-23T18:41:31,459 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,460 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,465 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.impl.ThreadContextDataInjector$ForDefaultThreadContextMap.injectContextData(ThreadContextDataInjector.java:118) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.createContextData(Log4jLogEvent.java:497) app//org.apache.logging.log4j.core.impl.Log4jLogEvent.(Log4jLogEvent.java:355) app//org.apache.logging.log4j.core.impl.DefaultLogEventFactory.createEvent(DefaultLogEventFactory.java:74) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:550) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=9586 (was 9596) 2024-11-23T18:41:31,469 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=9585 2024-11-23T18:41:31,482 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:31,482 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:31,487 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387291486,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:31,490 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,491 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46185 2024-11-23T18:41:31,492 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,492 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,492 INFO [Listener,port=46185 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46185: starting 2024-11-23T18:41:31,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b04a514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,493 DEBUG [Thread-38 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:31,494 DEBUG [Thread-38 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46185 2024-11-23T18:41:31,495 DEBUG [Thread-38 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,502 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387291502,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,505 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,505 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,506 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,506 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,508 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,509 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,509 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,509 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,509 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,510 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,512 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,512 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:31,513 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:31,513 DEBUG [Thread-38 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:31,513 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,513 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,513 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,513 DEBUG [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:31,514 INFO [Reader=1,bindAddress=localhost,port=46185 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:31,514 INFO [Reader=1,bindAddress=localhost,port=46185 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44958, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:31,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,523 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46185 2024-11-23T18:41:31,524 INFO [Listener,port=46185 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46185: stopping 2024-11-23T18:41:31,524 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,524 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,529 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.nio.ByteBuffer.limit(ByteBuffer.java:1529) java.base@17.0.11/java.nio.ByteBuffer.limit(ByteBuffer.java:267) java.base@17.0.11/java.nio.Buffer.(Buffer.java:245) java.base@17.0.11/java.nio.ByteBuffer.(ByteBuffer.java:288) java.base@17.0.11/java.nio.HeapByteBuffer.(HeapByteBuffer.java:64) java.base@17.0.11/java.nio.ByteBuffer.allocate(ByteBuffer.java:363) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.encodeMessage(EventChannelEncoder.java:377) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:172) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=9569 (was 9585) 2024-11-23T18:41:31,535 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=9568 2024-11-23T18:41:31,546 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:31,546 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:31,551 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387291550,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:31,553 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,554 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34199 2024-11-23T18:41:31,555 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,555 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,555 INFO [Listener,port=34199 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34199: starting 2024-11-23T18:41:31,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff36aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,556 DEBUG [Thread-41 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:31,556 DEBUG [Thread-41 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34199 2024-11-23T18:41:31,557 DEBUG [Thread-41 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,563 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387291562,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,564 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,565 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,565 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,565 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,567 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,567 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,567 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,567 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,567 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,568 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,568 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,568 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:31,568 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-23T18:41:31,568 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,568 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,568 DEBUG [Thread-41 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:31,569 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,569 DEBUG [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:31,569 INFO [Reader=1,bindAddress=localhost,port=34199 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:31,569 INFO [Reader=1,bindAddress=localhost,port=34199 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38768, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:31,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:31,580 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34199 2024-11-23T18:41:31,580 INFO [Listener,port=34199 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34199: stopping 2024-11-23T18:41:31,580 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:31,580 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:31,581 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:31,582 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38365 2024-11-23T18:41:31,583 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:31,583 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:31,583 INFO [Listener,port=38365 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38365: starting 2024-11-23T18:41:31,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ffee6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:31,584 DEBUG [Thread-44 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:31,584 DEBUG [Thread-44 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38365 2024-11-23T18:41:31,585 DEBUG [Thread-44 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,587 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:31,587 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,588 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,588 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,589 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:31,589 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:31,590 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:31,590 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,590 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,590 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:31,590 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:31,590 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:31,591 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-23T18:41:31,591 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:31,591 DEBUG [Thread-44 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:31,591 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:31,591 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:31,591 DEBUG [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:31,591 INFO [Reader=1,bindAddress=localhost,port=38365 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:31,592 INFO [Reader=1,bindAddress=localhost,port=38365 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39348, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:33,593 ERROR [Thread-44 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39348 remote=localhost/127.0.0.1:38365] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-23T18:41:33,593 DEBUG [Thread-44 {}] ipc.FailedServers(52): Added failed server with address localhost:38365 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-23T18:41:33,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:33,594 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38365 2024-11-23T18:41:33,594 INFO [Listener,port=38365 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38365: stopping 2024-11-23T18:41:33,594 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:33,594 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:33,600 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 170) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9523 (was 9568) 2024-11-23T18:41:33,606 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9523 2024-11-23T18:41:33,623 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:33,623 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:33,628 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387293628,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:33,632 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:33,639 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37149 2024-11-23T18:41:33,641 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:33,644 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:33,644 INFO [Listener,port=37149 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37149: starting 2024-11-23T18:41:33,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f6b41d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:33,646 DEBUG [Thread-47 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:33,646 DEBUG [Thread-47 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37149 2024-11-23T18:41:33,647 DEBUG [Thread-47 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,655 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387293654,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,656 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:33,657 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,657 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:33,657 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:33,659 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:33,659 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:33,659 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:33,659 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:33,659 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:33,660 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:33,660 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:33,660 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:33,660 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:33,660 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:33,660 DEBUG [Thread-47 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:33,660 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:33,661 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,661 DEBUG [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:33,661 INFO [Reader=1,bindAddress=localhost,port=37149 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:33,661 INFO [Reader=1,bindAddress=localhost,port=37149 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49842, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:33,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:33,674 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37149 2024-11-23T18:41:33,674 INFO [Listener,port=37149 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37149: stopping 2024-11-23T18:41:33,674 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:33,676 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:33,681 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeBytes(OutputStreamManager.java:353) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:96) app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68) app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: Listener,port=37149 java.base@17.0.11/java.io.BufferedOutputStream.write(BufferedOutputStream.java:118) app//org.apache.maven.surefire.api.util.internal.Channels$4.writeImpl(Channels.java:199) app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.write(AbstractNoninterruptibleWritableChannel.java:66) app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.writeBuffered(AbstractNoninterruptibleWritableChannel.java:49) app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.write(AbstractStreamEncoder.java:69) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.write(EventChannelEncoder.java:288) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:173) app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99) app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43) app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202) app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57) app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283) app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252) Potentially hanging thread: RpcServer idle connection scanner for port 37149 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9504 (was 9523) 2024-11-23T18:41:33,688 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9502 2024-11-23T18:41:33,703 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:33,703 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:33,708 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387293707,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:33,711 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:33,713 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42595 2024-11-23T18:41:33,713 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:33,713 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:33,713 INFO [Listener,port=42595 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42595: starting 2024-11-23T18:41:33,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37e2f4fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:33,715 DEBUG [Thread-50 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:33,715 DEBUG [Thread-50 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42595 2024-11-23T18:41:33,716 DEBUG [Thread-50 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,721 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387293720,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,722 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:33,722 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,723 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:33,723 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:33,724 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:33,724 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:33,725 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:33,725 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:33,725 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:33,725 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:33,725 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:33,725 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:33,726 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:33,726 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:33,726 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:33,726 DEBUG [Thread-50 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:33,726 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,726 DEBUG [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:33,726 INFO [Reader=1,bindAddress=localhost,port=42595 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:33,726 INFO [Reader=1,bindAddress=localhost,port=42595 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34370, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:33,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:33,736 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42595 2024-11-23T18:41:33,737 INFO [Listener,port=42595 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42595: stopping 2024-11-23T18:41:33,737 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:33,737 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:33,743 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9484 (was 9502) 2024-11-23T18:41:33,748 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9484 2024-11-23T18:41:33,763 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:33,763 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:33,767 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387293767,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:33,770 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:33,771 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=32859 2024-11-23T18:41:33,772 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:33,772 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:33,772 INFO [Listener,port=32859 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=32859: starting 2024-11-23T18:41:33,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533fa56c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:33,774 DEBUG [Thread-53 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:33,774 DEBUG [Thread-53 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:32859 2024-11-23T18:41:33,776 DEBUG [Thread-53 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,783 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387293782,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,784 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:33,784 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-23T18:41:33,785 DEBUG [Thread-53 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:33,785 INFO [Reader=1,bindAddress=localhost,port=32859 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-23T18:41:33,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:33,794 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 32859 2024-11-23T18:41:33,794 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:33,794 INFO [Listener,port=32859 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=32859: stopping 2024-11-23T18:41:33,795 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:33,800 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19) Potentially hanging thread: RpcServer.responder Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/java.lang.Thread.exit(Thread.java:864) Potentially hanging thread: RpcServer idle connection scanner for port 32859 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Listener,port=32859 app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9467 (was 9484) 2024-11-23T18:41:33,807 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9466 2024-11-23T18:41:33,821 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:33,821 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:33,826 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387293825,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:33,828 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:33,839 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:33,842 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T18:41:33,871 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5117 (auto-detected) 2024-11-23T18:41:33,872 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T18:41:33,890 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39049 2024-11-23T18:41:33,890 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:33,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cd2d84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:33,892 DEBUG [Thread-54 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:33,892 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39049 2024-11-23T18:41:33,893 DEBUG [Thread-54 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,897 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387293897,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,899 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:33,918 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:33,919 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,920 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 107 from saslServer. 2024-11-23T18:41:33,923 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(119): Will read input token of size 107 for processing by initSASLContext 2024-11-23T18:41:33,924 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:33,924 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:33,924 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:33,925 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:33,925 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:33,925 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-23T18:41:33,925 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:33,925 DEBUG [Thread-54 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:33,925 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,925 DEBUG [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:33,926 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:33,930 INFO [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33544, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:33,930 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93 2024-11-23T18:41:33,930 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:248 2024-11-23T18:41:33,931 DEBUG [BRPC Connection (465831974) to localhost:39049 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-23T18:41:33,933 DEBUG [BRPC Connection (465831974) to localhost:39049 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:234 2024-11-23T18:41:33,934 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2249 2024-11-23T18:41:33,934 DEBUG [BRPC Connection (465831974) to localhost:39049 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-23T18:41:33,935 DEBUG [BRPC Connection (465831974) to localhost:39049 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2235 2024-11-23T18:41:33,937 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117 2024-11-23T18:41:33,937 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18262 2024-11-23T18:41:33,938 DEBUG [BRPC Connection (465831974) to localhost:39049 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet 2024-11-23T18:41:33,945 DEBUG [BRPC Connection (465831974) to localhost:39049 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22365 2024-11-23T18:41:33,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:33,946 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39049 2024-11-23T18:41:33,953 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21 (was 19) Potentially hanging thread: NettyRpcServer-2-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-2-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=299 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9436 (was 9466) 2024-11-23T18:41:33,958 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9436 2024-11-23T18:41:33,968 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:33,968 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:33,972 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387293972,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:33,974 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:33,975 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:33,976 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44243 2024-11-23T18:41:33,976 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:33,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e988b3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-23T18:41:33,977 DEBUG [Thread-55 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:33,977 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44243 2024-11-23T18:41:33,979 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,984 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387293984,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:33,986 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:33,986 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:33,986 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:44243 org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-23T18:41:33,987 WARN [Thread-55 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732387293973 2024-11-23T18:41:34,081 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,084 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,084 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:34,084 DEBUG [Thread-55 {}] ipc.FailedServers(52): Added failed server with address localhost:44243 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,085 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44243 2024-11-23T18:41:34,091 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24 (was 21) Potentially hanging thread: NettyRpcServer-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=308 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9419 (was 9436) 2024-11-23T18:41:34,098 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9418 2024-11-23T18:41:34,111 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:34,112 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:34,116 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387294116,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:34,119 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,119 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,120 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34309 2024-11-23T18:41:34,120 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d681140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,121 DEBUG [Thread-56 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:34,122 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34309 2024-11-23T18:41:34,123 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,130 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387294130,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,132 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,132 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,132 DEBUG [NettyRpcServer-4-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,135 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,135 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,135 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,136 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,136 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,137 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,137 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:34,137 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:34309 javax.security.sasl.SaslException: No common protection layer between client and server at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?] at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-23T18:41:34,138 WARN [Thread-56 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732387294117 2024-11-23T18:41:34,234 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,238 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,238 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,238 DEBUG [NettyRpcServer-4-3 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,241 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,241 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,242 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,242 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,242 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,243 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,243 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:34,243 DEBUG [Thread-56 {}] ipc.FailedServers(52): Added failed server with address localhost:34309 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,243 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34309 2024-11-23T18:41:34,248 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27 (was 24) Potentially hanging thread: NettyRpcServer-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=317 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9403 (was 9418) 2024-11-23T18:41:34,252 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9401 2024-11-23T18:41:34,263 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:34,263 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:34,267 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387294266,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:34,269 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,269 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,270 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36975 2024-11-23T18:41:34,270 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b177205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,271 DEBUG [Thread-57 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:34,271 DEBUG [Thread-57 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36975 2024-11-23T18:41:34,272 DEBUG [Thread-57 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,279 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387294279,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,281 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,281 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,281 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,283 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,284 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,284 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,285 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,285 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,285 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,285 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:34,286 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:34,286 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,286 DEBUG [Thread-57 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:34,286 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,286 DEBUG [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:34,286 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:34,287 INFO [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40308, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:34,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,297 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36975 2024-11-23T18:41:34,298 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,298 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,299 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41929 2024-11-23T18:41:34,299 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@182caf4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,300 DEBUG [Thread-58 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:34,300 DEBUG [Thread-58 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41929 2024-11-23T18:41:34,301 DEBUG [Thread-58 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,303 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,303 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,303 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,304 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,304 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,305 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,305 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,305 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,305 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,305 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:34,305 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:34,305 DEBUG [Thread-58 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:34,306 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,306 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,306 DEBUG [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:34,306 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:34,306 INFO [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56370, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:34,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,313 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41929 2024-11-23T18:41:34,313 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,314 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,314 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39331 2024-11-23T18:41:34,315 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26391bf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,315 DEBUG [Thread-59 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:34,316 DEBUG [Thread-59 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39331 2024-11-23T18:41:34,317 DEBUG [Thread-59 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,318 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,318 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,318 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,319 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,320 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,320 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,320 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,320 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,321 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,321 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:34,321 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:34,321 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,321 DEBUG [Thread-59 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:34,321 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,321 DEBUG [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:34,321 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:34,322 INFO [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50152, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:34,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,328 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39331 2024-11-23T18:41:34,329 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,330 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,330 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44909 2024-11-23T18:41:34,330 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45a9e7de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,331 DEBUG [Thread-60 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:34,332 DEBUG [Thread-60 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44909 2024-11-23T18:41:34,332 DEBUG [Thread-60 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,334 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,334 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,334 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,335 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,336 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,336 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,336 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,336 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,337 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,337 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:34,337 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int 2024-11-23T18:41:34,337 DEBUG [Thread-60 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:34,337 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,337 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,337 DEBUG [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-23T18:41:34,337 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:34,338 INFO [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54032, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:34,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,347 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44909 2024-11-23T18:41:34,348 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,348 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,349 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45437 2024-11-23T18:41:34,349 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5544af88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,350 DEBUG [Thread-61 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:34,351 DEBUG [Thread-61 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45437 2024-11-23T18:41:34,351 DEBUG [Thread-61 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,353 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,353 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,353 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,355 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,355 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,355 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,356 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,356 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,356 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,356 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:34,357 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-23T18:41:34,357 DEBUG [Thread-61 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:34,357 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,357 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,357 DEBUG [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:34,357 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:34,358 INFO [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39506, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:34,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,369 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45437 2024-11-23T18:41:34,375 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37 (was 27) Potentially hanging thread: NettyRpcServer-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=362 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9377 (was 9401) 2024-11-23T18:41:34,380 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9376 2024-11-23T18:41:34,398 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:34,398 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:34,402 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387294402,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:34,404 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,405 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,406 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35755 2024-11-23T18:41:34,406 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195718a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,407 DEBUG [Thread-62 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-23T18:41:34,407 DEBUG [Thread-62 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35755 2024-11-23T18:41:34,409 DEBUG [BRPC Connection (465831974) to localhost:35755 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-23T18:41:34,409 WARN [BRPC Connection (465831974) to localhost:35755 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-23T18:41:34,409 DEBUG [BRPC Connection (465831974) to localhost:35755 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-23T18:41:34,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,409 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35755 2024-11-23T18:41:34,414 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 37) Potentially hanging thread: NettyRpcServer-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=371 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9365 (was 9376) 2024-11-23T18:41:34,419 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9364 2024-11-23T18:41:34,430 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:34,430 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:34,434 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387294434,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:34,437 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,437 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,438 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44923 2024-11-23T18:41:34,438 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-23T18:41:34,438 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-23T18:41:34,438 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-23T18:41:34,438 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-23T18:41:34,438 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-23T18:41:34,438 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-23T18:41:34,438 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-23T18:41:34,438 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-23T18:41:34,438 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f47b13c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,440 DEBUG [Thread-63 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-23T18:41:34,440 DEBUG [Thread-63 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44923 2024-11-23T18:41:34,442 WARN [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-23T18:41:34,442 INFO [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-23T18:41:34,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,449 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44923 2024-11-23T18:41:34,455 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41 (was 39) Potentially hanging thread: NettyRpcServer-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=380 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9352 (was 9364) 2024-11-23T18:41:34,460 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9352 2024-11-23T18:41:34,471 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:34,471 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:34,475 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387294475,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:34,478 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,478 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,479 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38301 2024-11-23T18:41:34,479 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2eb24d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,480 DEBUG [Thread-64 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:34,480 DEBUG [Thread-64 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38301 2024-11-23T18:41:34,481 DEBUG [Thread-64 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,487 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387294486,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,488 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,488 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,489 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,490 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,491 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,491 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,491 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,492 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,492 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,492 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:34,492 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:34,493 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,493 DEBUG [Thread-64 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:34,493 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,493 DEBUG [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:34,493 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:34,493 INFO [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60532, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:34,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,501 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38301 2024-11-23T18:41:34,507 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43 (was 41) Potentially hanging thread: NettyRpcServer-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=389 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9336 (was 9352) 2024-11-23T18:41:34,513 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9334 2024-11-23T18:41:34,528 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:34,528 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:34,532 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387294531,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:34,534 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,535 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,536 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35739 2024-11-23T18:41:34,536 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ea265a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,537 DEBUG [Thread-65 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:34,537 DEBUG [Thread-65 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35739 2024-11-23T18:41:34,538 DEBUG [Thread-65 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,543 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387294542,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,544 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,544 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,544 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,546 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,546 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,546 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,547 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,547 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,547 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,547 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:34,547 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-23T18:41:34,547 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,547 DEBUG [Thread-65 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:34,547 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,548 DEBUG [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:34,548 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:34,548 INFO [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44258, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:34,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:34,559 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35739 2024-11-23T18:41:34,559 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:34,560 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:34,561 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40595 2024-11-23T18:41:34,561 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:34,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283bce6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:34,562 DEBUG [Thread-66 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:34,562 DEBUG [Thread-66 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40595 2024-11-23T18:41:34,563 DEBUG [Thread-66 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,565 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:34,565 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,565 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,566 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:34,567 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:34,567 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:34,567 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,567 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:34,568 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:34,568 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:34,568 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf 2024-11-23T18:41:34,568 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:34,568 DEBUG [Thread-66 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:34,568 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:34,568 DEBUG [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:34,568 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:34,569 INFO [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57608, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:36,570 ERROR [Thread-66 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function. java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:57608 remote=localhost/127.0.0.1:40595] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?] at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?] at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?] at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?] 2024-11-23T18:41:36,572 DEBUG [Thread-66 {}] ipc.FailedServers(52): Added failed server with address localhost:40595 to list caused by java.io.IOException: Timeout while waiting connection header response 2024-11-23T18:41:36,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,573 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40595 2024-11-23T18:41:36,585 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47 (was 43) Potentially hanging thread: NettyRpcServer-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=407 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9313 (was 9334) 2024-11-23T18:41:36,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9313 2024-11-23T18:41:36,602 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:36,602 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:36,605 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387296605,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:36,607 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,607 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:36,609 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40939 2024-11-23T18:41:36,609 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb16e71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,610 DEBUG [Thread-67 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,610 DEBUG [Thread-67 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40939 2024-11-23T18:41:36,611 DEBUG [Thread-67 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,616 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387296616,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,618 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:36,618 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,618 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,619 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:36,619 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:36,620 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:36,620 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,620 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:36,620 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:36,620 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:36,621 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:36,621 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,621 DEBUG [Thread-67 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:36,621 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,621 DEBUG [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:36,621 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:36,621 INFO [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39462, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:36,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,629 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40939 2024-11-23T18:41:36,636 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49 (was 47) Potentially hanging thread: NettyRpcServer-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=416 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9297 (was 9313) 2024-11-23T18:41:36,641 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9297 2024-11-23T18:41:36,651 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:36,651 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:36,655 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387296654,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:36,656 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,657 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:36,658 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42805 2024-11-23T18:41:36,658 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@389e4ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,659 DEBUG [Thread-68 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,659 DEBUG [Thread-68 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42805 2024-11-23T18:41:36,660 DEBUG [Thread-68 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,664 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387296664,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,665 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:36,666 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,666 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,667 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:36,668 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext 2024-11-23T18:41:36,668 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext. 2024-11-23T18:41:36,668 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,668 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:36,668 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext 2024-11-23T18:41:36,669 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext. 2024-11-23T18:41:36,669 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth 2024-11-23T18:41:36,669 DEBUG [Thread-68 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:36,669 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,669 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,669 DEBUG [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:36,669 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:36,669 INFO [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40076, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:36,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,675 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42805 2024-11-23T18:41:36,681 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51 (was 49) Potentially hanging thread: NettyRpcServer-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=425 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9282 (was 9297) 2024-11-23T18:41:36,686 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9282 2024-11-23T18:41:36,698 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:36,698 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:36,701 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387296701,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:36,703 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,703 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:36,704 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40169 2024-11-23T18:41:36,704 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ae8233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,705 DEBUG [Thread-69 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,705 DEBUG [Thread-69 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40169 2024-11-23T18:41:36,706 DEBUG [Thread-69 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,711 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387296710,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,712 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext. 2024-11-23T18:41:36,712 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth. 2024-11-23T18:41:36,712 DEBUG [Thread-69 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:36,712 INFO [NettyRpcServer-17-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-23T18:41:36,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,720 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40169 2024-11-23T18:41:36,727 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=53 (was 51) Potentially hanging thread: NettyRpcServer-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=434 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9266 (was 9282) 2024-11-23T18:41:36,732 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=53, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9266 2024-11-23T18:41:36,743 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:36,743 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:36,746 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387296746,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:36,749 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,750 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35843 2024-11-23T18:41:36,750 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,750 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:36,750 INFO [Listener,port=35843 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35843: starting 2024-11-23T18:41:36,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cee337, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,753 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T18:41:36,753 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T18:41:36,757 DEBUG [Thread-72 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,771 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,778 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387296777,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,780 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,781 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,781 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,781 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:36,782 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,782 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,782 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:36,783 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,783 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,783 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,783 DEBUG [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:36,783 INFO [Reader=1,bindAddress=localhost,port=35843 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:36,788 INFO [Reader=1,bindAddress=localhost,port=35843 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40056, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:36,788 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:36,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,799 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35843 2024-11-23T18:41:36,799 INFO [Listener,port=35843 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35843: stopping 2024-11-23T18:41:36,799 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:36,799 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:36,805 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54 (was 53) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-1 java.base@17.0.11/java.lang.StringConcatHelper.simpleConcat(StringConcatHelper.java:424) java.base@17.0.11/java.lang.String.concat(String.java:2774) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.definePackage(BuiltinClassLoader.java:941) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineOrCheckPackage(BuiltinClassLoader.java:881) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.defineOrCheckPackage(ClassLoaders.java:211) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:849) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639) java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525) app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:294) app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:290) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.initialize(FastThreadLocal.java:177) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.get(FastThreadLocal.java:142) app//org.apache.hadoop.hbase.ipc.IPCUtil.execute(IPCUtil.java:304) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown(NettyRpcConnection.java:146) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:161) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 434) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9246 (was 9266) 2024-11-23T18:41:36,810 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9246 2024-11-23T18:41:36,820 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:36,820 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:36,825 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387296824,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:36,827 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,828 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37337 2024-11-23T18:41:36,828 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:36,828 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,828 INFO [Listener,port=37337 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37337: starting 2024-11-23T18:41:36,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c0063f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-23T18:41:36,829 DEBUG [Thread-75 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,831 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,835 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387296835,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,837 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:36,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,838 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:37337 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-23T18:41:36,838 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37337 2024-11-23T18:41:36,838 INFO [Listener,port=37337 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37337: stopping 2024-11-23T18:41:36,838 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:36,838 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:36,844 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55 (was 54) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9230 (was 9246) 2024-11-23T18:41:36,850 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9230 2024-11-23T18:41:36,863 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:36,863 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:36,866 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387296866,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:36,868 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,869 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41415 2024-11-23T18:41:36,869 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,869 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:36,869 INFO [Listener,port=41415 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41415: starting 2024-11-23T18:41:36,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db88aea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,870 DEBUG [Thread-78 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,872 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,876 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387296876,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,878 DEBUG [Reader=1,bindAddress=localhost,port=41415 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,879 DEBUG [Reader=1,bindAddress=localhost,port=41415 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,879 DEBUG [Reader=1,bindAddress=localhost,port=41415 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,880 DEBUG [Reader=1,bindAddress=localhost,port=41415 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:36,880 DEBUG [Reader=1,bindAddress=localhost,port=41415 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,880 DEBUG [Reader=1,bindAddress=localhost,port=41415 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,881 DEBUG [Reader=1,bindAddress=localhost,port=41415 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:36,882 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:36,882 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:41415 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-23T18:41:36,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,882 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41415 2024-11-23T18:41:36,883 INFO [Listener,port=41415 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41415: stopping 2024-11-23T18:41:36,883 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:36,883 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:36,889 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57 (was 55) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Relogin-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9215 (was 9230) 2024-11-23T18:41:36,891 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732387296867 2024-11-23T18:41:36,896 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9214 2024-11-23T18:41:36,906 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:36,906 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:36,909 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387296909,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:36,911 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,913 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41227 2024-11-23T18:41:36,913 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,913 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:36,913 INFO [Listener,port=41227 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41227: starting 2024-11-23T18:41:36,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d3af291, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,914 DEBUG [Thread-81 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,915 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,925 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387296925,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,926 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,927 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,927 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,928 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:36,928 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,928 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,929 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:36,929 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,929 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,929 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:36,929 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,929 DEBUG [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:36,929 INFO [Reader=1,bindAddress=localhost,port=41227 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:36,930 INFO [Reader=1,bindAddress=localhost,port=41227 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50438, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:36,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,935 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41227 2024-11-23T18:41:36,935 INFO [Listener,port=41227 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41227: stopping 2024-11-23T18:41:36,935 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:36,935 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:36,936 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,937 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34543 2024-11-23T18:41:36,937 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,937 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:36,937 INFO [Listener,port=34543 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34543: starting 2024-11-23T18:41:36,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ceae01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,938 DEBUG [Thread-84 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,940 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,942 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,942 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,942 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,943 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:36,943 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,943 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,943 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:36,944 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,944 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:36,944 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,944 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,944 DEBUG [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:36,944 INFO [Reader=1,bindAddress=localhost,port=34543 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:36,945 INFO [Reader=1,bindAddress=localhost,port=34543 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43724, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:36,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,950 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34543 2024-11-23T18:41:36,950 INFO [Listener,port=34543 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34543: stopping 2024-11-23T18:41:36,950 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:36,950 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:36,951 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,951 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41289 2024-11-23T18:41:36,952 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:36,952 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,952 INFO [Listener,port=41289 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41289: starting 2024-11-23T18:41:36,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3642942b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,953 DEBUG [Thread-87 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,954 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,956 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,956 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,956 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,957 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:36,957 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,957 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,958 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:36,958 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,958 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,958 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:36,958 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,958 DEBUG [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:36,958 INFO [Reader=1,bindAddress=localhost,port=41289 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:36,959 INFO [Reader=1,bindAddress=localhost,port=41289 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42156, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:36,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,967 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41289 2024-11-23T18:41:36,967 INFO [Listener,port=41289 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41289: stopping 2024-11-23T18:41:36,967 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:36,967 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:36,967 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,968 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44413 2024-11-23T18:41:36,969 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:36,969 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,969 INFO [Listener,port=44413 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44413: starting 2024-11-23T18:41:36,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6078785a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,970 DEBUG [Thread-90 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,971 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,972 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,973 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,973 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,973 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:36,974 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,974 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,974 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:36,974 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,974 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,974 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,974 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:36,974 DEBUG [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-23T18:41:36,975 INFO [Reader=1,bindAddress=localhost,port=44413 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:36,975 INFO [Reader=1,bindAddress=localhost,port=44413 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49964, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:36,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:36,983 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44413 2024-11-23T18:41:36,983 INFO [Listener,port=44413 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44413: stopping 2024-11-23T18:41:36,983 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:36,983 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:36,984 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:36,985 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33883 2024-11-23T18:41:36,985 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:36,985 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:36,985 INFO [Listener,port=33883 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33883: starting 2024-11-23T18:41:36,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@436da6e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:36,986 DEBUG [Thread-93 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:36,990 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,991 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,992 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,992 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,994 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:36,994 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,994 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,995 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:36,995 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:36,995 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:36,995 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:36,996 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:36,996 DEBUG [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:36,996 INFO [Reader=1,bindAddress=localhost,port=33883 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:36,997 INFO [Reader=1,bindAddress=localhost,port=33883 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54268, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:37,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:37,005 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33883 2024-11-23T18:41:37,005 INFO [Listener,port=33883 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33883: stopping 2024-11-23T18:41:37,005 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:37,006 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:37,011 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63 (was 57) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder app//org.apache.logging.log4j.core.pattern.VariablesNotEmptyReplacementConverter.format(VariablesNotEmptyReplacementConverter.java:80) app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44) app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384) app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:240) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:225) app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208) app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199) app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158) app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134) app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125) app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89) app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666) app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616) app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552) app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82) app//org.apache.logging.log4j.core.Logger.log(Logger.java:161) app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9189 (was 9214) 2024-11-23T18:41:37,017 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9189 2024-11-23T18:41:37,026 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:37,026 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:37,028 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387297028,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:37,029 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:37,030 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45473 2024-11-23T18:41:37,031 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:37,031 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:37,031 INFO [Listener,port=45473 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45473: starting 2024-11-23T18:41:37,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6046a1a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:37,032 DEBUG [Thread-96 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-23T18:41:37,034 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-23T18:41:37,034 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-23T18:41:37,034 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-23T18:41:37,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:37,035 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45473 2024-11-23T18:41:37,035 INFO [Listener,port=45473 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45473: stopping 2024-11-23T18:41:37,035 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:37,035 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:37,040 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63 (was 62) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9188 (was 9189) 2024-11-23T18:41:37,045 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9188 2024-11-23T18:41:37,053 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:37,053 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:37,055 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387297055,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:37,057 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:37,058 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35493 2024-11-23T18:41:37,058 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-23T18:41:37,058 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-23T18:41:37,058 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-23T18:41:37,058 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-23T18:41:37,058 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-23T18:41:37,058 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-23T18:41:37,058 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-23T18:41:37,058 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-23T18:41:37,059 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:37,059 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:37,059 INFO [Listener,port=35493 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35493: starting 2024-11-23T18:41:37,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bba0158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:37,059 DEBUG [Thread-99 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-23T18:41:37,061 WARN [Reader=1,bindAddress=localhost,port=35493 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-23T18:41:37,061 INFO [Reader=1,bindAddress=localhost,port=35493 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-23T18:41:37,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:37,067 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35493 2024-11-23T18:41:37,067 INFO [Listener,port=35493 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35493: stopping 2024-11-23T18:41:37,068 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:37,068 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:37,072 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=66 (was 63) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-10 app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.initialize(FastThreadLocal.java:183) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.get(FastThreadLocal.java:142) app//org.apache.hadoop.hbase.ipc.IPCUtil.execute(IPCUtil.java:304) app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown(NettyRpcConnection.java:146) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:161) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser$StackFrameBuffer.(StackStreamFactory.java:451) java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.initFrameBuffer(StackStreamFactory.java:541) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:368) java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243) java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499) app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97) app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121) app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159) app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142) app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022) app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875) app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63) Potentially hanging thread: FifoRpcScheduler.handler-pool-0 java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.unlinkCancelledWaiters(AbstractQueuedSynchronizer.java:1547) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1639) java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9187 (was 9188) 2024-11-23T18:41:37,078 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9187 2024-11-23T18:41:37,089 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:37,089 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:37,091 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387297091,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:37,093 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:37,094 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44691 2024-11-23T18:41:37,094 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:37,094 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:37,094 INFO [Listener,port=44691 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44691: starting 2024-11-23T18:41:37,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b428e61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:37,095 DEBUG [Thread-102 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:37,096 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,099 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387297099,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,100 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,101 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:37,101 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:37,102 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:37,102 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:37,102 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:37,102 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:37,103 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:37,103 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:37,103 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:37,103 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,103 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:37,103 INFO [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:37,104 INFO [Reader=1,bindAddress=localhost,port=44691 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51854, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:37,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:37,109 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44691 2024-11-23T18:41:37,109 INFO [Listener,port=44691 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44691: stopping 2024-11-23T18:41:37,109 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:37,110 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:37,115 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=66 (was 64) Potentially hanging thread: RpcServer.responder java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:216) java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9187 (was 9187) 2024-11-23T18:41:37,120 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=9187 2024-11-23T18:41:37,129 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:37,129 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:37,131 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387297131,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:37,133 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:37,134 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36757 2024-11-23T18:41:37,134 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:37,134 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:37,134 INFO [Listener,port=36757 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36757: starting 2024-11-23T18:41:37,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dfb7c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:37,135 DEBUG [Thread-105 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:37,136 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,139 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387297139,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,140 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,141 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:37,141 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:37,142 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:37,142 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:37,142 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:37,142 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:37,143 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:37,143 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:37,143 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,143 DEBUG [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:37,143 INFO [Reader=1,bindAddress=localhost,port=36757 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:37,143 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:37,144 INFO [Reader=1,bindAddress=localhost,port=36757 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41752, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:37,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:37,151 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36757 2024-11-23T18:41:37,151 INFO [Listener,port=36757 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36757: stopping 2024-11-23T18:41:37,152 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:37,152 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:37,152 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:37,153 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33517 2024-11-23T18:41:37,154 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:37,154 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:37,154 INFO [Listener,port=33517 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33517: starting 2024-11-23T18:41:37,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aaff2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:37,155 DEBUG [Thread-108 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:37,156 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,157 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,158 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:37,158 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:37,158 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:37,159 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:37,159 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:37,159 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:37,160 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:37,160 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:37,160 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:37,160 DEBUG [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:37,160 INFO [Reader=1,bindAddress=localhost,port=33517 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:37,160 INFO [Reader=1,bindAddress=localhost,port=33517 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59334, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,164 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:39,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,165 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:33517 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-23T18:41:39,165 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33517 2024-11-23T18:41:39,165 INFO [Listener,port=33517 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33517: stopping 2024-11-23T18:41:39,165 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:39,166 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:39,178 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67 (was 65) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=9184 (was 9187) 2024-11-23T18:41:39,183 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9183 2024-11-23T18:41:39,192 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,192 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,195 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299194,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,196 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,197 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38041 2024-11-23T18:41:39,198 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:39,198 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,198 INFO [Listener,port=38041 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38041: starting 2024-11-23T18:41:39,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e1e1bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,199 DEBUG [Thread-111 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,200 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,201 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732387299195 2024-11-23T18:41:39,203 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387299203,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,205 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,205 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:39,205 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,206 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,207 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:39,207 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,207 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,208 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:39,208 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,208 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,208 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,208 DEBUG [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:39,208 INFO [Reader=1,bindAddress=localhost,port=38041 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,209 INFO [Reader=1,bindAddress=localhost,port=38041 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42746, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,214 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38041 2024-11-23T18:41:39,215 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:39,215 INFO [Listener,port=38041 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38041: stopping 2024-11-23T18:41:39,215 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:39,220 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69 (was 67) Potentially hanging thread: RpcServer.responder java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:198) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9183 (was 9183) 2024-11-23T18:41:39,225 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9182 2024-11-23T18:41:39,234 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,234 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,237 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299236,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,239 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,240 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33231 2024-11-23T18:41:39,240 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:39,240 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,240 INFO [Listener,port=33231 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33231: starting 2024-11-23T18:41:39,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48688786, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,241 DEBUG [Thread-114 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,242 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,245 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387299245,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,246 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,247 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:39,247 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,247 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,248 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:39,248 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,248 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,248 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS 2024-11-23T18:41:39,249 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,249 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,249 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,249 DEBUG [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:39,249 INFO [Reader=1,bindAddress=localhost,port=33231 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,249 INFO [Reader=1,bindAddress=localhost,port=33231 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48314, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,255 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33231 2024-11-23T18:41:39,255 INFO [Listener,port=33231 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33231: stopping 2024-11-23T18:41:39,255 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:39,255 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:39,260 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 68) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcServer.responder java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndExchangeByte(Unsafe.java:1535) java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetByte(Unsafe.java:1544) java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetBoolean(Unsafe.java:1784) java.base@17.0.11/java.lang.invoke.VarHandleBooleans$FieldInstanceReadWrite.compareAndSet(VarHandleBooleans.java:178) java.base@17.0.11/java.lang.invoke.VarHandleGuards.guard_LII_Z(VarHandleGuards.java:137) java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:125) app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9182 (was 9182) 2024-11-23T18:41:39,266 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9182 2024-11-23T18:41:39,274 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,275 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,277 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299276,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,279 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,280 INFO [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45777 2024-11-23T18:41:39,281 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting 2024-11-23T18:41:39,281 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,281 INFO [Listener,port=45777 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45777: starting 2024-11-23T18:41:39,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@623dac1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,282 DEBUG [Thread-117 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,284 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,286 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387299286,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,288 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,288 INFO [Reader=1,bindAddress=localhost,port=45777 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-23T18:41:39,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,294 INFO [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45777 2024-11-23T18:41:39,294 INFO [Listener,port=45777 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45777: stopping 2024-11-23T18:41:39,294 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped 2024-11-23T18:41:39,295 INFO [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping 2024-11-23T18:41:39,301 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 69) Potentially hanging thread: RPCClient-NioEventLoopGroup-18-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9180 (was 9182) 2024-11-23T18:41:39,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=70, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9180 2024-11-23T18:41:39,321 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,321 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,323 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299322,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,324 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,325 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,326 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35343 2024-11-23T18:41:39,326 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bbc8708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,327 DEBUG [Thread-118 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,328 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,330 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387299330,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,332 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,332 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,333 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,334 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,334 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,335 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,335 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,335 DEBUG [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:39,335 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,336 INFO [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35558, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,337 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,344 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35343 2024-11-23T18:41:39,351 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72 (was 70) Potentially hanging thread: NettyRpcServer-19-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-19-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=475 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9175 (was 9180) 2024-11-23T18:41:39,357 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72, OpenFileDescriptor=475, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9174 2024-11-23T18:41:39,366 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,366 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,368 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299367,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,369 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,369 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,370 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43385 2024-11-23T18:41:39,370 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e99cbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-23T18:41:39,371 DEBUG [Thread-119 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,372 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,374 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387299374,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,376 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:39,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,376 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:43385 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections. 2024-11-23T18:41:39,376 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43385 2024-11-23T18:41:39,382 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74 (was 72) Potentially hanging thread: NettyRpcServer-20-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-20-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=484 (was 475) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9172 (was 9174) 2024-11-23T18:41:39,389 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74, OpenFileDescriptor=484, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9171 2024-11-23T18:41:39,398 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,398 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,400 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299399,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,401 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,401 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,402 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35707 2024-11-23T18:41:39,402 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bd0cf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,403 DEBUG [Thread-120 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,404 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,407 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387299407,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,408 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,409 DEBUG [NettyRpcServer-21-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,410 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,411 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,411 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,412 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:39,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,412 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:35707 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server 2024-11-23T18:41:39,412 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35707 2024-11-23T18:41:39,419 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76 (was 74) Potentially hanging thread: NettyRpcServer-21-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-21-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=493 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9168 (was 9171) 2024-11-23T18:41:39,425 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9167 2024-11-23T18:41:39,434 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,434 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,436 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299436,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,438 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,438 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,439 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41223 2024-11-23T18:41:39,439 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cfb5440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,440 DEBUG [Thread-121 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,441 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,443 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387299443,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,445 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,445 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,446 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,446 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,447 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,447 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,447 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,447 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,447 DEBUG [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:39,447 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,448 INFO [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33886, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,454 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41223 2024-11-23T18:41:39,455 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,455 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,456 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34827 2024-11-23T18:41:39,456 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6531e50e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,462 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732387299437 2024-11-23T18:41:39,485 DEBUG [Thread-122 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,486 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,487 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,487 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,489 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,490 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,490 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,490 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,490 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,490 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,490 DEBUG [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:39,491 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,491 INFO [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36966, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,497 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34827 2024-11-23T18:41:39,498 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,498 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,499 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46863 2024-11-23T18:41:39,499 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65efcd8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,500 DEBUG [Thread-123 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,501 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,502 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,502 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,503 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,504 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,504 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,505 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,505 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,505 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,505 DEBUG [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:39,505 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,505 INFO [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37388, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,512 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46863 2024-11-23T18:41:39,513 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,513 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,514 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44489 2024-11-23T18:41:39,514 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6823333f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,515 DEBUG [Thread-124 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,516 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,517 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,517 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,518 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,519 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,519 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,519 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,519 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,519 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,519 DEBUG [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int 2024-11-23T18:41:39,520 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,520 INFO [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49160, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,527 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44489 2024-11-23T18:41:39,528 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,528 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,529 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37285 2024-11-23T18:41:39,529 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6af79073, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,530 DEBUG [Thread-125 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,531 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,532 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,532 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,534 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,534 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,535 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,535 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,535 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,535 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,535 DEBUG [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:39,535 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,536 INFO [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47350, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,545 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37285 2024-11-23T18:41:39,551 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86 (was 76) Potentially hanging thread: NettyRpcServer-26-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-23-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-24-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-24-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-26-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-23-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-22-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-25-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-22-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-25-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=538 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9157 (was 9167) 2024-11-23T18:41:39,556 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86, OpenFileDescriptor=538, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9157 2024-11-23T18:41:39,565 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,565 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,567 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299567,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,569 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,569 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,570 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41681 2024-11-23T18:41:39,570 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5602196b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,571 DEBUG [Thread-126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-23T18:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null 2024-11-23T18:41:39,573 WARN [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call 2024-11-23T18:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes 2024-11-23T18:41:39,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,573 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41681 2024-11-23T18:41:39,580 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88 (was 86) Potentially hanging thread: NettyRpcServer-27-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-27-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=547 (was 538) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9154 (was 9157) 2024-11-23T18:41:39,587 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88, OpenFileDescriptor=547, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9154 2024-11-23T18:41:39,595 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,595 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,597 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299596,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,598 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,598 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,599 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43313 2024-11-23T18:41:39,599 WARN [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! ********* 2024-11-23T18:41:39,599 WARN [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients 2024-11-23T18:41:39,599 WARN [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true). 2024-11-23T18:41:39,599 WARN [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user 2024-11-23T18:41:39,599 WARN [Time-limited test {}] ipc.RpcServer(380): impersonation is possible! 2024-11-23T18:41:39,599 WARN [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible, 2024-11-23T18:41:39,599 WARN [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml 2024-11-23T18:41:39,599 WARN [Time-limited test {}] ipc.RpcServer(383): **************************** 2024-11-23T18:41:39,599 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d31da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,600 DEBUG [Thread-127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false 2024-11-23T18:41:39,602 WARN [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1 2024-11-23T18:41:39,602 INFO [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-23T18:41:39,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,608 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43313 2024-11-23T18:41:39,616 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90 (was 88) Potentially hanging thread: NettyRpcServer-28-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-28-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=556 (was 547) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9151 (was 9154) 2024-11-23T18:41:39,623 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90, OpenFileDescriptor=556, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9150 2024-11-23T18:41:39,631 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,632 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,633 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299633,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,635 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,635 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,636 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41377 2024-11-23T18:41:39,636 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e52a280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,637 DEBUG [Thread-128 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,637 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,640 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387299640,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,641 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,642 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,643 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,643 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,643 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,644 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,644 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,644 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,644 DEBUG [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:39,644 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,645 INFO [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44926, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,651 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41377 2024-11-23T18:41:39,658 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92 (was 90) Potentially hanging thread: NettyRpcServer-29-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-29-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=565 (was 556) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9148 (was 9150) 2024-11-23T18:41:39,666 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92, OpenFileDescriptor=565, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9148 2024-11-23T18:41:39,677 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:39,677 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:39,679 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387299679,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:39,681 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,681 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,682 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37771 2024-11-23T18:41:39,682 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f3ac68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,682 DEBUG [Thread-129 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,683 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,686 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387299685,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,687 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,687 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,688 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,689 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,689 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,689 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,689 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,689 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:39,689 DEBUG [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:39,689 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,690 INFO [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58120, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:39,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:39,697 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37771 2024-11-23T18:41:39,698 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:39,698 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:39,699 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:38049 2024-11-23T18:41:39,699 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:39,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5fb8fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:39,700 DEBUG [Thread-130 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:39,701 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,702 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,702 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,704 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:39,704 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,704 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:39,705 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:39,705 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:39,705 DEBUG [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf 2024-11-23T18:41:39,705 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:39,706 INFO [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37668, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:41,708 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false 2024-11-23T18:41:41,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:41,709 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:38049 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException 2024-11-23T18:41:41,709 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:38049 2024-11-23T18:41:41,722 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96 (was 92) Potentially hanging thread: NettyRpcServer-30-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-30-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-31-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-31-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=583 (was 565) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9154 (was 9148) - AvailableMemoryMB LEAK? - 2024-11-23T18:41:41,730 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9153 2024-11-23T18:41:41,740 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:41,740 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:41,742 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387301742,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:41,744 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:41,744 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:41,745 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35691 2024-11-23T18:41:41,745 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:41,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405fc0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:41,746 DEBUG [Thread-131 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:41,747 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,749 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387301749,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,751 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:41,751 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,752 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:41,752 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:41,752 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:41,753 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:41,753 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:41,753 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,753 DEBUG [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:41,753 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:41,753 INFO [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51442, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:41,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:41,759 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35691 2024-11-23T18:41:41,765 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98 (was 96) Potentially hanging thread: NettyRpcServer-32-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-32-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=592 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9151 (was 9153) 2024-11-23T18:41:41,767 WARN [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732387301743 2024-11-23T18:41:41,771 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9150 2024-11-23T18:41:41,784 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:41,784 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:41,786 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387301786,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:41,788 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:41,788 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:41,789 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36639 2024-11-23T18:41:41,789 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:41,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@282ae2e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:41,790 DEBUG [Thread-132 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:41,791 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,793 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387301793,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,794 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:41,794 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,795 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer. 2024-11-23T18:41:41,796 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:41,796 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer. 2024-11-23T18:41:41,797 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse() 2024-11-23T18:41:41,797 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:41,797 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,797 DEBUG [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth 2024-11-23T18:41:41,797 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS) 2024-11-23T18:41:41,797 INFO [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39800, version=4.0.0-alpha-1-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto 2024-11-23T18:41:41,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:41,803 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36639 2024-11-23T18:41:41,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100 (was 98) Potentially hanging thread: NettyRpcServer-33-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-33-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=601 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9148 (was 9150) 2024-11-23T18:41:41,815 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100, OpenFileDescriptor=601, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=9147 2024-11-23T18:41:41,823 INFO [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty. 2024-11-23T18:41:41,823 INFO [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required 2024-11-23T18:41:41,825 INFO [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732387301825,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM 2024-11-23T18:41:41,826 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto 2024-11-23T18:41:41,827 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T18:41:41,827 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37503 2024-11-23T18:41:41,827 INFO [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10 2024-11-23T18:41:41,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a33f7a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T18:41:41,828 DEBUG [Thread-133 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true 2024-11-23T18:41:41,829 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,832 INFO [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732387301832,hbase/localhost for hbase/localhost@EXAMPLE.COM 2024-11-23T18:41:41,834 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true 2024-11-23T18:41:41,834 INFO [NettyRpcServer-34-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto 2024-11-23T18:41:41,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T18:41:41,840 INFO [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37503 2024-11-23T18:41:41,847 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=102 (was 100) Potentially hanging thread: NettyRpcServer-34-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: NettyRpcServer-34-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=610 (was 601) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=9144 (was 9147) 2024-11-23T18:41:41,847 INFO [Time-limited test {}] impl.DefaultInternalKdcServerImpl(102): Default Internal kdc server stopped.